<?xml version="1.0" encoding="utf-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
    <article-id pub-id-type="publisher-id">v20i8e10779</article-id>
    <article-id pub-id-type="pmid">30072361</article-id>
    <article-id pub-id-type="doi">10.2196/10779</article-id>
    <article-categories>
      <subj-group subj-group-type="heading">
        <subject>Original Paper</subject>
      </subj-group>
      <subj-group subj-group-type="article-type">
        <subject>Original Paper</subject>
      </subj-group>
    </article-categories>
    <title-group>
      <article-title>Improving Consumer Understanding of Medical Text: Development and Validation of a New SubSimplify Algorithm to Automatically Generate Term Explanations in English and Spanish</article-title>
    </title-group>
    <contrib-group>
      <contrib contrib-type="editor">
        <name>
          <surname>Eysenbach</surname>
          <given-names>Gunther</given-names>
        </name>
      </contrib>
    </contrib-group>
    <contrib-group>
      <contrib contrib-type="reviewer">
        <name>
          <surname>Qenam</surname>
          <given-names>Basel</given-names>
        </name>
      </contrib>
      <contrib contrib-type="reviewer">
        <name>
          <surname>Mavragani</surname>
          <given-names>Amaryllis</given-names>
        </name>
      </contrib>
    </contrib-group>
    <contrib-group>
      <contrib contrib-type="author" id="contrib1" corresp="yes">
      <name name-style="western">
        <surname>Kloehn</surname>
        <given-names>Nicholas</given-names>
      </name>
      <degrees>MA, PhD</degrees>
      <xref rid="aff1" ref-type="aff">1</xref>
      <address>
        <institution>Department of Linguistics</institution>
        <institution>University of Arizona</institution>
        <addr-line>Communications Building, Room 109</addr-line>
        <addr-line>PO Box 210025</addr-line>
        <addr-line>Tucson, AZ, 85721</addr-line>
        <country>United States</country>
        <phone>1 904 314 4654</phone>
        <fax>1 520 621 8105</fax>
        <email>nick.kloehn@gmail.com</email>
      </address>  
      <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-4359-3331</ext-link></contrib>
      <contrib contrib-type="author" id="contrib2">
        <name name-style="western">
          <surname>Leroy</surname>
          <given-names>Gondy</given-names>
        </name>
        <degrees>MS, PhD</degrees>
        <xref rid="aff1" ref-type="aff">1</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0003-4751-6680</ext-link>
      </contrib>
      <contrib contrib-type="author" id="contrib3">
        <name name-style="western">
          <surname>Kauchak</surname>
          <given-names>David</given-names>
        </name>
        <degrees>PhD</degrees>
        <xref rid="aff2" ref-type="aff">2</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-5208-634X</ext-link>
      </contrib>
      <contrib contrib-type="author" id="contrib4">
        <name name-style="western">
          <surname>Gu</surname>
          <given-names>Yang</given-names>
        </name>
        <degrees>BA</degrees>
        <xref rid="aff1" ref-type="aff">1</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-4107-3578</ext-link>
      </contrib>
      <contrib contrib-type="author" id="contrib5">
        <name name-style="western">
          <surname>Colina</surname>
          <given-names>Sonia</given-names>
        </name>
        <degrees>PhD</degrees>
        <xref rid="aff3" ref-type="aff">3</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-4893-9047</ext-link>
      </contrib>
      <contrib contrib-type="author" id="contrib6">
        <name name-style="western">
          <surname>Yuan</surname>
          <given-names>Nicole P</given-names>
        </name>
        <degrees>MA, MPH, PhD</degrees>
        <xref rid="aff4" ref-type="aff">4</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-2441-4692</ext-link>
      </contrib>
      <contrib contrib-type="author" id="contrib7">
        <name name-style="western">
          <surname>Revere</surname>
          <given-names>Debra</given-names>
        </name>
        <degrees>MA, MLIS</degrees>
        <xref rid="aff5" ref-type="aff">5</xref>
        <ext-link ext-link-type="orcid">http://orcid.org/0000-0002-3863-7774</ext-link>
      </contrib>
    </contrib-group>
    <aff id="aff1">
    <sup>1</sup>
    <institution>Department of Linguistics</institution>
    <institution>University of Arizona</institution>  
    <addr-line>Tucson, AZ</addr-line>
    <country>United States</country></aff>
    <aff id="aff2">
    <sup>2</sup>
    <institution>Computer Science Department</institution>
    <institution>Pomona College</institution>  
    <addr-line>Claremont, CA</addr-line>
    <country>United States</country></aff>
    <aff id="aff3">
    <sup>3</sup>
    <institution>Department of Spanish and Portuguese</institution>
    <institution>University of Arizona</institution>  
    <addr-line>Tucson, AZ</addr-line>
    <country>United States</country></aff>
    <aff id="aff4">
    <sup>4</sup>
    <institution>Health Promotion Sciences Division</institution>
    <institution>Mel and Enid Zuckerman College of Public Health</institution>  
    <institution>University of Arizona</institution>  
    <addr-line>Tucson, AZ</addr-line>
    <country>United States</country></aff>
    <aff id="aff5">
    <sup>5</sup>
    <institution>Department of Health Services</institution>
    <institution>School of Public Health</institution>  
    <institution>University of Washington</institution>  
    <addr-line>Seattle, WA</addr-line>
    <country>United States</country></aff>
    <author-notes>
      <corresp>Corresponding Author: Nicholas Kloehn 
      <email>nick.kloehn@gmail.com</email></corresp>
    </author-notes>
    <pub-date pub-type="collection"><month>08</month><year>2018</year></pub-date>
    <pub-date pub-type="epub">
      <day>02</day>
      <month>08</month>
      <year>2018</year>
    </pub-date>
    <volume>20</volume>
    <issue>8</issue>
    <elocation-id>e10779</elocation-id>
    <!--history from ojs - api-xml-->
    <history>
      <date date-type="received">
        <day>13</day>
        <month>4</month>
        <year>2018</year>
      </date>
      <date date-type="rev-request">
        <day>23</day>
        <month>5</month>
        <year>2018</year>
      </date>
      <date date-type="rev-recd">
        <day>13</day>
        <month>6</month>
        <year>2018</year>
      </date>
      <date date-type="accepted">
        <day>16</day>
        <month>6</month>
        <year>2018</year>
      </date>
    </history>
    <!--(c) the authors - correct author names and publication date here if necessary. Date in form ', dd.mm.yyyy' after jmir.org-->
    <copyright-statement>©Nicholas Kloehn, Gondy Leroy, David Kauchak, Yang Gu, Sonia Colina, Nicole P Yuan, Debra Revere. Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 02.08.2018.</copyright-statement>
    <copyright-year>2018</copyright-year>
    <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
      <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on http://www.jmir.org/, as well as this copyright and license information must be included.</p>
    </license>  
    <self-uri xlink:href="http://www.jmir.org/2018/8/e10779/" xlink:type="simple"/>
    <abstract>
      <sec sec-type="background">
        <title>Background</title>
        <p>While health literacy is important for people to maintain good health and manage diseases, medical educational texts are often written beyond the reading level of the average individual. To mitigate this disconnect, text simplification research provides methods to increase readability and, therefore, comprehension. One method of text simplification is to isolate particularly difficult terms within a document and replace them with easier synonyms (lexical simplification) or an explanation in plain language (semantic simplification). Unfortunately, existing dictionaries are seldom complete, and consequently, resources for many difficult terms are unavailable. This is the case for English and Spanish resources.</p>
      </sec>
      <sec sec-type="objective">
        <title>Objective</title>
        <p>Our objective was to automatically generate explanations for difficult terms in both English and Spanish when they are not covered by existing resources. The system we present combines existing resources for explanation generation using a novel algorithm (SubSimplify) to create additional explanations.</p>
      </sec>
      <sec sec-type="methods">
        <title>Methods</title>
        <p>SubSimplify uses word-level parsing techniques and specialized medical affix dictionaries to identify the morphological units of a term and then source their definitions. While the underlying resources are different, SubSimplify applies the same principles in both languages. To evaluate our approach, we used term familiarity to identify difficult terms in English and Spanish and then generated explanations for them. For each language, we extracted 400 difficult terms from two different article types (General and Medical topics) balanced for frequency. For English terms, we compared SubSimplify’s explanation with the explanations from the Consumer Health Vocabulary, WordNet Synonyms and Summaries, as well as Word Embedding Vector (WEV) synonyms. For Spanish terms, we compared the explanation to WordNet Summaries and WEV Embedding synonyms. We evaluated quality, coverage, and usefulness for the simplification provided for each term. Quality is the average score from two subject experts on a 1-4 Likert scale (two per language) for the synonyms or explanations provided by the source. Coverage is the number of terms for which a source could provide an explanation. Usefulness is the same expert score, however, with a 0 assigned when no explanations or synonyms were available for a term.</p>
      </sec>
      <sec sec-type="results">
        <title>Results</title>
        <p>SubSimplify resulted in quality scores of 1.64 for English (<italic>P</italic>&#60;.001) and 1.49 for Spanish (<italic>P</italic>&#60;.001), which were lower than those of existing resources (Consumer Health Vocabulary [CHV]=2.81). However, in coverage, SubSimplify outperforms all existing written resources, increasing the coverage from 53.0% to 80.5% in English and from 20.8% to 90.8% in Spanish (<italic>P</italic>&#60;.001). This result means that the usefulness score of SubSimplify (1.32; <italic>P</italic>&#60;.001) is greater than that of most existing resources (eg, CHV=0.169).</p>
      </sec>
      <sec sec-type="conclusions">
        <title>Conclusions</title>
        <p>Our approach is intended as an additional resource to existing, manually created resources. It greatly increases the number of difficult terms for which an easier alternative can be made available, resulting in greater actual usefulness.</p>
      </sec>
    </abstract>
    <kwd-group>
      <kwd>text simplification</kwd>
      <kwd>health literacy</kwd>
      <kwd>natural language processing</kwd>
      <kwd>terminology</kwd>
    </kwd-group></article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background and Significance</title>
        <p>Text is an important resource for health-related information as it is easy to create and distribute. Furthermore, health literature is widely available in the form of web-based resources for people to obtain information on medical conditions, diseases, and modalities [<xref ref-type="bibr" rid="ref1">1</xref>]. However, these documents are often written at a level beyond the comprehension of the average reader [<xref ref-type="bibr" rid="ref2">2</xref>]. This disconnect reflects an overall trend in misinformation regarding health conditions [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>].</p>
        <p>To mitigate this problem, researchers have sought automatic ways to improve the readability of these texts and the resulting reader comprehension. This natural language programming (NLP) task is known as <italic>text simplification</italic> [<xref ref-type="bibr" rid="ref5">5</xref>] and has been used to create supervised [<xref ref-type="bibr" rid="ref6">6</xref>], semisupervized [<xref ref-type="bibr" rid="ref7">7</xref>], and fully automatic tools [<xref ref-type="bibr" rid="ref8">8</xref>] to make texts easier for consumers to digest by increasing readability [<xref ref-type="bibr" rid="ref9">9</xref>]. A central challenge for this research is to develop resources and techniques that enhance the quality and accuracy of these systems. Even though deep neural network approaches and other automated translation algorithms are increasingly being developed, it will take time before they can be applied with sufficient impact and precise simplifications. We intend for our algorithm to supplement existing resources as well as generate useful inputs for other algorithms.</p>
        <p>The first step is identifying what makes text difficult. Some of the previous studies have focused on simplifying individual terms, while others have focused on grammatical structures. To identify the difficulty of individual terms, we use <italic>term familiarity</italic>. For a given term, this measure can be calculated by extracting the likelihood that a term occurs in common language usage [<xref ref-type="bibr" rid="ref10">10</xref>], which we estimate according to the term’s frequency in the Google Web Corpus [<xref ref-type="bibr" rid="ref11">11</xref>]. In this work, we add to the body of research that identifies these terms and replaces them with easier synonyms [<xref ref-type="bibr" rid="ref12">12</xref>]. However, we went beyond existing approaches by generating new explanations for terms that do not exist in the available resources. To do this, we developed and evaluated a new algorithm to generate new explanations. We generated explanations of terms in plain language using word internal parsing and affix dictionaries with SubSimplify.</p>
      </sec>
      <sec>
        <title>Resources for Finding Explanations for Difficult Terms</title>
        <p>Ideally, there would be an endless resource of expert-written explanations for difficult terms, optimized for the general public in multiple languages. However, few resources are able to provide appropriate explanations at all and even fewer are able to automatically or semiautomatically produce such explanations.</p>
        <p>The resource closest to ideal is the English Consumer Health Vocabulary (CHV) [<xref ref-type="bibr" rid="ref13">13</xref>], which is included in the Unified Medical Language System (UMLS) [<xref ref-type="bibr" rid="ref14">14</xref>]. This resource was manually created and provides synonyms as well as definitions for medical terms in a consumer-friendly language. For the purposes of text simplification, these plain language definitions and simple synonyms double as ready-made explanations for difficult terms. However, the number of explanations is low relative to the overall number of difficult terms that occur in a given medical text. The CHV contains 2567 unique definitions and 88,529 synonyms for concepts found in the UMLS. We did not employ the UMLS as a resource because this system focuses on mapping complex medical concepts onto ontologies and is not designed to relate health information to patients or any other person outside the medical domain.</p>
        <p>Previous research has shown that the CHV can be used to simplify texts [<xref ref-type="bibr" rid="ref15">15</xref>-<xref ref-type="bibr" rid="ref17">17</xref>], but it has also been shown to contain jargon words and not enough consumer-friendly vocabulary when providing summaries for specialized research [<xref ref-type="bibr" rid="ref18">18</xref>]. Furthermore, while this resource is well tailored to text simplification, it is limited to English terms and explanations. In summary, the CHV provides explanations that can be automatically sourced in a given simplification system. However, CHV is only in English, is for relatively few terms, and can at times contain jargon beyond the reading level of the average reader.</p>
        <p>While not being medically focused, WordNet is a useful resource for text simplification. It is an online lexical database containing terms and definitions, as well as interword semantic relations such as hypernyms, hyponyms, synonyms, and antonyms [<xref ref-type="bibr" rid="ref19">19</xref>]. WordNet provides 128,391 word-sense definitions in English and is also available in Spanish, albeit in a less complete form [<xref ref-type="bibr" rid="ref20">20</xref>]. Since WordNet is not a medical resource, many of its explanations are not optimal for medical text simplification, and when several senses are provided for a word, it is not always clear which best suits the medical sense. Previously, WordNet has been used to provide synonyms for lexical simplification [<xref ref-type="bibr" rid="ref21">21</xref>]. For example, hyponym-hypernym relations have been used to generate synonyms that are simpler (more general) for text simplification [<xref ref-type="bibr" rid="ref22">22</xref>]. In other areas, this resource has been used to simplify texts in the domain of biomolecules [<xref ref-type="bibr" rid="ref23">23</xref>] and in texts written for non-native English speakers [<xref ref-type="bibr" rid="ref24">24</xref>]. In summary, while WordNet is larger than CHV and also available in Spanish, the resource is not always optimal for giving the definition for medical terms.</p>
        <p>Recent developments using neural networks trained on large bodies of text have produced larger resources such as word embeddings, where words are represented by multidimensional word vectors. The resulting vectors position the word relative to each other in a multidimensional space and have been shown to possess semantic and syntactic relations that allow us to automatically find synonyms and semantically related terms [<xref ref-type="bibr" rid="ref12">12</xref>]. Given a word, we can use its vector representation to find the word whose vector is nearest to this word. Often, this nearest vector is a synonymous word. One freely available version of this resource is the pretrained Global Vectors for Word Representation (also known as GLoVe) [<xref ref-type="bibr" rid="ref25">25</xref>]. Prior work has shown that these vectors can prove to be useful to isolate simple yet more frequent terms in the areas of text simplification [<xref ref-type="bibr" rid="ref26">26</xref>]. However, they can include spurious matches because the approach cannot differentiate antonyms from synonyms. Given that this resource is totally automated, a word vector model can be produced from any language given a relatively large body of text. This means that this resource is also available for Spanish, with pretrained vectors available online [<xref ref-type="bibr" rid="ref27">27</xref>]. In our study, we employed the GLoVe pretrained vectors for English and for Spanish [<xref ref-type="bibr" rid="ref27">27</xref>], labeling the approach more generally as Word Embedding Vectors (WEV).</p>
        <p>In all, the methods that exist for explanation generation range from specific, and precise, with low coverage to high coverage, with a much lower relative accuracy. In the next section, we describe our approach, which exists on the spectrum between these resources.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Using Morphological Information to Generate Explanations</title>
        <p>We first describe the role that morphological units play in medical terminology and then our algorithm, which extracts information and generates explanations using these morphological units.</p>
        <p>The resources described above make use of a word’s definition in isolation without reference to the internal characteristics of that word, (ie, the morphology of the word). While it is not always the case, often romance languages contain morphological units that contain relatively clear semantics, such as the case for the prefix <italic>anti</italic>- (“against”), or the suffix -<italic>s</italic> (indicating plural). In certain words in English and in Spanish, these can help one to decipher the meaning of a word. In medicine, many terms, both in English and Spanish, originate from Greek and Latin [<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref29">29</xref>]. Greek and Latin affixes have meanings commonly unknown to the average reader, but they nevertheless reflect the overall meaning of a word. While at times the meaning of a word is a direct function of the composition of the meaning of these morphological units, to a large degree in English and Spanish, terms composed of these units tend to have a gestalt effect. On the extreme end, a term may completely differ from the meaning of its morphological units (eg, “ledger” does not mean “ledge”+<italic>er</italic>). However, this problem of semantic drift is small for medical terms, seemingly because medical terms are less affected by semantic drift than more nonmedical, frequent terms.</p>
        <p>Affixes that compose medical terms commonly have clear definitions that reflect a word’s meaning. For example, given the prefix <italic>cardio-</italic> we know that this term’s meaning relates to “the heart.” Several resources containing these affixes and their definitions are freely available online [<xref ref-type="bibr" rid="ref30">30</xref>-<xref ref-type="bibr" rid="ref33">33</xref>]. From these, we created a unique dictionary of affixes along with their definitions for each language. We extracted 586 unique affixes for English and 498 affixes for Spanish. We define an affix as any morphological unit that has some denotation apart from the word itself. Affixes are categorized by their position, with prefixes occurring at the beginning of the words and suffixes occurring at the end of words. A root is any morphological unit that can stand alone as a single word. For example, the term <italic>cardiovascular</italic> contains the prefix <italic>cardio-</italic>, and the root <italic>vascular</italic>. Independently, these morphological units denote <italic>the heart</italic> and <italic>consisting of a vessel or vessels</italic>, respectively. Although many resources may not contain a definition for <italic>cardiovascular</italic>, by parsing these morphological units, we can automatically generate an explanation that reflects the actual denotation of the term: <italic>relating to the heart and blood vessels</italic>.</p>
        <p>In both Spanish and English, words may be composed of multiple suffixes, roots, and prefixes. SubSimplify exploits this fact to generate an explanation for a term. <xref ref-type="table" rid="table1">Table 1</xref> shows the examples of affixes and their definitions in both Spanish and English.</p>
        <p>In addition to these affix dictionaries, we use word stemming [<xref ref-type="bibr" rid="ref34">34</xref>] to isolate stemmed, or lemmatized, versions of terms. Stemming and lemmatization are two different methods of reducing a term to something similar to its root, but in a way that does not always reflect the actual root. For example, a resource like WordNet may have a definition for <italic>Gastrointestine</italic>, but not <italic>Gastroinstestinal</italic>. By stemming and stripping the affix <italic>-al</italic>, we increase the ability to find explanations using all resources.</p>
        <p><xref ref-type="fig" rid="figure1">Figure 1</xref> provides an overview of our SubSimplify algorithm. The input to SubSimplify is a term we assume to be difficult, and we recursively lookup affixes and generate an explanation by accumulating the definitions of each affix and root identified. When finished, we align these definitions to provide an explanation of the term.</p>
        <p>We use affix dictionaries to identify morphological units programmatically. First, the system identifies affixes and then takes the part of the word that is not an affix and performs a database lookup on stemmed variants of the term. To avoid spurious matches, we work from larger to smaller suffixes, and thus, <italic>anti-</italic> as in <italic>anti-hero</italic> would match before <italic>a-</italic> as in <italic>a-symmetry</italic>. This process occurs iteratively until no affixes are matched, or until there is no root left. In order to describe this process in sequence, <xref ref-type="boxed-text" rid="box1">Textbox 1</xref> gives a detailed description of each step.</p>
        <p>Since words may contain multiple suffixes, the process occurs multiple times where possible. That is, when we extract a root, it is possible that that root may yet contain another suffix or prefix. To highlight this, we provide an example with the term <italic>hyperglycemic</italic> in <xref ref-type="boxed-text" rid="box2">Textbox 2</xref>.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Examples of affixes and corresponding definitions in English and Spanish.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="270"/>
            <col width="190"/>
            <col width="360"/>
            <col width="150"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Language and affix</td>
                <td>Type</td>
                <td>Definition</td>
                <td>Origin</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="2"><bold>English</bold></td>
                <td><break/></td>
                <td><break/></td>
                <td><break/></td>
              </tr>
              <tr valign="top">
                <td><break/></td>
                <td>adip-</td>
                <td>prefix</td>
                <td><italic>Of or relating to fat or fatty tissue</italic></td>
                <td>Latin</td>
              </tr>
              <tr valign="top">
                <td><break/></td>
                <td>-dipsia</td>
                <td>suffix</td>
                <td><italic>(condition of) thirst</italic></td>
                <td>Greek</td>
              </tr>
              <tr valign="top">
                <td colspan="2"><bold>Spanish</bold></td>
                <td><break/></td>
                <td><break/></td>
                <td><break/></td>
              </tr>
              <tr valign="top">
                <td><break/></td>
                <td>pireto-</td>
                <td>prefix</td>
                <td><italic>Forma prefija que significa fiebre</italic></td>
                <td>Latin or Greek</td>
              </tr>
              <tr valign="top">
                <td><break/></td>
                <td>-opsia</td>
                <td>suffix</td>
                <td><italic>Forma sufija que significa visión</italic></td>
                <td>Greek</td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>SubSimplify flow diagram.</p>
          </caption>
          <graphic xlink:href="jmir_v20i8e10779_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <boxed-text id="box1" position="float">
          <title>A description of each step in the SubSimplify algorithm.</title>
          <p>Affix Identification:</p>
          <p>All affixes in the affix dictionary are compared to the term from the longest to shortest length. If the term contains the affix characters at the beginning (for prefixes) or ending (for suffixes), the system considers this an affix match.</p>
          <p>Affix Definition:</p>
          <p>For each affix match, the affix dictionary definition is added to the newly constructed explanation.</p>
          <p>Root Extraction:</p>
          <p>The root of the term is extracted by removing the prefix or suffix. Since this can remove some of the characters of the root unintentionally, we consider the root the remaining characters plus single character variations of the root at the edge where the term was matched.</p>
          <p>Search Resources:</p>
          <p>The extracted root is then searched in WordNet and in CHV. If it is not found, we reintroduce the root to this same process until no matches are found.</p>
        </boxed-text>
        <boxed-text id="box2" position="float">
          <title>SubSimplify algorithm application to hyperglycemic.</title>
          <p>Affix Identification:</p>
          <p>We iteratively go through the affix dictionary and match the prefix <italic>hyper-</italic> in <italic>hyperglycemic</italic>.</p>
          <p>Affix Definition:</p>
          <p>The definition for <italic>hyper</italic> —“<italic>denotes something as extreme or beyond normal”</italic> —is added to the explanation for the term <italic>hyperglycemic</italic>.</p>
          <p>Root Extraction:</p>
          <p>We extract <italic>glycemic</italic> from <italic>hyperglycemic.</italic></p>
          <p>Search Resources:</p>
          <p>WordNet and CHV are searched for <italic>glycemic</italic> and all single character variants of <italic>glycemic</italic> (eg, <italic>aglycemic</italic>). When not found, we rerun this entire process on <italic>glycemic</italic>, saving the explanation so far.</p>
        </boxed-text>
        <table-wrap position="float" id="table2">
          <label>Table 2</label>
          <caption>
            <p>Example English explanations.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="250"/>
            <col width="180"/>
            <col width="570"/>
            <thead>
              <tr valign="top">
                <td>Explanation resource</td>
                <td>Example term</td>
                <td>Explanation</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>CHV<sup>a</sup></td>
                <td><italic>pheochromocytoma</italic></td>
                <td>A usually benign, well-encapsulated, lobular, vascular tumor of chromaffin tissue of the Adrenal Medulla</td>
              </tr>
              <tr valign="top">
                <td>WordNet Summary</td>
                <td><italic>Coryza</italic></td>
                <td>an inflammation of the mucous membrane lining the nose (usually associated with nasal discharge)</td>
              </tr>
              <tr valign="top">
                <td>WordNet Synonym</td>
                <td><italic>attenuated</italic></td>
                <td>rarefy</td>
              </tr>
              <tr valign="top">
                <td>SubSimplify</td>
                <td><italic>hyperglycemic</italic></td>
                <td>hyper-glyc-em-ic, “extreme” or “beyond normal”-sugar-em-pertaining to</td>
              </tr>
              <tr valign="top">
                <td>Word Vector Nearest Neighbor</td>
                <td><italic>toxoplasma</italic></td>
                <td>gondii</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table2fn1">
              <p><sup>a</sup>CHV: Consumer Health Vocabulary.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>This process repeats until there is either no root left, or until the remaining root fails to be identified by any resource. For <italic>glycemic</italic>, the system will identify <italic>-ic</italic> and subsequently <italic>glyc-</italic> before halting at <italic>-em-</italic>.</p>
        <p>If the term contains “-” or any other <italic>nonword</italic> characters, we split these as well. The parsed affixes and roots are then aligned with their explanations to provide an affix-by-affix breakdown of the term. For any affix that is not identified in the system, as is the case with <italic>-em-</italic> in <italic>hyperglycemic</italic>, the definition of the root remains the root itself. Upon presenting the term, these affixes are matched with their definition both by order and by color in order to make identification as easy as possible for a writer. An example explanation for <italic>hyperglycemic</italic> is shown in <xref ref-type="table" rid="table2">Table 2</xref>. This table contains explanations for a few different difficult terms to highlight their quality when present. Note that not all resources contain explanations for all terms, so it is extremely rare that all resources can provide an explanation for a single term.</p>
        <p>While the CHV [<xref ref-type="bibr" rid="ref35">35</xref>] and WordNet Summary resources provided full-sentence explanations (semantic simplification), the WEV and WordNet Synonym provide single-word explanations of each term (lexical simplification). SubSimplify provides a hybrid of the two: for the individual parsed subword units, either a synonym or brief description is presented.</p>
        <p>Next, we describe 2 studies designed to evaluate the quality, coverage, and usefulness of these explanations in English and Spanish.</p>
      </sec>
      <sec>
        <title>Studies</title>
        <p>To evaluate the quality, coverage, and usefulness of the newly generated explanations and how they compare to existing resources, we conducted two studies: one in English and one in Spanish.</p>
        <sec>
          <title>Study 1: English Term Explanation Generation</title>
          <sec>
            <title>Study Stimuli</title>
            <sec>
              <title>Stimuli</title>
              <p>To obtain a range of medical terms that occur in common texts, we extracted 20 documents from Wikipedia written on a medical topic and 100 PubMed abstracts. From these documents, we extracted the difficult terms using term familiarity. For the purposes of this study, we identified difficult terms as those having a frequency less than the 5000th ranked term in the Google Web Corpus, which previous work showed to be a reasonable criterion [<xref ref-type="bibr" rid="ref7">7</xref>]. Given these difficult terms, we selected 200 terms from each resource type (PubMed and Wikipedia) balanced across all documents (100 and 20, respectively). To investigate the effect of frequency, we also balanced each set of 200 difficult terms by frequency. Two groups were extracted based upon high and low frequencies. High-frequency terms were those which had frequencies in the upper most tertile, and low-frequency terms were those which had frequencies in the lowest tertile. In all, the study contained 400 total terms that were evenly split across high and low frequency, document source, and the documents themselves.</p>
            </sec>
            <sec>
              <title>Explanation Generation</title>
              <p>We compared our approach to four previous approaches: CHV, WordNet Synonyms and Summaries, and WEV. These resources provided explanations when an exact match could be found for the term in their database.</p>
            </sec>
          </sec>
          <sec>
            <title>Metrics</title>
            <p>For each of the 400 terms, we calculated 3 metrics: quality, coverage, and usefulness. Quality was judged by subject experts (SEs). The SEs in this study were required to (1) be a native speaker of the language and (2) have at least a master’s degree in a public health or a medical-related field. The experts typically had experience evaluating the quality of medical resources, and for this study, they were financially compensated for their time.</p>
            <p>For quality<italic>,</italic> the two SEs reviewed each term along with the candidate definitions and explanations. For each definition or explanation, the SEs annotated how useful it was on a 4-point Likert scale. <xref ref-type="table" rid="table3">Table 3</xref> provides a description of each rating level. Coverage was measured by calculating the percentage of terms for which an explanation was provided by each source. Usefulness is a broader measure than quality and takes the availability of terms and resources into account. When a term is not found, it receives a score of 0. While quality gives us an idea of how accurate resource explanations are, usefulness tells us how well such a resource would perform if we were to employ it for all terms.</p>
          </sec>
          <sec>
          <title>Procedure</title>
          <p>The SEs evaluated the 400 terms and the corresponding explanations provided by each resource. The order of the presentation of explanations was randomized for each term. For each of the terms, the SEs scored the term on both the quality and coverage metrics described above. We then calculated usefulness by normalizing quality by coverage.</p>
          <p>In order to give a visual idea of how this study was performed, <xref ref-type="fig" rid="figure2">Figure 2</xref> contains a flowchart containing the steps of the study.</p>  
          <table-wrap position="float" id="table3">
            <label>Table 3</label>
            <caption>
              <p>Likert quality scale.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="250"/>
              <col width="750"/>
              <thead>
                <tr valign="top">
                  <td>Rating</td>
                  <td>Description</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>1</td>
                  <td>Explanation <italic>is not useful</italic> to someone annotating the text.</td>
                </tr>
                <tr valign="top">
                  <td>2</td>
                  <td>Explanation <italic>is a little useful</italic> to someone annotating the text.</td>
                </tr>
                <tr valign="top">
                  <td>3</td>
                  <td>Explanation <italic>is useful</italic> to someone annotating the text.</td>
                </tr>
                <tr valign="top">
                  <td>4</td>
                  <td>Explanation <italic>is very useful</italic> to someone annotating the text.</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
          <fig id="figure2" position="float">
            <label>Figure 2</label>
            <caption>
              <p>Steps for English term explanation generation study.</p>
            </caption>
            <graphic xlink:href="jmir_v20i8e10779_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <table-wrap position="float" id="table4">
            <label>Table 4</label>
            <caption>
              <p>English study results.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="250"/>
              <col width="140"/>
              <col width="180"/>
              <col width="180"/>
              <col width="140"/>
              <col width="110"/>
              <thead>
                <tr valign="bottom">
                  <td>Metrics</td>
                  <td>CHV<sup>a</sup></td>
                  <td>WordNet Synonym</td>
                  <td>WordNet Summary</td>
                  <td>SubSimplify</td>
                  <td>WEV<sup>b</sup></td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Quality (1-4 scale)</td>
                  <td>2.81</td>
                  <td>2.09</td>
                  <td>3.32</td>
                  <td>1.64</td>
                  <td>1.64</td>
                </tr>
                <tr valign="top">
                  <td>Coverage (N=400), %</td>
                  <td>6.0</td>
                  <td>53.0</td>
                  <td>53.0</td>
                  <td>80.5</td>
                  <td>83.8</td>
                </tr>
                <tr valign="top">
                  <td>Usefulness (0-4 scale)</td>
                  <td>0.169</td>
                  <td>1.11</td>
                  <td>1.76</td>
                  <td>1.32</td>
                  <td>1.38</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table4fn1">
                <p><sup>a</sup>CHV: Consumer Health Vocabulary.</p>
              </fn>
              <fn id="table4fn2">
                <p><sup>b</sup>WEV: Word Embedding Vector.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap></sec>
          <sec>
            <title>Evaluation Outcomes</title>
            <sec>
              <title>Interoperator Variability</title>
              <p>To compare the variability in quality scores between each of the SEs, we calculated Crohnbach alpha. Since we did not limit the quality ratings to a rank order, it was possible for each term to have multiple explanations that received the best score per term. Therefore, we calculated Crohnbach alpha in two ways. First, in a conservative version, we calculated whether each SE chose all of the same explanations as the best for each term, and in a more liberal version, whether each SE chose one of the same explanations as the best for each term. For English, the results were 0.69 and 0.90 for the conservative and liberal version, respectively. We, therefore, determined that interoperator reliability was high enough to average their ratings. <xref ref-type="table" rid="table4">Table 4</xref> shows the results of the quality, coverage, and usefulness metrics for each explanation source in English.</p>
              <p>In <xref ref-type="table" rid="table4">Table 4</xref>, we see that each column represents explanation sources and the 3 rows give the metrics averaged across SEs. For example, CHV received a mean quality score of 2.81 when present, but could only provide explanations for 24 out of 400 total terms. Subsequently, its usefulness was only 0.169 for the 400 terms. Recall that this resource represents the one that is manually generated to aid lexical simplification in medical documents. As a consequence, the quality rating was relatively high, but the coverage was by far the lowest. Next, we see that WordNet Summaries and Synonyms each provided the same number of explanations. However, the Summaries (Semantic Simplification) scored much higher than the Synonyms (Lexical Simplification) at 3.32 versus 2.09, respectively. Again, given that they only provide explanations for 212 terms, their usefulness was only 1.76 and 1.11, respectively. While SubSimplify had a 1.64 quality score when present, its coverage was 322, whereas that of WordNet was 212, representing an increase from 53.0% to 80.5% in coverage of the 400 <italic>difficult terms</italic>. Consequently, the usefulness of SubSimplify was 1.32, greater than that of WordNet Synonyms and CHV. Last, WEV provided the greatest coverage and performed identically to SubSimplify in quality (1.64), but had greater coverage (335) and quality (1.38). However, as we describe in the next subsection, there was a clear difference between the quality performance of SubSimplify and WEV. SubSimplify performed better with lower-frequency words and in more technical literature than WEV.</p>
            </sec>
            <sec>
              <title>Quality</title>
              <p>To evaluate significance, we performed a 2×2×5 analysis of variance (ANOVA), with quality as the dependent variable. The independent measures were document source (Wikipedia or PubMed), frequency (Low or High), and the five explanation sources (CHV, WordNet Synonym, WordNet Summary, SubSimplify, and WEV). There were main effects for frequency (<italic>F</italic><sub>1,2186</sub> = 3.859, <italic>P</italic>&#60;.02) and explanation type (<italic>F</italic><sub>4,2186</sub> = 260.1, <italic>P</italic>&#60;.001). This indicates that on average, the resources performed significantly better with lower-frequency terms and that there were significant differences between the resources.</p>
              <p>In addition to the main effects, there was a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>8,2186</sub>=2.993, <italic>P</italic>&#60;.001; <xref ref-type="fig" rid="figure3">Figure 3</xref>). <xref ref-type="fig" rid="figure3">Figure 3</xref> contains the mean quality of each resource at low and high frequencies. Given that our documents contained medical terminology, we expected low-frequency words to be the rarest and, therefore, most technical. They presented the hardest target for any system attempting to summarize these documents. For example, CHV, which is written specifically for medical terms, has much greater performance for low-frequency terms than for high-frequency terms (3.12 vs 1.74). Furthermore, WordNet Synonyms and Summaries both performed slightly better for low-frequency terms than for high-frequency ones. Interestingly, SubSimplify also followed this pattern. However, WEV had the opposite trend. Not only did WEV perform better on high-frequency terms than on low-frequency terms but also it performed slightly poorer than SubSimplify for low-frequency terms (compare SubSimplify’s 1.67 rating to WEV’s 1.63 rating for low-frequency terms).</p>
            </sec>
            <sec>
              <title>Coverage</title>
              <p>To evaluate the effect of frequency and document source on the coverage of each resource, we performed another 2×2×5 ANOVA with coverage as the dependent variable. There were main effects for frequency (<italic>F</italic><sub>1,3970</sub>=3.859, <italic>P</italic>&#60;.001) and explanation type (<italic>F</italic><sub>4,3970</sub>=260.1, <italic>P</italic>&#60;.001; refer to <xref ref-type="table" rid="table2">Table 2</xref>, row 2). This indicates that explanations, on average, had significantly greater coverage for high-frequency terms than for low-frequency terms.</p>
              <p>There was a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>8,3970</sub>=6.557, <italic>P</italic>&#60;.001) and a significant interaction between explanation type and document source (<italic>F</italic><sub>4,3970</sub>=11.523, <italic>P</italic>&#60;.001; <xref ref-type="fig" rid="figure4">Figure 4</xref>).</p>
              <fig id="figure3" position="float">
                <label>Figure 3</label>
                <caption>
                  <p>Explanation type-frequency interaction for quality in English. CHV: Consumer Health Vocabulary, WEV: Word Embedding Vector.</p>
                </caption>
                <graphic xlink:href="jmir_v20i8e10779_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
              </fig>
              <fig id="figure4" position="float">
                <label>Figure 4</label>
                <caption>
                  <p>Coverage-frequency interaction (left) and coverage-document type interaction (right) in English. CHV: Consumer Health Vocabulary, WEV: Word Embedding Vector.</p>
                </caption>
                <graphic xlink:href="jmir_v20i8e10779_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
              </fig>
              <p>As seen in the left side of <xref ref-type="fig" rid="figure4">Figure 4</xref>, term frequency affects the coverage of each explanation type. Whereas CHV had similar coverage for low- and high-frequency terms, WordNet had much greater coverage for high-frequency terms than for low-frequency terms (0.47 vs 0.77). SubSimplify then increased the coverage of low-frequency terms from 0.47 to 0.78 and of high-frequency terms from 0.77 to 0.79. Last, WEV slightly increased the coverage of low-frequency terms (from 0.77 to 0.78), but increased that of high-frequency terms from 0.79 to 0.94. This indicates that SubSimplify performed quite similar on low- and high-frequency terms, as is the case with CHV, but with an overall much greater coverage.</p>
              <p>Next, we look at the interaction of document source and coverage. Recall that PubMed contains more technical medical terms than Wikipedia sources, and therefore, it constitutes terms that should contain more technical jargon. In the right side of <xref ref-type="fig" rid="figure4">Figure 4</xref>, we see that CHV, SubSimplify, and WEV each had greater coverage in PubMed than in Wikipedia, whereas WordNet had greater coverage in Wikipedia documents. The x-axis depicts the proportional coverage for each explanation source in PubMed and Wikipedia, and the y-axis includes the change in coverage. WordNet, for example, provides fewer explanations for PubMed (0.48) than for Wikipedia (0.52), whereas SubSimplify provides more for PubMed (0.8) than for Wikipedia (0.75). One critical point to note is that SubSimplify has equivalent coverage to WEV. This indicates that SubSimplify performed as well as the fully automated WEV within technical medical text.</p>
            </sec>
            <sec>
              <title>Usefulness</title>
              <p>Next, we performed another 2×2×5 ANOVA with usefulness as the dependent variable. There was only a main effect for explanation type (<italic>F</italic><sub>4,3970</sub>=95.170, <italic>P</italic>&#60;.001; refer to <xref ref-type="table" rid="table2">Table 2</xref>, row 3) and frequency (<italic>F</italic><sub>1,3970</sub>=14.663, <italic>P</italic>&#60;.001). This indicates that the usefulness ratings were significantly different across the different explanations and that usefulness ratings were significantly greater for high-frequency terms on average.</p>
              <p>There was a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>7,3970</sub>=5.390, <italic>P</italic>&#60;.001) and a significant interaction between explanation type and document source (<italic>F</italic><sub>4,3970</sub>=6.387, <italic>P</italic>&#60;.001; <xref ref-type="fig" rid="figure5">Figure 5</xref>).</p>
              <p>As seen in <xref ref-type="fig" rid="figure5">Figure 5</xref> on the left, term frequency affected the usefulness of each explanation type. While CHV had a greater usefulness for low-frequency versus high-frequency terms, WordNet Synonyms had much greater usefulness for high-frequency than low-frequency terms (1.50 vs 1.00). WordNet Summaries had the greatest usefulness score for high-frequency terms (2.40) and smaller, but still quite high, scores for low-frequency terms (1.62). Meanwhile, SubSimplify had greater usefulness for low-frequency terms than for high-frequency terms (1.43 vs 1.35). Last, WEV had a greater score for high-frequency than for low-frequency terms (1.64 vs 1.42). This affirmed the idea that SubSimplify is a resource that performs best for low-frequency terms.</p>
              <p>Next, we look at the interaction of document source and coverage. Recall that PubMed contains more technical medical terms than Wikipedia sources, and, therefore, constitutes terms that should contain more technical jargon. In <xref ref-type="fig" rid="figure5">Figure 5</xref>, we see that WordNet performed better in Wikipedia and that SubSimplify and CHV both performed better in usefulness for PubMed. This affirmed that SubSimplify performs best on more technical documents.</p>
            </sec>
          </sec>
          <sec>
            <title>Summary</title>
            <p>For the English study, we found that SubSimplify performed better than existing medical resources for coverage and had a relatively high quality given its coverage. Furthermore, SubSimplify, much like CHV, performed better for low-frequency and more technical terms, than for high-frequency terms. In addition, this resource had the greatest coverage of all the resources for terms found in the PubMed abstracts. These quality, coverage, and usefulness results suggest that SubSimplify is better equipped to generate explanations for low-frequency and technical terms than the other existing resources.</p>
          </sec>
        </sec>
        <sec>
          <title>Study 2: Spanish Term Explanation Generation</title>
          <p>The second study evaluated our approach in Spanish. This study was identical to the English study, save for two differences. First, there was no Spanish language CHV and WordNet in Spanish only contained summaries (no Spanish synonyms; the version we used contained only Spanish terms mapped to English synonyms). Therefore, we only used WordNet Summaries and compared only three possible explanation resources: our approach (SubSimplify), WordNet, and WEV. Second, since there were no Spanish language PubMed abstracts available, for our second resource, we used Medline Plus [<xref ref-type="bibr" rid="ref36">36</xref>] instead, which is a resource for medical articles geared toward people interested in health information. Last, all instructions, ratings, and explanations were in Spanish.</p>
          <fig id="figure5" position="float">
            <label>Figure 5</label>
            <caption>
              <p>Explanation type-frequency interaction (left) and explanation type-document source interaction (right) in English usefulness measures. CHV: Consumer Health Vocabulary, WEV: Word Embedding Vector.</p>
            </caption>
            <graphic xlink:href="jmir_v20i8e10779_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
          <sec>
            <title>Study Stimuli</title>
            <sec>
              <title>Stimuli</title>
              <p>We tested 400 medical terms balanced for frequency. To get a range of medical terms that occurred in both common texts and more technical texts, we extracted 20 documents from Wikipedia written on the topic of disease and 20 Medline Plus articles. From these documents, we first extracted difficult terms using the same term familiarity threshold. Our cutoff was a frequency less than the 5000th most common term in the Spanish Google Web Corpus [<xref ref-type="bibr" rid="ref10">10</xref>]. Within these terms, we balanced the terms across low and high frequency. In all, we split these 400 terms across high and low frequency and document source.</p>
            </sec>
            <sec>
              <title>Explanation Generation</title>
              <p>We compared our approach to two previous approaches: WordNet and WEV. These resources provided explanations when an exact match could be found for the term in their database.</p>
            </sec>
          </sec>
          <sec>
            <title>Metrics</title>
            <p>For each of the 400 terms, we extracted explanations. For each explanation, we calculated quality, coverage, and usefulness.</p>
          </sec>
          <sec>
            <title>Procedure</title>
            <p>The procedure was identical to English except that all instructions and explanations were written in Spanish. The SEs were both bilingual Spanish-English speakers who had nevertheless received Master of Public Health degrees in English.</p>
          </sec>
          <sec>
            <title>Evaluation Outcomes</title>
            <sec>
              <title>Interoperator Variability</title>
              <p>Again, we calculated Crohnbach alpha in both a liberal and conservative version. For Spanish, the results were 0.64 and 0.90 for the conservative and liberal versions, respectively. We, therefore, again determined that the interoperator reliability was high enough to collapse their quality ratings into one group.</p>
              <p><xref ref-type="table" rid="table5">Table 5</xref> shows the results of the quality, coverage, and usefulness for each explanation source in Spanish. It can be seen that WordNet had the highest average quality rating of the three resources (2.64), but provided the lowest coverage at 20.5%, with a low resulting usefulness (0.543). The coverage of SubSimplify was much greater at 90%, with a lower average quality rating (1.49). Last, WEV provided a higher quality rating (1.84) but with a lower coverage than SubSimplify (89.75%). Regarding usefulness, WEV outperformed SubSimplify (1.77 vs 1.24).</p>
              <p>Given these results, we performed ANOVAs to understand the relationship between frequency and document source for the quality, coverage, and usefulness of each explanation type.</p>
            </sec>
            <sec>
              <title>Quality</title>
              <p>We performed a 2×2×3 ANOVA to evaluate the effect of document source (Wikipedia or PubMed) and frequency (High or Low) on the quality ratings for each of the three explanation resources (WordNet, SubSimplify, and WEV). There were main effects for frequency (<italic>F</italic><sub>1,1590</sub>=13.39, <italic>P</italic>&#60;.001) and explanation type (<italic>F</italic><sub>2,1590</sub>=98.805, <italic>P</italic>&#60;.001; refer to <xref ref-type="table" rid="table3">Table 3</xref> row 1). This indicates that on average, the explanations performed significantly better on high-frequency terms and that there was a significant difference between the average quality of explanations based upon their type.</p>
              <p>There was also a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>2,1590</sub>=12.010, <italic>P</italic>&#60;.001; <xref ref-type="fig" rid="figure6">Figure 6</xref>). <xref ref-type="fig" rid="figure6">Figure 6</xref> shows the interaction of frequency with explanation type. The x-axis depicts the mean quality for each explanation source at low and high frequency, and the y-axis includes the change in mean quality ratings as a line. SubSimplify performed better for low-frequency terms (1.54) than for high-frequency terms (1.48), whereas WEV performed worse for low-frequency terms (1.68) than for high-frequency terms (2.03).</p>
            </sec>
            <sec>
              <title>Coverage</title>
              <p>We performed a 2×2×3 ANOVA to evaluate the effect of document source (Wikipedia or MedlinePlus) and frequency (High or Low) on the coverage ratings for each of the three explanation resources (WordNet, SubSimplify, and WEV). There were also main effects for frequency (<italic>F</italic><sub>1,2382</sub>=7.180, <italic>P</italic>&#60;.001) and explanation type (<italic>F</italic><sub>2,2382</sub>=1142.361, <italic>P</italic>&#60;.001; refer to <xref ref-type="table" rid="table3">Table 3</xref>). This indicates that on average, the explanations had significantly better coverage on high-frequency terms and that there was a significant difference between the average coverage of explanations based upon their type.</p>
              <p>There was a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>2,2382</sub>=4.465, <italic>P</italic>&#60;.015), and a significant interaction between explanation type and document source (<italic>F</italic><sub>4,2382</sub>=6.259, <italic>P</italic>&#60;.001; <xref ref-type="fig" rid="figure7">Figure 7</xref>). For Spanish terms, term frequency affected the coverage of each resource as seen in <xref ref-type="fig" rid="figure7">Figure 7</xref> on the left. For WordNet, there was slightly greater coverage for low-frequency terms (0.22) than for high-frequency terms (0.20), but both were quite low. For SubSimplify, there was greater coverage for high-frequency terms (0.92) than for low-frequency terms (0.87). This was also the case for WEV, with high and low coverages at 0.93 and 0.86, respectively. This indicates that SubSimplify had the greatest coverage for low-frequency terms and WEV had the greatest coverage for high-frequency terms in Spanish.</p>
              <p>For document source, WordNet had a greater coverage for MedlinePlus terms (0.25) than for Wikipedia terms (0.20). Likewise, SubSimplify performed better on the more technical terms of MedlinePlus (0.93) than on the more general terms of Wikipedia (0.87). WEV, however, had the opposite effect, with 0.93 for Wikipedia and 0.86 for MedlinePlus. In short, WEV performed better on less technical texts and higher-frequency terms, whereas SubSimplify performed better on low-frequency terms and more technical texts.</p>
              <table-wrap position="float" id="table5">
                <label>Table 5</label>
                <caption>
                  <p>Spanish study results.</p>
                </caption>
                <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                  <col width="300"/>
                  <col width="250"/>
                  <col width="250"/>
                  <col width="200"/>
                  <thead>
                    <tr valign="bottom">
                      <td>Metrics</td>
                      <td>WordNet Summary</td>
                      <td>SubSimplify</td>
                      <td>WEV<sup>a</sup></td>
                    </tr>
                  </thead>
                  <tbody>
                    <tr valign="top">
                      <td>Quality (1-4 scale)</td>
                      <td>2.64</td>
                      <td>1.49</td>
                      <td>1.84</td>
                    </tr>
                    <tr valign="top">
                      <td>Coverage (N=400), %</td>
                      <td>20.5</td>
                      <td>90.0</td>
                      <td>89.7</td>
                    </tr>
                    <tr valign="top">
                      <td>Usefulness (0-4 scale)</td>
                      <td>0.543</td>
                      <td>1.24</td>
                      <td>1.77</td>
                    </tr>
                  </tbody>
                </table>
                <table-wrap-foot>
                  <fn id="table5fn1">
                    <p><sup>a</sup>WEV: Word Embedding Vector.</p>
                  </fn>
                </table-wrap-foot>
              </table-wrap>
              <fig id="figure6" position="float">
                <label>Figure 6</label>
                <caption>
                  <p>Explanation type-frequency interaction for quality in Spanish. WEV: Word Embedding Vector.</p>
                </caption>
                <graphic xlink:href="jmir_v20i8e10779_fig6.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
              </fig>
              <fig id="figure7" position="float">
                <label>Figure 7</label>
                <caption>
                  <p>Coverage-frequency interaction (left) and coverage-document type interaction (right) in Spanish. WEV: Word Embedding Vector.</p>
                </caption>
                <graphic xlink:href="jmir_v20i8e10779_fig7.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
              </fig>
              <fig id="figure8" position="float">
                <label>Figure 8</label>
                <caption>
                  <p>Explanation type-frequency interaction for frequency. WEV: Word Embedding Vector.</p>
                </caption>
                <graphic xlink:href="jmir_v20i8e10779_fig8.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
              </fig>
            </sec>
            <sec>
              <title>Usefulness</title>
              <p>We performed the same 2×2×3 ANOVA, but this time with usefulness as the dependent measure. The results indicated main effects for frequency (<italic>F</italic><sub>1,2382</sub>=16.197, <italic>P</italic>&#60;.001), explanation type (<italic>F</italic><sub>2,2382</sub> = 230.268, <italic>P</italic>&#60;.001), and document source (<italic>F</italic><sub>1,2382</sub>=6.737, <italic>P</italic>&#60;.001). These results indicate that explanations, on average, performed significantly better on high-frequency terms and on Wikipedia documents. This also indicates that there was a significant difference in the usefulness measures for explanation type (refer to <xref ref-type="table" rid="table3">Table 3</xref>, row 3).</p>
              <p>Furthermore, there was a significant two-way interaction between explanation type and frequency (<italic>F</italic><sub>2,2382</sub>=17.911, <italic>P&#60;</italic>.001). <xref ref-type="fig" rid="figure8">Figure 8</xref> depicts this interaction. Notice that WordNet Summaries performed nearly identically on low- and high-frequency terms (0.6). This pattern was also true for SubSimplify (1.4 for both). Last, WEV performed much better in usefulness on high-frequency terms than on low-frequency terms (1.9 vs 1.4).</p>
            </sec>
            <sec>
              <title>Summary</title>
              <p>For the Spanish study, we found that SubSimplify performed better than WordNet and WEV for coverage. Specifically, it performed best for low-frequency and more technical terms; its average quality was lower than that of the other two resources, but was better at low frequency.</p>
            </sec>
          </sec>
        </sec>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>In both English and Spanish, SubSimplify had its best quality ratings at low frequency, which were similar for both languages. Furthermore, in both languages, SubSimplify had similar results regarding coverage. In both the English and Spanish studies, we saw that SubSimplify greatly outperformed the existing resources with its ability to provide multiword explanations for difficult terms. Namely, SubSimplify outperformed CHV and WordNet Summaries in English in quality, and in Spanish, it outperformed WordNet in this same measure. Furthermore, it provided the most explanations at low frequencies and in more technical texts.</p>
      <p>At the same time, much of the quality and coverage that we have shown covers overlapping data. Here we have evaluated the coverage of these resources if we were to employ all of them into a single system. Doing so will highlight the role that SubSimplify can play in a larger simplification system. Given the 400 terms in each language, the charts in <xref ref-type="fig" rid="figure7">Figure 7</xref> highlight the cumulative coverage of each resource. Given that not all resources cover the same words, these bar graphs show the coverage of a system that includes each nonoverlapping explanation or synonym from the previous resource.</p>
      <p>Left, CHV provides the lowest coverage with 23, WordNet then provides 212, SubSimplify provides 322, and, finally, WEV provides 336 out of 400. On the right, we see the number of explanations that our system can provide as we add each resource in Spanish. For example, if we only used CHV, we would only be able to provide explanations for 23 terms. However, as we add each resource, the number of (nonoverlapping) terms for which we can provide explanations increases. As we add WordNet, we can provide explanations for 222 terms; then by adding SubSimplify, we can provide explanations for 349. Last, by adding WEV, we can provide a total of 385 explanations out of the 400 total terms, or 99% of all terms.</p>
      <fig id="figure9" position="float">
        <label>Figure 9</label>
        <caption>
          <p>Cumulative coverage in English (left) and Spanish (right). CHV: Consumer Health Vocabulary, WEV: Word Embedding Vector.</p>
        </caption>
        <graphic xlink:href="jmir_v20i8e10779_fig9.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <fig id="figure10" position="float">
        <label>Figure 10</label>
        <caption>
          <p>Cumulative usefulness in English (left) and Spanish (right). CHV: Consumer Health Vocabulary, WEV: Word Embedding Vector.</p>
        </caption>
        <graphic xlink:href="jmir_v20i8e10779_fig10.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <p>SubSimplify acts as an intermediate resource between the fully automatic, synonym-providing source of WEV and the annotator-written resources of CHV and WordNet. In its semiautomated approach, it increases the coverage of total terms for which any simplification system can provide multiword explanations. This also is made apparent when we look at the cumulative usefulness of all explanations, which can be found in <xref ref-type="fig" rid="figure9">Figure 9</xref>. Here we have provided the cumulative quality rating for all terms additively. For example, in English (left), the CHV quality is the average rating of explanations for all 400 terms, most of them being 0. Then +WordNet Synonyms gives the average quality of these two resources combined. When two resources each provide an explanation, we take the higher rated explanation of the two. The result is that by employing SubSimplify in conjunction with all resources, this simplification system can provide explanations with a 2.64 usefulness rating in English and 2.09 in Spanish (right).</p>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>The aim of the English and Spanish studies was to evaluate the efficacy of employing SubSimplify to medical texts, and the results revealed what was expected. Compared with WordNet and CHV, the quality of explanations was, on average, lower. This may be an indication of a few different issues, which we will expand upon in the Limitations.</p>
        <p>Surprisingly, the fully automatic system of WEV outperformed our expectations. In creating SubSimplify, we imagined that there would be many spurious matches and synonyms that were unrelated to the difficult terms, but the results showed a better performance than expected. Based upon this, we are motivated to employ WEV as another way to source root synonyms in SubSimplify. That is, in its current form, SubSimplify performs a term lookup in WordNet (and CHV, in English) after parsing each affix. Based upon these results, we are motivated to have the system perform a WEV lookup at this stage as well.</p>
        <p>One challenge with SubSimplify is to maximize the understandability of the explanations themselves. While the existing resources contain a single definition of the term, SubSimplify relies upon combining multiple definitions to sew together a single explanation. Currently, the system employs a color-coding scheme that relates the morphological units on one line to their definitions on another. This may make it difficult to read for some people, effectively lowering the quality when people are not used to seeing these definitions. In order to allay this issue, our team plans on implementing a few different formats and testing them out in the near future in an interactive Web program.</p>
      </sec>
      <sec>
        <title>Limitations</title>
        <p>SubSimplify is naturally limited by two factors. First, not all difficult medical terms contain subword units, and additionally, not all subword units match or are totally accurate. This is because not all terms contain morphological units, and the system has no way of knowing where the characters within a word are actual examples of suffixes. Beyond that, even if they are correct matches of affixes, there is no guarantee that the actual meaning of the terms is directly reflective of the affixes that are found within. For example, it may provide a meaning for <italic>anti-</italic> and -<italic>bodies</italic> in <italic>antibodies</italic>. But the explanation <italic>against bodies</italic> does not reflect the actual meaning of the term.</p>
        <p>Another possibility is that many of the explanations generated by SubSimplify can be incomplete. For example, it may provide a meaning for -<italic>al</italic> in <italic>distal</italic> but no meaning for <italic>dist-</italic>. The result would then be difficult for anyone to understand. Nevertheless, the system does provide a bridge between hand annotated and automatic texts and, therefore, should be subject to these sorts of exceptions and problematic cases.</p>
        <p>Second, SubSimplify, by virtue of using WordNet and CHV, is limited as well to the coverage of those resources. However, we believe that this work presents useful addition to a system aimed at providing explanations for complex terms.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>The niche of SubSimplify is to exploit the regularities of morphological units in medical terminology to provide a window into breaking down the jargon of difficult terms into digestible terms. SubSimplify will improve as the resources used to create it do. Furthermore, we want to look at multiword phrases as oftentimes they reveal the contextual meaning that a single-word context cannot provide alone. This approach is intended as an additional resource that one can add to other methods to automatically provide explanations for difficult texts. The explanations generated by this system greatly increase the number of difficult terms for which an easier alternative can be made available and, thereby, present an advance in the area of text simplification in the medical domain.</p>
      </sec>
    </sec>
  </body>
  <back>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">ANOVA</term>
          <def>
            <p>analysis of variance</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">CHV</term>
          <def>
            <p>Consumer Health Vocabulary</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">NLP</term>
          <def>
            <p>Natural Language Processing</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">SE</term>
          <def>
            <p>subject expert</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">UMLS</term>
          <def>
            <p>Unified Medical Language System</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">WEV</term>
          <def>
            <p>Word Embedding Vector</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>Research reported in this paper was supported by the National Library of Medicine of the National Institutes of Health under Award Number R01LM011975. The content is solely the responsibility of the authors and does not necessarily represent the official views of the National Institutes of Health.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="web">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Fox</surname>
            <given-names>S</given-names>
          </name>
        </person-group>
        <source>Pew Research Center</source>  
        <year>2011</year>  
        <month>02</month>  
        <day>01</day>  
        <access-date>2006-08-10</access-date>
        <comment>Health Topics 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://www.pewinternet.org/pdfs/PIP_Health_Report_July_2003.pdf">http://www.pewinternet.org/pdfs/PIP_Health_Report_July_2003.pdf</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="5I2STSU61"/></comment> </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Davis</surname>
            <given-names>TC</given-names>
          </name>
          <name name-style="western">
            <surname>Mayeaux</surname>
            <given-names>EJ</given-names>
          </name>
          <name name-style="western">
            <surname>Fredrickson</surname>
            <given-names>D</given-names>
          </name>
          <name name-style="western">
            <surname>Bocchini</surname>
            <given-names>JA</given-names>
          </name>
          <name name-style="western">
            <surname>Jackson</surname>
            <given-names>RH</given-names>
          </name>
          <name name-style="western">
            <surname>Murphy</surname>
            <given-names>PW</given-names>
          </name>
        </person-group>
        <article-title>Reading ability of parents compared with reading level of pediatric patient education materials</article-title>
        <source>Pediatrics</source>  
        <year>1994</year>  
        <month>03</month>  
        <volume>93</volume>  
        <issue>3</issue>  
        <fpage>460</fpage>  
        <lpage>468</lpage>  
        <pub-id pub-id-type="medline">8115206</pub-id></nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Kindig</surname>
            <given-names>D</given-names>
          </name>
          <name name-style="western">
            <surname>Panzer</surname>
            <given-names>A</given-names>
          </name>
          <name name-style="western">
            <surname>Nielsen-Bohlman</surname>
            <given-names>L</given-names>
          </name>
        </person-group>
        <source>Health literacy: a prescription to end confusion</source>  
        <year>2004</year>  
        <publisher-loc>Washington, D.C</publisher-loc>
        <publisher-name>National Academies Press</publisher-name>
        <fpage>a</fpage> </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Paasche-Orlow</surname>
            <given-names>MK</given-names>
          </name>
          <name name-style="western">
            <surname>Parker</surname>
            <given-names>RM</given-names>
          </name>
          <name name-style="western">
            <surname>Gazmararian</surname>
            <given-names>JA</given-names>
          </name>
          <name name-style="western">
            <surname>Nielsen-Bohlman</surname>
            <given-names>LT</given-names>
          </name>
          <name name-style="western">
            <surname>Rudd</surname>
            <given-names>RR</given-names>
          </name>
        </person-group>
        <article-title>The prevalence of limited health literacy</article-title>
        <source>J Gen Intern Med</source>  
        <year>2005</year>  
        <month>02</month>  
        <volume>20</volume>  
        <issue>2</issue>  
        <fpage>175</fpage>  
        <lpage>184</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://onlinelibrary.wiley.com/resolve/openurl?genre=article&#38;sid=nlm:pubmed&#38;issn=0884-8734&#38;date=2005&#38;volume=20&#38;issue=2&#38;spage=175"/>
        </comment>  
        <pub-id pub-id-type="doi">10.1111/j.1525-1497.2005.40245.x</pub-id>
        <pub-id pub-id-type="medline">15836552</pub-id>
        <pub-id pub-id-type="pii">JGI40245</pub-id>
        <pub-id pub-id-type="pmcid">PMC1490053</pub-id></nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Shardlow</surname>
            <given-names>M</given-names>
          </name>
        </person-group>
        <article-title>A survey of automated text simplification</article-title>
        <source>International Journal of Advanced Computer Science and Applications</source>  
        <year>2014</year>  
        <volume>4</volume>  
        <issue>1</issue>  
        <fpage>58</fpage>  
        <lpage>70</lpage> </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Blum</surname>
            <given-names>S</given-names>
          </name>
          <name name-style="western">
            <surname>Levenston</surname>
            <given-names>E</given-names>
          </name>
        </person-group>
        <article-title>Universals of lexical simplification</article-title>
        <source>Language learning</source>  
        <year>1978</year>  
        <month>12</month>  
        <day>01</day>  
        <volume>28</volume>  
        <issue>2</issue>  
        <fpage>399</fpage>  
        <lpage>415</lpage> </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
        <name name-style="western">
        <surname>Leroy</surname>  
        <given-names>G</given-names></name>  
        <name name-style="western">
        <surname>Endicott</surname>  
        <given-names>JE</given-names></name>  
        <name name-style="western">
        <surname>Kauchak</surname>  
        <given-names>D</given-names></name>  
        <name name-style="western">
        <surname>Mouradi</surname>  
        <given-names>O</given-names></name>  
        <name name-style="western">
        <surname>Just</surname>  
        <given-names>M</given-names></name> </person-group>
        <article-title>User evaluation of the effects of a text simplification algorithm using term familiarity on perception, understanding, learning, and information retention</article-title>
        <source>J Med Internet Res</source>  
        <year>2013</year>  
        <month>07</month>  
        <day>31</day>  
        <volume>15</volume>  
        <issue>7</issue>  
        <fpage>e144</fpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2013/7/e144/"/>
        </comment>  
        <pub-id pub-id-type="doi">10.2196/jmir.2569</pub-id>
        <pub-id pub-id-type="medline">23903235</pub-id>
        <pub-id pub-id-type="pii">v15i7e144</pub-id>
        <pub-id pub-id-type="pmcid">PMC3742397</pub-id></nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Feblowitz</surname>
            <given-names>D</given-names>
          </name>
          <name name-style="western">
            <surname>Kauchak</surname>
            <given-names>D</given-names>
          </name>
        </person-group>
        <article-title>Sentence simplification as tree transduction</article-title>
        <year>2013</year>  
        <month>08</month>  
        <day>08</day>  
        <conf-name>The Second Workshop on Predicting and Improving Text Readability for Target Reader Populations</conf-name>
        <conf-date>2013</conf-date>
        <conf-loc>Sofia</conf-loc>
        <fpage>1</fpage>  
        <lpage>10</lpage> </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Pitler</surname>
            <given-names>E</given-names>
          </name>
          <name name-style="western">
            <surname>Nenkova</surname>
            <given-names>A</given-names>
          </name>
        </person-group>
        <article-title>Revisiting readability: A unified framework for predicting text quality</article-title>
        <year>2008</year>  
        <month>10</month>  
        <day>25</day>  
        <conf-name>Conference on Empirical Methods in Natural Language Processing</conf-name>
        <conf-date>October</conf-date>
        <conf-loc>University of Edinburgh</conf-loc>
        <fpage>186</fpage>  
        <lpage>195</lpage> </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Brants</surname>
            <given-names>T</given-names>
          </name>
          <name name-style="western">
            <surname>Franz</surname>
            <given-names>A</given-names>
          </name>
        </person-group>
        <article-title>Web 1T 5-gram, 10 European Languages Version 1 LDC2009T25</article-title>
        <source>WebT 5-gram Version</source>  
        <year>2009</year>  
        <publisher-loc>Philadelphia</publisher-loc>
        <publisher-name>Linguistic Data Consortium</publisher-name></nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Leroy</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Kauchak</surname>
            <given-names>D</given-names>
          </name>
        </person-group>
        <article-title>The effect of word familiarity on actual and perceived text difficulty</article-title>
        <source>J Am Med Inform Assoc</source>  
        <year>2014</year>  
        <month>02</month>  
        <volume>21</volume>  
        <issue>e1</issue>  
        <fpage>e169</fpage>  
        <lpage>72</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24100710"/>
        </comment>  
        <pub-id pub-id-type="doi">10.1136/amiajnl-2013-002172</pub-id>
        <pub-id pub-id-type="medline">24100710</pub-id>
        <pub-id pub-id-type="pii">amiajnl-2013-002172</pub-id>
        <pub-id pub-id-type="pmcid">PMC3957403</pub-id></nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Kandula</surname>
            <given-names>S</given-names>
          </name>
          <name name-style="western">
            <surname>Curtis</surname>
            <given-names>D</given-names>
          </name>
          <name name-style="western">
            <surname>Zeng-Treitler</surname>
            <given-names>Q</given-names>
          </name>
        </person-group>
        <article-title>A semantic and syntactic text simplification tool for health content</article-title>
        <source>AMIA Annu Symp Proc</source>  
        <year>2010</year>  
        <month>11</month>  
        <day>13</day>  
        <volume>2010</volume>  
        <fpage>366</fpage>  
        <lpage>70</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/21347002"/>
        </comment>  
        <pub-id pub-id-type="medline">21347002</pub-id>
        <pub-id pub-id-type="pmcid">PMC3041424</pub-id></nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Zeng</surname>
            <given-names>QT</given-names>
          </name>
          <name name-style="western">
            <surname>Tse</surname>
            <given-names>T</given-names>
          </name>
          <name name-style="western">
            <surname>Divita</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Keselman</surname>
            <given-names>A</given-names>
          </name>
          <name name-style="western">
            <surname>Crowell</surname>
            <given-names>J</given-names>
          </name>
          <name name-style="western">
            <surname>Browne</surname>
            <given-names>AC</given-names>
          </name>
        </person-group>
        <article-title>Exploring lexical forms: first-generation consumer health vocabularies</article-title>
        <source>AMIA Annu Symp Proc</source>  
        <year>2006</year>  
        <fpage>1155</fpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/17238774"/>
        </comment>  
        <pub-id pub-id-type="medline">17238774</pub-id>
        <pub-id pub-id-type="pii">86196</pub-id>
        <pub-id pub-id-type="pmcid">PMC1839519</pub-id></nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Bodenreider</surname>
            <given-names>O</given-names>
          </name>
        </person-group>
        <article-title>The Unified Medical Language System (UMLS): integrating biomedical terminology</article-title>
        <source>Nucleic Acids Res</source>  
        <year>2004</year>  
        <month>01</month>  
        <day>01</day>  
        <volume>32</volume>  
        <issue>Database issue</issue>  
        <fpage>D267</fpage>  
        <lpage>70</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/14681409"/>
        </comment>  
        <pub-id pub-id-type="doi">10.1093/nar/gkh061</pub-id>
        <pub-id pub-id-type="medline">14681409</pub-id>
        <pub-id pub-id-type="pii">32/suppl_1/D267</pub-id>
        <pub-id pub-id-type="pmcid">PMC308795</pub-id></nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Zielstorff</surname>
            <given-names>RD</given-names>
          </name>
        </person-group>
        <article-title>Controlled vocabularies for consumer health</article-title>
        <source>J Biomed Inform</source>  
        <year>2003</year>  
        <volume>36</volume>  
        <issue>4-5</issue>  
        <fpage>326</fpage>  
        <lpage>33</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S1532046403000960"/>
        </comment>  
        <pub-id pub-id-type="medline">14643728</pub-id>
        <pub-id pub-id-type="pii">S1532046403000960</pub-id></nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Zeng</surname>
            <given-names>QT</given-names>
          </name>
          <name name-style="western">
            <surname>Tse</surname>
            <given-names>T</given-names>
          </name>
          <name name-style="western">
            <surname>Crowell</surname>
            <given-names>J</given-names>
          </name>
          <name name-style="western">
            <surname>Divita</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Roth</surname>
            <given-names>L</given-names>
          </name>
          <name name-style="western">
            <surname>Browne</surname>
            <given-names>AC</given-names>
          </name>
        </person-group>
        <article-title>Identifying consumer-friendly display (CFD) names for health concepts</article-title>
        <source>AMIA Annu Symp Proc</source>  
        <year>2005</year>  
        <fpage>859</fpage>  
        <lpage>63</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/16779162"/>
        </comment>  
        <pub-id pub-id-type="medline">16779162</pub-id>
        <pub-id pub-id-type="pii">58480</pub-id>
        <pub-id pub-id-type="pmcid">PMC1560732</pub-id></nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Zeng-Treitler</surname>
            <given-names>Q</given-names>
          </name>
          <name name-style="western">
            <surname>Goryachev</surname>
            <given-names>S</given-names>
          </name>
          <name name-style="western">
            <surname>Kim</surname>
            <given-names>H</given-names>
          </name>
          <name name-style="western">
            <surname>Keselman</surname>
            <given-names>A</given-names>
          </name>
          <name name-style="western">
            <surname>Rosendale</surname>
            <given-names>D</given-names>
          </name>
        </person-group>
        <article-title>Making texts in electronic health records comprehensible to consumers: a prototype translator</article-title>
        <source>AMIA Annu Symp Proc</source>  
        <year>2007</year>  
        <month>10</month>  
        <day>11</day>  
        <fpage>846</fpage>  
        <lpage>50</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/18693956"/>
        </comment>  
        <pub-id pub-id-type="medline">18693956</pub-id>
        <pub-id pub-id-type="pmcid">PMC2655860</pub-id></nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Qenam</surname>
            <given-names>B</given-names>
          </name>
          <name name-style="western">
            <surname>Kim</surname>
            <given-names>TY</given-names>
          </name>
          <name name-style="western">
            <surname>Carroll</surname>
            <given-names>MJ</given-names>
          </name>
          <name name-style="western">
            <surname>Hogarth</surname>
            <given-names>M</given-names>
          </name>
        </person-group>
        <article-title>Text Simplification Using Consumer Health Vocabulary to Generate Patient-Centered Radiology Reporting: Translation and Evaluation</article-title>
        <source>J Med Internet Res</source>  
        <year>2017</year>  
        <month>12</month>  
        <day>18</day>  
        <volume>19</volume>  
        <issue>12</issue>  
        <fpage>e417</fpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2017/12/e417/"/>
        </comment>  
        <pub-id pub-id-type="doi">10.2196/jmir.8536</pub-id>
        <pub-id pub-id-type="medline">29254915</pub-id>
        <pub-id pub-id-type="pii">v19i12e417</pub-id>
        <pub-id pub-id-type="pmcid">PMC5748472</pub-id></nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Fellbaum</surname>
            <given-names>C</given-names>
          </name>
          <name name-style="western">
            <surname>Miller</surname>
            <given-names>G</given-names>
          </name>
        </person-group>
        <article-title>A semantic network of English verbs</article-title>
        <source>WordNet: An Electronic Lexical Database</source>  
        <year>1998</year>  
        <publisher-loc>Cambridge</publisher-loc>
        <publisher-name>MIT Press</publisher-name></nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Saggion</surname>
            <given-names>H</given-names>
          </name>
          <name name-style="western">
            <surname>Bott</surname>
            <given-names>S</given-names>
          </name>
          <name name-style="western">
            <surname>Rello</surname>
            <given-names>L</given-names>
          </name>
        </person-group>
        <article-title>Comparing resources for Spanish lexical simplification</article-title>
        <source>Statistical Language and Speech Processing: First International Conference, SLSP 2013, Tarragona, Spain, July 29-31, 2013, Proceedings (Lecture Notes in Computer Science)</source>  
        <year>2013</year>  
        <publisher-loc>Heidelberg Berlin</publisher-loc>
        <publisher-name>Springer</publisher-name>
        <fpage>236</fpage>  
        <lpage>247</lpage> </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Shardlow</surname>
            <given-names>M</given-names>
          </name>
        </person-group>
        <article-title>A Comparison of Techniques to Automatically Identify Complex Words</article-title>
        <year>2013</year>  
        <month>08</month>  
        <day>04</day>  
        <conf-name>In51st Annual Meeting of the Association for Computational Linguistics Proceedings of the Student Research Workshop</conf-name>
        <conf-date>2013</conf-date>
        <conf-loc>Sofia</conf-loc>
        <fpage>103</fpage>  
        <lpage>109</lpage> </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Thomas</surname>
            <given-names>S</given-names>
          </name>
          <name name-style="western">
            <surname>Anderson</surname>
            <given-names>S</given-names>
          </name>
        </person-group>
        <article-title>WordNet-based lexical simplification of a document</article-title>
        <year>2012</year>  
        <month>09</month>  
        <day>19</day>  
        <conf-name>11th Conference on Natural Language Processing (KOVENS)</conf-name>
        <conf-date>2012</conf-date>
        <conf-loc>Vienna</conf-loc>
        <fpage>80</fpage>  
        <lpage>88</lpage> </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Shehu</surname>
            <given-names>A</given-names>
          </name>
          <name name-style="western">
            <surname>Plaku</surname>
            <given-names>E</given-names>
          </name>
        </person-group>
        <article-title>A Survey of Computational Treatments of Biomolecules by Robotics-Inspired Methods Modeling Equilibrium Structure and Dynamic</article-title>
        <source>jair</source>  
        <year>2016</year>  
        <month>11</month>  
        <day>29</day>  
        <volume>57</volume>  
        <fpage>509</fpage>  
        <lpage>572</lpage>  
        <pub-id pub-id-type="doi">10.1613/jair.5040</pub-id></nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="web">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Paetzold</surname>
            <given-names>G</given-names>
          </name>
        </person-group>
        <source>White Rose eTheses Online</source>  
        <access-date>2018-04-12</access-date>
        <publisher-name>University of Sheffield</publisher-name>
        <comment>Lexical Simplification for Non-Native English Speakers. [Doctoral dissertation] 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://etheses.whiterose.ac.uk/15332/1/Final_Version_Thesis.pdf">http://etheses.whiterose.ac.uk/15332/1/Final_Version_Thesis.pdf</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="6ydMFhua8"/></comment> </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Mikolov</surname>
            <given-names>T</given-names>
          </name>
          <name name-style="western">
            <surname>Sutskever</surname>
            <given-names>I</given-names>
          </name>
          <name name-style="western">
            <surname>Chen</surname>
            <given-names>K</given-names>
          </name>
          <name name-style="western">
            <surname>Corrado</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Dean</surname>
            <given-names>J</given-names>
          </name>
        </person-group>
        <article-title>Distributed representations of words and phrases and their compositionality</article-title>
        <year>2013</year>  
        <conf-name>Advances in neural information processing systems</conf-name>
        <conf-date>2013</conf-date>
        <conf-loc>Lake Tahoe</conf-loc>
        <fpage>3111</fpage>  
        <lpage>3119</lpage> </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="confproc">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Glavaš</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Štajner</surname>
            <given-names>S</given-names>
          </name>
        </person-group>
        <article-title>Simplifying Lexical Simplification: Do We Need Simplified Corpora?</article-title>
        <year>2015</year>  
        <conf-name>The Association for Computational Linguistics and the 7th International Joint Conference on Natural Language Processing Short Papers Vol. 2</conf-name>
        <conf-date>2015</conf-date>
        <conf-loc>Beijing</conf-loc>
        <fpage>63</fpage>  
        <lpage>68</lpage> </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="web">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Cardellino</surname>
            <given-names>C</given-names>
          </name>
        </person-group>
        <source>Spanish Billion Words Corpus and Embeddings</source>  
        <year>2016</year>  
        <access-date>2018-04-12</access-date>
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://crscardellino.me/SBWCE/">http://crscardellino.me/SBWCE/</ext-link>
          <ext-link ext-link-type="webcite" xlink:href="6ydMMtUOT"/>
        </comment> </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Tindall</surname>
            <given-names>A</given-names>
          </name>
        </person-group>
        <article-title>Medical Terms</article-title>
        <source>Medical Terms: Their Roots and Origins</source>  
        <year>1997</year>  
        <publisher-loc>Boca Raton</publisher-loc>
        <publisher-name>CRC Press</publisher-name></nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Green</surname>
            <given-names>T</given-names>
          </name>
        </person-group>
        <source>The Greek &#38; Latin Roots of English</source>  
        <year>2014</year>  
        <month>07</month>  
        <day>17</day>  
        <publisher-loc>Lanham</publisher-loc>
        <publisher-name>Rowman &#38; Littlefield</publisher-name></nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="web">
        <source>Wikipedia, The Free Encyclopedia</source>  
        <access-date>2018-04-12</access-date>
        <comment>List of medical roots, suffixes and prefixes 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="https://en.wikipedia.org/wiki/List_of_medical_roots,_suffixes_and_prefixes">https://en.wikipedia.org/wiki/List_of_medical_roots,_suffixes_and_prefixes</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="6ydMu8G7g"/></comment> </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="web">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>McAuley</surname>
            <given-names>D</given-names>
          </name>
        </person-group>
        <source>Global RXPh: The Clinicians Ultimate Reference</source>  
        <year>2018</year>  
        <month>04</month>  
        <day>26</day>  
        <access-date>2018-04-12</access-date>
        <comment>Medical TERMINOLOGY-A thru Z 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://www.globalrph.com/medterm.htm">http://www.globalrph.com/medterm.htm</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="6ydMsGnXB"/></comment> </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="web">
        <source>Gramáticas</source>  
        <access-date>2018-04-12</access-date>
        <comment>Prefijos y Sufijos Médicos 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://www.gramaticas.net/2013/10/prefijos-y-sufijos-medicos.html">http://www.gramaticas.net/2013/10/prefijos-y-sufijos-medicos.html</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="6ydMnxHAa"/></comment> </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="web">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Salazar</surname>
            <given-names>M</given-names>
          </name>
        </person-group>
        <source>Enfermería: Blog dedicado a difundir el conocimiento de la Ciencia y Arte de la Enfermería y la Educación</source>  
        <year>2013</year>  
        <month>02</month>  
        <day>26</day>  
        <access-date>2018-04-12</access-date>
        <comment>TERMINOLOGÍA MÉDICA, SUFIJOS, PREFIJOS Y ABREVIATURAS 
        <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://msalazar081858.blogspot.com/2013/02/terminologia-medica-sufijos-prefijos-y.html">http://msalazar081858.blogspot.com/2013/02/terminologia-medica-sufijos-prefijos-y.html</ext-link>
        <ext-link ext-link-type="webcite" xlink:href="6ydMxm4bu"/></comment> </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Willett</surname>
            <given-names>P</given-names>
          </name>
        </person-group>
        <article-title>The Porter stemming algorithm: then and now</article-title>
        <source>Program</source>  
        <year>2006</year>  
        <month>07</month>  
        <day>1</day>  
        <volume>40</volume>  
        <issue>3</issue>  
        <fpage>219</fpage>  
        <lpage>223</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1108/00330330610681295"/>
        </comment>  
        <pub-id pub-id-type="doi">10.1108/00330330610681295</pub-id></nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Miller</surname>
            <given-names>N</given-names>
          </name>
          <name name-style="western">
            <surname>Lacroix</surname>
            <given-names>EM</given-names>
          </name>
          <name name-style="western">
            <surname>Backus</surname>
            <given-names>JE</given-names>
          </name>
        </person-group>
        <article-title>MEDLINEplus: building and maintaining the National Library of Medicine's consumer health Web service</article-title>
        <source>Bull Med Libr Assoc</source>  
        <year>2000</year>  
        <month>01</month>  
        <volume>88</volume>  
        <issue>1</issue>  
        <fpage>11</fpage>  
        <lpage>7</lpage>  
        <comment>
          <ext-link xmlns:xlink="http://www.w3.org/1999/xlink" ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/10658959"/>
        </comment>  
        <pub-id pub-id-type="medline">10658959</pub-id>
        <pub-id pub-id-type="pmcid">PMC35193</pub-id></nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="book">
        <person-group person-group-type="author">
          <name name-style="western">
            <surname>Leroy</surname>
            <given-names>G</given-names>
          </name>
          <name name-style="western">
            <surname>Endicott</surname>
            <given-names>JE</given-names>
          </name>
        </person-group>
        <person-group person-group-type="editor">
          <name name-style="western">
            <surname>Xing</surname>
            <given-names>C</given-names>
          </name>
          <name name-style="western">
            <surname>Crestani</surname>
            <given-names>F</given-names>
          </name>
          <name name-style="western">
            <surname>Rauber</surname>
            <given-names>A</given-names>
          </name>
        </person-group>
        <article-title>Term Familiarity to Indicate Perceived and Actual Difficulty of Text in Medical Digital Libraries</article-title>
        <source>Digital Libraries: For Cultural Heritage, Knowledge Dissemination, and Future Creation</source>  
        <year>2011</year>  
        <month>10</month>  
        <publisher-loc>Berlin, Heidelberg</publisher-loc>
        <publisher-name>Springer</publisher-name>
        <fpage>307</fpage>  
        <lpage>310</lpage> </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
