<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v22i12e18725</article-id>
      <article-id pub-id-type="pmid">33284117</article-id>
      <article-id pub-id-type="doi">10.2196/18725</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Patient Challenges and Needs in Comprehending Laboratory Test Results: Mixed Methods Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>He</surname>
            <given-names>Daqing</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Zekri</surname>
            <given-names>Jamal</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Zhang</surname>
            <given-names>Zhan</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>School of Computer Science and Information Systems</institution>
            <institution>Pace University</institution>
            <addr-line>1 Pace Plaza</addr-line>
            <addr-line>New York, NY, 10078</addr-line>
            <country>United States</country>
            <phone>1 9147733254</phone>
            <email>zzhang@pace.edu</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-6973-6903</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Citardi</surname>
            <given-names>Daniel</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1337-2685</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Xing</surname>
            <given-names>Aiwen</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-4901-7025</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Luo</surname>
            <given-names>Xiao</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3649-9785</ext-link>
        </contrib>
        <contrib id="contrib5" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Lu</surname>
            <given-names>Yu</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9704-7950</ext-link>
        </contrib>
        <contrib id="contrib6" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>He</surname>
            <given-names>Zhe</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-3608-0244</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>School of Computer Science and Information Systems</institution>
        <institution>Pace University</institution>
        <addr-line>New York, NY</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Statistics</institution>
        <institution>Florida State University</institution>
        <addr-line>Tallahassee, FL</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>School of Engineering and Technology</institution>
        <institution>Indiana University-Purdue University Indianapolis</institution>
        <addr-line>Indianapolis, IN</addr-line>
        <country>United States</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>School of Information</institution>
        <institution>Florida State University</institution>
        <addr-line>Tallahassee, FL</addr-line>
        <country>United States</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Zhan Zhang <email>zzhang@pace.edu</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>12</month>
        <year>2020</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>7</day>
        <month>12</month>
        <year>2020</year>
      </pub-date>
      <volume>22</volume>
      <issue>12</issue>
      <elocation-id>e18725</elocation-id>
      <history>
        <date date-type="received">
          <day>14</day>
          <month>3</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>12</day>
          <month>6</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>11</day>
          <month>8</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>11</day>
          <month>11</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©Zhan Zhang, Daniel Citardi, Aiwen Xing, Xiao Luo, Yu Lu, Zhe He. Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 07.12.2020.</copyright-statement>
      <copyright-year>2020</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on http://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://www.jmir.org/2020/12/e18725/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Patients are increasingly able to access their laboratory test results via patient portals. However, merely providing access does not guarantee comprehension. Patients could experience confusion when reviewing their test results.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>The aim of this study is to examine the challenges and needs of patients when comprehending laboratory test results.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>We conducted a web-based survey with 203 participants and a set of semistructured interviews with 13 participants. We assessed patients’ perceived challenges and needs (both informational and technological needs) when they attempted to comprehend test results, factors associated with patients’ perceptions, and strategies for improving the design of patient portals to communicate laboratory test results more effectively. Descriptive and correlation analysis and thematic analysis were used to analyze the survey and interview data, respectively.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>Patients face a variety of challenges and confusion when reviewing laboratory test results. To better comprehend laboratory results, patients need different types of information, which are grouped into 2 categories—generic information (eg, reference range) and personalized or contextual information (eg, treatment options, prognosis, what to do or ask next). We also found that several intrinsic factors (eg, laboratory result normality, health literacy, and technology proficiency) significantly impact people’s perceptions of using portals to view and interpret laboratory results. The desired enhancements of patient portals include providing timely explanations and educational resources (eg, a health encyclopedia), increasing usability and accessibility, and incorporating artificial intelligence–based technology to provide personalized recommendations.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>Patients face significant challenges in interpreting the meaning of laboratory test results. Designers and developers of patient portals should employ user-centered approaches to improve the design of patient portals to present information in a more meaningful way.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>consumer health information</kwd>
        <kwd>health information technology</kwd>
        <kwd>patient portal</kwd>
        <kwd>clinical laboratory test</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Motivation</title>
        <p>Rapid and accurate communication of laboratory test results to patients is critical [<xref ref-type="bibr" rid="ref1">1</xref>]. Growing evidence suggests that patients are increasingly interested in direct access to their test results, regardless of the indication of the results [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref3">3</xref>]. Policies, guidelines, and initiatives have been implemented to encourage health care organizations to provide patients with easy access to their test results through patient-facing technologies [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>], such as patient portals, which are often linked to electronic health record systems [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>]. It is well recognized by the health informatics community that increasing patients’ access to their data can lead to better patient-centered medical care [<xref ref-type="bibr" rid="ref8">8</xref>], greater patient engagement in medical decision making [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref9">9</xref>], and enhanced patient-clinician relationship [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Despite the potential benefits of such direct access, there are long-standing yet unresolved concerns that many patients may not be able to fully understand their laboratory test results and evaluate the indications for their health [<xref ref-type="bibr" rid="ref12">12</xref>,<xref ref-type="bibr" rid="ref13">13</xref>].</p>
        <p>Why does merely providing access to laboratory test results not guarantee comprehension? First, laboratory tests are complex. Even medical professionals cannot efficiently and accurately interpret them [<xref ref-type="bibr" rid="ref14">14</xref>], let alone patients. Second, many patients have limited health literacy and numeracy, thereby making it difficult to interpret the test results and decide corresponding actions (eg, call their doctor for an urgent appointment vs regular follow-up visits) [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Finally, the current interface of patient portals also poses significant challenges in viewing and comprehending test results [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>]. Many portals only provide test results to patients in a tabular format with standard reference ranges, similar to the format received by clinicians. As such, these portals could be of little use to those who are supposed to benefit from them most [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref19">19</xref>]. Although some visual aids, such as color, have been used in patient portals to help patients identify out-of-range values [<xref ref-type="bibr" rid="ref20">20</xref>], test results are often presented with unfamiliar abbreviations and units, lacking guidance on whether the results are concerning [<xref ref-type="bibr" rid="ref21">21</xref>]. Therefore, lay individuals have difficulty identifying meaningful information out of their test results and making informed decisions [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
        <p>Despite the urgent need to address these issues, only a few studies have looked into how to better deliver test results along with the support of people with specific needs [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. There is a lack of in-depth, empirical understanding of patients’ experiences, challenges, and needs related to comprehending laboratory test results via patient portals. To that end, we conducted a mixed methods study to address this research gap by examining the 3 following research questions (RQs):</p>
        <list list-type="bullet">
          <list-item>
            <p>RQ1: What are patients’ perceived challenges and needs when they attempt to comprehend test results?</p>
          </list-item>
          <list-item>
            <p>RQ2: What factors are associated with patients’ perceptions?</p>
          </list-item>
          <list-item>
            <p>RQ3: How can we improve the design of patient portals to communicate laboratory test results to patients in a more meaningful and understandable manner?</p>
          </list-item>
        </list>
      </sec>
      <sec>
        <title>Related Work</title>
        <p>The literature on supporting patients’ comprehension of test results is sparse but growing. A few studies attempted to understand patients’ confusion about their laboratory test results by examining online posts. For example, Reynolds et al [<xref ref-type="bibr" rid="ref25">25</xref>] examined patient question posts in an online health forum and found that patients asked questions about diagnosis, treatment, laboratory report, risk, and prognosis. In a similar vein, Zhang et al [<xref ref-type="bibr" rid="ref26">26</xref>] examined question posts in a social question and answer site and found that patients’ confusions primarily centered on understanding the meaning of laboratory values, specific terminology, and the causes of abnormal or inconsistent results. However, one major limitation of these studies is the lack of direct patient input; as indicated by previous work, failing to involve patients in the process of developing patient-facing applications could lead to limited uptake of these systems [<xref ref-type="bibr" rid="ref27">27</xref>]. Therefore, a comprehensive understanding of patients’ needs is of utmost importance for designing supportive technologies.</p>
        <p>A few studies addressed this limitation by eliciting patient opinions and insights through user studies, such as surveys and interviews. For example, Giardina et al [<xref ref-type="bibr" rid="ref23">23</xref>] examined patient perceptions of receiving test results via online portals and found that most patients did not receive any explanatory information at the time they received the result. They also stated that current online portals are not designed to present test results to patients in a meaningful way. In another study, Reiff et al [<xref ref-type="bibr" rid="ref28">28</xref>] conducted semistructured interviews with parents of pediatric outpatients to identify the needs of families undergoing chromosomal microarray (CMA) testing. They found that incomplete comprehension of test results and scientific uncertainty were prominent challenges faced by families. Receiving results from nongeneticists and misleading internet searches, among others, was found to contribute to the misunderstanding of CMA testing results. Building upon their work, we conducted a mixed methods study to gain further insight into patients’ challenges and needs when comprehending laboratory test results.</p>
        <p>Gaining an in-depth and thorough understanding of patients’ information needs, faced challenges, and preferences when comprehending laboratory test results is vital to inform the design of patient portals and other patient-facing informatics tools [<xref ref-type="bibr" rid="ref21">21</xref>]. However, only a few studies explored patients’ information needs and challenges when interpreting their test results and how to improve the design of patient portals for better support. For example, Nystrom et al [<xref ref-type="bibr" rid="ref29">29</xref>] designed and evaluated a patient-centered test result interface that consists of visual ranges of laboratory values, nontechnical descriptions of the test and the result, and links to reputable internet resources where patients could learn more about their result. User testing indicates that these features were perceived as usable because they account for patients’ needs and preferences for understanding laboratory test results. Zikmund-Fisher et al [<xref ref-type="bibr" rid="ref21">21</xref>] tested the effect of presenting patients with an additional reference point in visual displays of laboratory test results regarding when test results become clinically concerning. They found that providing patients with such evaluative cues can substantially reduce the perceived urgency of out-of-range results that normally do not require immediate action. These studies, however, did not have a user study component; therefore, patients were left out of the system design process. This study contributes to this line of research by examining patients’ needs through holistic user studies and leveraging the findings to suggest design implications for patient portals to promote patients’ comprehension of test results.</p>
      </sec>
      <sec>
        <title>Aim of This Study</title>
        <p>The ultimate goal of this study is to understand the design requirements and specific strategies for improving patient portals to communicate laboratory test results to patients in a way that is more meaningful and understandable. The development of these requirements and strategies requires an empirical understanding of patients’ perceived challenges and needs when attempting to comprehend test results.</p>
      </sec>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Design</title>
        <p>We used a mixed methods approach [<xref ref-type="bibr" rid="ref30">30</xref>], combining a web-based survey, and semistructured interviews, to gain an empirical and thorough understanding of patients’ experiences, challenges, and needs in comprehending laboratory test results. The reason for using both survey and interview methodologies is that they are suitable for creative knowledge generation of a multilayered issue [<xref ref-type="bibr" rid="ref31">31</xref>], such as our research context, which encompasses many interdependent factors (eg, people, technology, data, and knowledge). In addition, these 2 methods are complementary to each other, allowing us to compare and triangulate the findings from each study [<xref ref-type="bibr" rid="ref32">32</xref>]. For example, the survey study allowed us to broadly understand our research questions, whereas the interview study helped us obtain more in-depth insights. Both methods were used jointly to ensure a mix of both broad and deep understanding of the 3 research questions. The survey (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>) was developed in an iterative manner by the researchers and pilot-tested with a small group of people (n=10) to ensure the clarity, appropriateness, and relevance of the questions. The interviews (<xref ref-type="supplementary-material" rid="app2">Multimedia Appendix 2</xref>) were informed by the exploratory survey. This study was approved by the Pace University Institutional Review Board (IRB# 19-08).</p>
      </sec>
      <sec>
        <title>Data Collection</title>
        <p>Data collection occurred between July and September 2019. We recruited 203 participants from Amazon Mechanical Turk (MTurk) to participate in our survey. Using MTurk to study patients’ perceptions has been proven reliable and effective [<xref ref-type="bibr" rid="ref33">33</xref>]. To determine their eligibility, we first asked potential respondents whether they had recently taken any laboratory test and whether they had used patient portals to view the results. If they responded with <italic>yes</italic> to both questions, we asked them to read and electronically sign the informed consent form and complete the survey. Toward the end of the survey, we invited them to participate in a follow-up interview study. Potential participants were instructed to email researchers to arrange a suitable time for the interview.</p>
        <sec>
          <title>Online Survey</title>
          <p><xref ref-type="table" rid="table1">Table 1</xref> shows survey participant characteristics, including an almost equal number of male and female participants (104/203, 51.2% and 98/203, 48.3%, respectively). Most of the participants were White (141/203, 69.5%), aged between 26 and 49 years (129/203, 63.5%), had a bachelor’s or higher degree (134/203, 66.1%), had a full-time job (157/203, 77.3%), and self-reported having above medium health literacy (111/203, 54.7%) and technology proficiency (148/203, 72.9%).</p>
          <table-wrap position="float" id="table1">
            <label>Table 1</label>
            <caption>
              <p>Characteristics of survey participants (N=203).</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="30"/>
              <col width="570"/>
              <col width="400"/>
              <thead>
                <tr valign="top">
                  <td colspan="2">Participant characteristics</td>
                  <td>Participants, n (%)</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Age (years)</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>18-25</td>
                  <td>38 (18.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>26-49</td>
                  <td>129 (63.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>50-64</td>
                  <td>32 (15.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>65 and older</td>
                  <td>4 (2.0)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Gender</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Male</td>
                  <td>104 (51.2)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Female</td>
                  <td>98 (48.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td>1 (0.5)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Race or ethnicity</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Asian or Pacific Islander</td>
                  <td>9 (4.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>African American</td>
                  <td>34 (16.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Hispanic/Latino</td>
                  <td>12 (5.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>American Indian</td>
                  <td>4 (2.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>White</td>
                  <td>141 (69.5)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td>3 (1.5)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Education</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Doctorate degree</td>
                  <td>6 (3.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Master’s degree</td>
                  <td>32 (15.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Bachelor’s degree</td>
                  <td>96 (47.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Associate degree</td>
                  <td>28 (13.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>High school degree</td>
                  <td>40 (19.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td>1 (0.4)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Employment status</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Unemployed</td>
                  <td>17 (8.4)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Part time</td>
                  <td>23 (11.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Full time</td>
                  <td>157 (77.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td>6 (3.0)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Employment industry</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Government</td>
                  <td>16 (7.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Health care</td>
                  <td>23 (11.3)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Education</td>
                  <td>32 (15.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Finance</td>
                  <td>22 (10.8)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Information technology</td>
                  <td>44 (21.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Other</td>
                  <td>66 (32.5)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Health literacy</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>High</td>
                  <td>44 (21.7)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Medium-high</td>
                  <td>67 (33.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Medium</td>
                  <td>81 (39.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Low-medium</td>
                  <td>10 (5.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Low</td>
                  <td>1 (0.4)</td>
                </tr>
                <tr valign="top">
                  <td colspan="3">
                    <bold>Technology proficiency</bold>
                  </td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>High</td>
                  <td>67 (33.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Medium-high</td>
                  <td>81 (39.9)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Medium</td>
                  <td>52 (25.6)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Low-medium</td>
                  <td>2 (1.0)</td>
                </tr>
                <tr valign="top">
                  <td>
                    <break/>
                  </td>
                  <td>Low</td>
                  <td>1 (0.5)</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
          <p>The survey assessed 3 domain areas: (1) participants’ sociodemographic characteristics, (2) participants’ experiences and information needs about understanding laboratory results, and (3) participants’ perceptions of reviewing results via patient portals. Participants’ characteristics collected included age, gender, race/ethnicity, education, and occupation. As health literacy and technology proficiency may affect patient use of the patient portal to view test results [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref19">19</xref>,<xref ref-type="bibr" rid="ref34">34</xref>], we also asked them to rate their technology proficiency and health literacy on a scale of 1 to 5 (<italic>1</italic> denotes low literacy, whereas <italic>5</italic> denotes high literacy). Patients’ experiences include whether they understand the results, what actions have been taken to make sense of the test results, if they have confusion regarding the result, and what types of information or advice are needed. Patients’ perceptions of the patient portal include the usability of patient portals, challenges or concerns (if any) with using patient portals, and tailoring the patient portal to their needs.</p>
          <p>We directed participants to answer survey questions based on their most recent experience of reviewing laboratory test results in a patient portal. However, respondents may have difficulty recalling such an experience, and thus, we provided a commonly seen laboratory test—the lipid profile (eg, total cholesterol, low-density cholesterol, and high-density cholesterol)—to help participants situate themselves in the context (<xref ref-type="table" rid="table2">Table 2</xref>). The test results were presented following the format currently implemented in many patient portals, that is, using tables to show test values and units. The reason for choosing this laboratory test is that it is routinely performed among adults for screening and diagnostic purposes; therefore, many adults are likely familiar with the test [<xref ref-type="bibr" rid="ref35">35</xref>]. The survey took about 10-15 min to complete, and the respondents received US $2 after they completed the survey.</p>
          <table-wrap position="float" id="table2">
            <label>Table 2</label>
            <caption>
              <p>Lipid profile.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="310"/>
              <col width="330"/>
              <col width="360"/>
              <thead>
                <tr valign="top">
                  <td>Type</td>
                  <td>Reference range</td>
                  <td>Results</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>HDL<sup>a</sup> cholesterol</td>
                  <td>&#62;39 mg/Dl</td>
                  <td>52 mg/dL</td>
                </tr>
                <tr valign="top">
                  <td>LDL<sup>b</sup> cholesterol</td>
                  <td>0-99 mg/dL</td>
                  <td>115 mg/dL</td>
                </tr>
                <tr valign="top">
                  <td>Total cholesterol</td>
                  <td>100-199 mg/dL</td>
                  <td>185 mg/dL</td>
                </tr>
                <tr valign="top">
                  <td>Triglycerides</td>
                  <td>0-149 mg/dL</td>
                  <td>164 mg/dL</td>
                </tr>
              </tbody>
            </table>
            <table-wrap-foot>
              <fn id="table2fn1">
                <p><sup>a</sup>HDL: high-density lipoprotein.</p>
              </fn>
              <fn id="table2fn2">
                <p><sup>b</sup>LDL: low-density lipoprotein.</p>
              </fn>
            </table-wrap-foot>
          </table-wrap>
        </sec>
        <sec>
          <title>Semistructured Interview</title>
          <p>To further understand patients’ information needs and challenges related to understanding laboratory test results, we conducted semistructured interviews with 13 people who had participated in the previous survey (<xref ref-type="table" rid="table3">Table 3</xref>). The interviews were conducted via Skype by 2 trained researchers and lasted from 30 min to 1 hour. During the interview, we asked participants to walk through the process of receiving a test result and probed them with questions about their understanding of the results, information needs, challenges faced, perceptions of patient-provider communication, and facilitators and barriers to using the patient portal. All the interviews were audio recorded with the participants’ permission and transcribed verbatim. All the participants received a US $20 Amazon gift card as compensation for their time.</p>
          <table-wrap position="float" id="table3">
            <label>Table 3</label>
            <caption>
              <p>Characteristics of interview participants.</p>
            </caption>
            <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
              <col width="90"/>
              <col width="110"/>
              <col width="110"/>
              <col width="140"/>
              <col width="160"/>
              <col width="230"/>
              <col width="160"/>
              <thead>
                <tr valign="top">
                  <td>ID</td>
                  <td>Gender</td>
                  <td>Age (years)</td>
                  <td>Ethnicity</td>
                  <td>Occupation</td>
                  <td>Technology proficiency level</td>
                  <td>Health literacy level</td>
                </tr>
              </thead>
              <tbody>
                <tr valign="top">
                  <td>Patient 1</td>
                  <td>Female</td>
                  <td>50-64</td>
                  <td>White</td>
                  <td>Unemployed</td>
                  <td>4</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 2</td>
                  <td>Male</td>
                  <td>50-64</td>
                  <td>African American</td>
                  <td>Retired</td>
                  <td>5</td>
                  <td>4</td>
                </tr>
                <tr valign="top">
                  <td>Patient 3</td>
                  <td>Female</td>
                  <td>50-64</td>
                  <td>White</td>
                  <td>Retired</td>
                  <td>5</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 4</td>
                  <td>Male</td>
                  <td>50-64</td>
                  <td>White</td>
                  <td>Technical analyst</td>
                  <td>5</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 5</td>
                  <td>Male</td>
                  <td>26-49</td>
                  <td>White</td>
                  <td>IT clerk</td>
                  <td>5</td>
                  <td>3</td>
                </tr>
                <tr valign="top">
                  <td>Patient 6</td>
                  <td>Male</td>
                  <td>26-49</td>
                  <td>White</td>
                  <td>Dog trainer</td>
                  <td>4</td>
                  <td>3</td>
                </tr>
                <tr valign="top">
                  <td>Patient 7</td>
                  <td>Female</td>
                  <td>50-64</td>
                  <td>White</td>
                  <td>Landscaper</td>
                  <td>3</td>
                  <td>4</td>
                </tr>
                <tr valign="top">
                  <td>Patient 8</td>
                  <td>Female</td>
                  <td>50-64</td>
                  <td>White</td>
                  <td>Executive assistant</td>
                  <td>5</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 9</td>
                  <td>Male</td>
                  <td>18-25</td>
                  <td>Asian</td>
                  <td>Medical assistant</td>
                  <td>4</td>
                  <td>4</td>
                </tr>
                <tr valign="top">
                  <td>Patient 10</td>
                  <td>Male</td>
                  <td>26-49</td>
                  <td>Asian</td>
                  <td>Visual artist</td>
                  <td>5</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 11</td>
                  <td>Female</td>
                  <td>26-49</td>
                  <td>White</td>
                  <td>Office assistant</td>
                  <td>2</td>
                  <td>4</td>
                </tr>
                <tr valign="top">
                  <td>Patient 12</td>
                  <td>Female</td>
                  <td>26-49</td>
                  <td>White</td>
                  <td>Sales</td>
                  <td>5</td>
                  <td>5</td>
                </tr>
                <tr valign="top">
                  <td>Patient 13</td>
                  <td>Female</td>
                  <td>26-49</td>
                  <td>White</td>
                  <td>Unemployed</td>
                  <td>4</td>
                  <td>4</td>
                </tr>
              </tbody>
            </table>
          </table-wrap>
        </sec>
      </sec>
      <sec>
        <title>Data Analysis</title>
        <p>We used descriptive statistics to analyze the survey data. Correlations between the variables were evaluated using the chi-square test or Fisher exact test where appropriate. In particular, we tested the association between several intrinsic factors (eg, age, gender, educational background, technology proficiency, health literacy, and normality of results) and participants’ perceptions of using patient portals to review laboratory results. All the tests were 2-sided, and the level of significance was set at 0.05. Statistical analysis was conducted using IBM SPSS 25.0. Detailed quantitative results are provided in <xref ref-type="table" rid="table4">Tables 4</xref> and <xref ref-type="table" rid="table5">5</xref> and in the tables in <xref ref-type="supplementary-material" rid="app3">Multimedia Appendix 3</xref>.</p>
        <table-wrap position="float" id="table4">
          <label>Table 4</label>
          <caption>
            <p>Patient perceptions of interpreting test results and the association to normality of laboratory test results and health literacy (n=199).</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="400"/>
            <col width="180"/>
            <col width="210"/>
            <col width="180"/>
            <thead>
              <tr valign="bottom">
                <td colspan="2">Patient perception</td>
                <td>Total<sup>a</sup>, n (%)</td>
                <td>Normality of results (<italic>P</italic> value)</td>
                <td>Health literacy (<italic>P</italic> value)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="5">
                  <bold>Did your physician communicate with you about the test results before you viewed the results?</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Yes</td>
                <td>112 (56.3)</td>
                <td>.001<sup>b</sup></td>
                <td>.18</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>No</td>
                <td>87 (43.7)</td>
                <td>.001<sup>b</sup></td>
                <td>.18</td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>Did you understand the result?</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Yes</td>
                <td>120 (60.3)</td>
                <td>.12</td>
                <td>.03<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>No</td>
                <td>9 (4.52)</td>
                <td>.12</td>
                <td>.03<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Not sure</td>
                <td>70 (35.2)</td>
                <td>.12</td>
                <td>.03<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>How did you know the test result was abnormal or normal?<sup>d</sup></bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Visual cue on patient portals</td>
                <td>86 (43.2)</td>
                <td>.98</td>
                <td>.02<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Clinician’s explanation</td>
                <td>104 (52.3)</td>
                <td>.60</td>
                <td>.21</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Personal knowledge</td>
                <td>70 (35.2)</td>
                <td>.90</td>
                <td>.50</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>11 (5.5)</td>
                <td>.59</td>
                <td>.18</td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>What kinds of confusion did you have?<sup>d</sup> (n=136)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Normal versus abnormal laboratory result</td>
                <td>43 (31.6)</td>
                <td>.84</td>
                <td>.64</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Medical terminology</td>
                <td>83 (61.0)</td>
                <td>.37</td>
                <td>.01<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Meaning of the results</td>
                <td>54 (39.7)</td>
                <td>.66</td>
                <td>.10</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Effects on my future health</td>
                <td>47 (34.6)</td>
                <td>.80</td>
                <td>.61</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Treatment option</td>
                <td>27 (19.9)</td>
                <td>.69</td>
                <td>.68</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>3 (2.2)</td>
                <td>.87</td>
                <td>.16</td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>How did you feel when you saw the result?</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Negative</td>
                <td>29 (14.6)</td>
                <td>&#60;.001<sup>e</sup></td>
                <td>.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Positive</td>
                <td>144 (72.4)</td>
                <td>&#60;.001<sup>e</sup></td>
                <td>.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Indifferent</td>
                <td>26 (13.1)</td>
                <td>&#60;.001<sup>e</sup></td>
                <td>.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>What actions have you taken after viewing your test results?<sup>d</sup></bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Spoke with family and/or friends</td>
                <td>101 (50.8)</td>
                <td>.86</td>
                <td>.45</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Looked up information on the web</td>
                <td>123 (61.8)</td>
                <td>.79</td>
                <td>.60</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Posted questions in online health forums</td>
                <td>22 (11.1)</td>
                <td>&#60;.001<sup>e</sup></td>
                <td>.89</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Created graph of results</td>
                <td>9 (4.5)</td>
                <td>.30</td>
                <td>.41</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Emailed doctor</td>
                <td>19 (9.6)</td>
                <td>.19</td>
                <td>.27</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Called doctor</td>
                <td>22 (11.1)</td>
                <td>.77</td>
                <td>.88</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Made doctor’s appointment</td>
                <td>34 (17.1)</td>
                <td>.09</td>
                <td>.09</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>11 (5.5)</td>
                <td>.89</td>
                <td>.27</td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>Do you need more information to interpret test results?</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Agree</td>
                <td>118 (59.3)</td>
                <td>.005<sup>b</sup></td>
                <td>.01<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Neither agree nor disagree</td>
                <td>58 (29.2)</td>
                <td>.005<sup>b</sup></td>
                <td>.01<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Disagree</td>
                <td>23 (11.6)</td>
                <td>.005<sup>b</sup></td>
                <td>.01<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="5">
                  <bold>What types of information do you need?<sup>d</sup></bold>
                  <bold>(n=119)</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Prognosis</td>
                <td>55 (46.2)</td>
                <td>.007<sup>b</sup></td>
                <td>.68</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Treatment options</td>
                <td>79 (66.4)</td>
                <td>.86</td>
                <td>.11</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>What to do or ask</td>
                <td>57 (47.9)</td>
                <td>.50</td>
                <td>.74</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Lifestyle changes</td>
                <td>42 (35.3)</td>
                <td>.50</td>
                <td>.72</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Connecting with local support group</td>
                <td>4 (3.4)</td>
                <td>.57</td>
                <td>.48</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Other</td>
                <td>1 (0.8)</td>
                <td>.45</td>
                <td>.27</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table4fn1">
              <p><sup>a</sup>In total, 4 participants could not remember the normality of their test results. These 4 participants were excluded from this analysis; therefore, the total number of cases was 199.</p>
            </fn>
            <fn id="table4fn2">
              <p><sup>b</sup>Results with <italic>P</italic> value&#60;.01.</p>
            </fn>
            <fn id="table4fn3">
              <p><sup>c</sup>Results with <italic>P</italic> value&#60;.05.</p>
            </fn>
            <fn id="table4fn4">
              <p><sup>d</sup>Participants can select more than one option for those questions.</p>
            </fn>
            <fn id="table4fn5">
              <p><sup>e</sup>Results with <italic>P</italic> value&#60;.001.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <table-wrap position="float" id="table5">
          <label>Table 5</label>
          <caption>
            <p>The association between 2 factors (health literacy and technology proficiency) and patient perceptions of using patient portals to understand and review laboratory test results (N=203).</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="350"/>
            <col width="0"/>
            <col width="180"/>
            <col width="0"/>
            <col width="180"/>
            <col width="0"/>
            <col width="260"/>
            <thead>
              <tr valign="top">
                <td colspan="2">Patient perceptions</td>
                <td colspan="2">Participants, n (%)</td>
                <td colspan="2">Health literacy (<italic>P</italic> value)</td>
                <td colspan="2">Technology proficiency (<italic>P</italic> value)</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="8">
                  <bold>I am comfortable with using patient portals to review my</bold>
                  <bold>laboratory</bold>
                  <bold>results</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Agree</td>
                <td colspan="2">185 (91.1)</td>
                <td colspan="2">.06</td>
                <td>.09</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Neither agree nor disagree</td>
                <td colspan="2">18 (8.9)</td>
                <td colspan="2">.06</td>
                <td>.09</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Disagree</td>
                <td colspan="2">0 (0.0)</td>
                <td colspan="2">.06</td>
                <td>.09</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>I never had any trouble checking my test results on the patient portal</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Agree</td>
                <td colspan="2">161 (79.3)</td>
                <td colspan="2">.38</td>
                <td>.002<sup>a</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Neither agree nor disagree</td>
                <td colspan="2">25 (12.3)</td>
                <td colspan="2">.38</td>
                <td>.002<sup>a</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Disagree</td>
                <td colspan="2">17 (8.4)</td>
                <td colspan="2">.38</td>
                <td>.002<sup>a</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>I find the patient portal can make me review my tests quickly</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Agree</td>
                <td colspan="2">187 (91.6)</td>
                <td colspan="2">.53</td>
                <td>&#60;.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Neither agree nor disagree</td>
                <td colspan="2">15 (7.4)</td>
                <td colspan="2">.53</td>
                <td>&#60;.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Disagree</td>
                <td colspan="2">1 (1.0)</td>
                <td colspan="2">.53</td>
                <td>&#60;.001<sup>b</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>I find the patient portal is useful to understand my</bold>
                  <bold>laboratory</bold>
                  <bold>results</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Agree</td>
                <td colspan="2">166 (81.3)</td>
                <td colspan="2">.15</td>
                <td>.02<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Neither agree nor disagree</td>
                <td colspan="2">30 (14.8)</td>
                <td colspan="2">.15</td>
                <td>.02<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Disagree</td>
                <td colspan="2">7 (3.9)</td>
                <td colspan="2">.15</td>
                <td>.02<sup>c</sup></td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>I have used the resources provided by patient portals to understand my results</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Agree</td>
                <td colspan="2">131 (64.0)</td>
                <td colspan="2">.03<sup>c</sup></td>
                <td>.22</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Neither agree nor disagree</td>
                <td colspan="2">40 (19.7)</td>
                <td colspan="2">.03<sup>c</sup></td>
                <td>.22</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Disagree</td>
                <td colspan="2">32 (16.3)</td>
                <td colspan="2">.03<sup>c</sup></td>
                <td>.22</td>
              </tr>
              <tr valign="top">
                <td colspan="8">
                  <bold>Is there anything that would make the portal better for you?<sup>d</sup></bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Make it more user-friendly</td>
                <td colspan="2">61 (29.1)</td>
                <td colspan="2">.70</td>
                <td>.82</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Allow me to send a message to my physician</td>
                <td colspan="2">83 (40.0)</td>
                <td colspan="2">.04<sup>c</sup></td>
                <td>.009<sup>a</sup></td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Include a health encyclopedia that contains more information about the test</td>
                <td colspan="2">99 (47.3)</td>
                <td colspan="2">.48</td>
                <td>.32</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Provide timely test result explanation and follow-up instructions</td>
                <td colspan="2">108 (52.7)</td>
                <td colspan="2">.007<sup>a</sup></td>
                <td>.07</td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td colspan="2">Other</td>
                <td colspan="2">2 (1.0)</td>
                <td colspan="2">.03<sup>c</sup></td>
                <td>.53</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table5fn1">
              <p><sup>a</sup>Results with <italic>P</italic> value <italic>P</italic>&#60;.01.</p>
            </fn>
            <fn id="table5fn2">
              <p><sup>b</sup>Results with <italic>P</italic> value <italic>P</italic>&#60;.001.</p>
            </fn>
            <fn id="table5fn3">
              <p><sup>c</sup>Results with <italic>P</italic> value <italic>P</italic>&#60;.05.</p>
            </fn>
            <fn id="table5fn4">
              <p><sup>d</sup>Participants can select more than one option for those questions.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>We used thematic analysis [<xref ref-type="bibr" rid="ref36">36</xref>] to analyze the interview data. We first reviewed the interview transcripts to obtain an overview of the context. In the subsequent stage, we transferred data into NVivo (QSR International), a program for organizing, storing, and manipulating qualitative data. Two authors conducted open coding over the interview data and met regularly to discuss codes and code definitions, consolidate and refine codes until they reached consensus. In the second round of analysis, coded data were grouped under themes using affinity diagrams [<xref ref-type="bibr" rid="ref37">37</xref>]. Themes describing the challenges and information needs of interpreting laboratory results and perceptions of receiving and reviewing results via current delivery mechanisms emerged after the second round of coding.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Challenges in Comprehending Laboratory Test Results</title>
        <p>The results showed that only 55.2% (112/203) of participants reported that they were contacted by their physician before reviewing the results in the patient portal, suggesting that it is not always the case for patients to receive timely notification and explanation of the results from their physician. One participant explained: “I didn’t actually talk to the doctor until probably like a week after that, because I wasn't able to get a hold of them” (P10). This finding is consistent with previous work stating that patients usually did not receive any explanatory information from their physician when they received the result [<xref ref-type="bibr" rid="ref38">38</xref>]. Compared with patients who received normal results, those with abnormal results were more likely to be contacted (<italic>P</italic>=.001). When the physician explanations were absent, it is surprising to see that some participants chose not to bother their physician:</p>
        <disp-quote>
          <p>We have these mega practices, they’re incredibly busy, and I think if I did, I’d probably be afraid of being labeled as a trouble patient, and you don't want to be a trouble patient. I want to reserve those chips for when I really need them.</p>
          <attrib>P3</attrib>
        </disp-quote>
        <p>We also found that participants with relatively high technology proficiency were more likely to possess positive opinions regarding the usefulness of patient portals (eg, the patient portal was useful in reviewing the test quickly [187/203, 91.1%; <italic>P</italic>&#60;.001] and understanding their laboratory results [166/203, 81.3%; <italic>P</italic>=.02]). In a similar vein, participants with relatively high health literacy would be more likely to use resources provided by patient portals to make sense of the laboratory results (131/203, 64%; <italic>P</italic>=.03). These findings align with previous research showing that there is a digital divide in the use and adoption of health technologies—people with higher literacy and numeracy can benefit more from using health technologies to manage their health [<xref ref-type="bibr" rid="ref19">19</xref>].</p>
        <p>Finally, our participants indicated that existing patient portals lack sufficient and useful information for patients to understand their results: “In my particular case, I am trying to understand the meaning of the various test results. I didn't see anything on the patient portal that helped me with that at all” (P4). The lack of timely explanation from a primary care physician exacerbates this challenge, leaving patients entangled with how to interpret the meaning of their results accurately.</p>
      </sec>
      <sec>
        <title>Patients’ Confusion About Laboratory Test Results</title>
        <p>Given these challenges, it is not surprising that many participants have difficulties understanding their laboratory results. We found that 70 survey participants (70/199, 35.2%) were unsure whether they understood their laboratory results. We believe this is because making sense of laboratory test results is a multilayered issue. Patients need to not only know whether they received a normal or abnormal result but also understand the meaning and indications of the test results. Despite participants reporting using different mechanisms to interpret the normality of test results (eg, visual cues on patient portals and personal knowledge), we found that 136 out of 203 participants (136/203, 67.0%) still reported having confusion about different aspects of their laboratory results, including medical terminology, reference range, the meaning of laboratory value, and the effects on their health care.</p>
        <p>Participants deemed technical jargons, such as medical terminology and abbreviations, to be most confusing (83/136, 61.0%):</p>
        <disp-quote>
          <p>Just some of the specifics to the result. Like in the blood tests, words like neutrophils, or CBC, or bilirubin, however you pronounce that. Those very specific things that as a person who is not trained in medicine, I didn't understand without having to consult the internet.</p>
          <attrib>P4</attrib>
        </disp-quote>
        <p>Although some patient portals have started implementing consumer-friendly vocabularies [<xref ref-type="bibr" rid="ref39">39</xref>], this challenge persists—those explanations of medical terms may use additional medical jargon, which further confuses the patients. This perception was echoed by other participants who stated that patient portals overly use professional medical terms that are not easy to understand for lay individuals.</p>
        <p>Our participants also had trouble differentiating between normal and abnormal laboratory results (43/136, 31.6%), in particular, whether a specific laboratory value falls into the normal range: “Some of the numbers don't even have a point of reference for me, so I don't understand that” (P12). In addition, some participants also reported confusion about some test results that are not in a numerical format, such as <italic>false positive</italic> and <italic>nonreactive</italic>. One participant explained:</p>
        <disp-quote>
          <p>I get what the HIV means. And then like you actually even click further to details and it didn't even say “Negative”, it is actually “Reactive”. That’s what I didn't understand. I didn't know what non-reactive meant.</p>
          <attrib>P6</attrib>
        </disp-quote>
        <p>For those who received out-of-range test results, they reported that they struggled to understand how bad their abnormal results were and how alarmed they should feel.</p>
        <p>Finally, participants expressed that they lacked an understanding of the meaning of the results (54/136, 39.7%) and the effects on their overall health care (47/136, 35.6%):</p>
        <disp-quote>
          <p>I don't know exactly what it (test result) meant clinically and for my overall health. I just kind of had a general idea. So I’m undergoing cancer treatment and so they’re checking my white blood cell counts. Those were low, but I don't really understand are they too low? I don't know necessarily what all of the things are. Like the neutrophils and I don't know what all of that means if it’s out of range.</p>
          <attrib>P13</attrib>
        </disp-quote>
      </sec>
      <sec>
        <title>Unmet Information Needs</title>
        <p>A considerable number of participants (119/203, 58.6%) expressed an interest in obtaining more pertinent information in addition to that provided by patient portals to interpret their test results. In particular, both normality of the laboratory test (<italic>P</italic>=.005) and health literacy (<italic>P</italic>=.01) were significantly associated with this requestparticipants with abnormal results (43/58, 74.1%) and relatively high health literacy were more interested in receiving additional information (116/192, 60.4%). Some participants would like to know more generic information, such as reference range and diagnostic abilities of a specific test (eg, “I guess I liked to know an explanation of what the normal levels should be [...] and what each of the tests was testing” [P9]). In contrast, others preferred personalized and contextual information (eg, “If it was more personalized based on a patient’s history, that would definitely be more helpful because then it would be something you could really know [if you] need to act on this or not” [P12]). The types of most needed personalized information include treatment options, what to do or ask, prognosis, and lifestyle changes.</p>
        <p>More specifically, many of the participants (79/119, 66.4%) would like to know more information about treatment options, including medication and medical procedures: “I want to know something like treatment options or medications” (P8). Of particular interest here is that 35.5% (50/119) of the participants whose results were normal also expressed an interest in knowing the treatment options. One possible explanation is that the participants were asked to report the normality of their most recent laboratory test result. Therefore, some participants may receive normal results for the latest test, but they may have received an abnormal result before. We speculate that these participants expressed their general preferences and information needs not only based on the most recent result but also on their overall experience.</p>
        <p>Our participants also needed assistance in deciphering the meaning of the results and understanding the potential prognosis (56/119, 47.1%):</p>
        <disp-quote>
          <p>I would like to see a little more information as far as what might be causing it if there is a result that’s not normal. I would like to see a little more as far as what possibly could be there.</p>
          <attrib>P12</attrib>
        </disp-quote>
        <p>Interestingly, laboratory test normality was strongly associated with whether participants would like to know prognosis information (<italic>P</italic>=.007).</p>
        <p>There was also a demand to be informed about what they could do to cope with the bothersome symptoms they were experiencing or what they should ask the doctor during the next clinic visit (58/119, 48.7%): “I liked to know suggestions of what I should do. […] If I get an abnormal result, I would be wondering what I could be doing to improve” [P9]. In particular, people also wanted to know whether changing their lifestyle (eg, diet and exercise) could be of any help (42/119, 35.3%). Knowing such information could help them better manage their health care (eg, take actions to address elevated creatinine levels) and make informed decisions (eg, whether and when to see a doctor).</p>
      </sec>
      <sec>
        <title>Emotional Aspect of Viewing Laboratory Results</title>
        <p>Despite only 29 out of 203 participants (29/203, 14.3%) reporting the experience of negative emotions (eg, concerns, anxiety, and frustration) while viewing the results, we noticed that laboratory result normality was strongly associated with feelings (<italic>P</italic>&#60;.001). This is consistent with previous work [<xref ref-type="bibr" rid="ref23">23</xref>]—participants with abnormal results were more likely to experience negative emotions (25/58, 43.1%). One participant shared her story and expressed the necessity of getting emotional support when the result is abnormal:</p>
        <disp-quote>
          <p>I was diagnosed having cancer. So I need to do a bunch of lab tests regularly to keep track of everything. I was kind of scared every single time when I was checking the test results online. I hoped to see improved numbers. But they were not always good and when that happened, my level of anxiety skyrocketed in a second. It can be very tough.</p>
          <attrib>P10</attrib>
        </disp-quote>
      </sec>
      <sec>
        <title>Actions Taken</title>
        <p>To fulfill their knowledge gap and emotional needs, participants took several actions, including speaking with family and/or friends (101/203, 49.6%), looking up information online (124/203, 61.1%), posting questions in online health care forums (23/203, 11.3%), emailing and/or calling a doctor (42/203, 20.7%), and making a doctor’s appointment (35/203, 17.2%). These findings show that many people sought information about their results from sources other than their physician [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
        <p>In particular, laboratory test normality was significantly associated with posting questions online (<italic>P</italic>&#60;.001)—participants with abnormal results would be more likely to post questions in online health care forums compared with those receiving normal results (14/58, 24% vs 8/141, 5.7%). By doing so, they were able to prepare questions or sought relevant information before their next clinic visit. However, several participants reported finding misleading and/or disturbing information on the internet (eg, “once you hit the forums, you often get crazy time”). A few participants also mentioned that online searches helped clarify medical terms but did not yield much useful information for comprehending specific results (eg, “It’s too general. It’s not helpful”).</p>
      </sec>
      <sec>
        <title>Strategies to Improve Patient Portals</title>
        <p>When asked how to make the patient portal better and useful for the comprehension of laboratory results, participants ranked “provide timely test result explanation and follow-up instructions” (108/203, 52.7%), “include a health encyclopedia that contains more information about the test” (99/203, 47.3%), and “allow me to send a message to my physician” (83/203, 40.8%) as the top 3 needed features.</p>
        <p>Our participants also emphasized the importance of making the patient portals “more user-friendly” (61/203, 29.1%). For instance, patient portals should be designed with patients in mind rather than for clinician interpretation only, as one participant stated:</p>
        <disp-quote>
          <p>So maybe making it less like I’m looking at the system that the doctor would be using and more something that’s designed with the patient in mind and understanding who the patient is and not somebody who’s going to use like the very specific language.</p>
          <attrib>P6</attrib>
        </disp-quote>
        <p>They further suggested that visualizing historical results (“What I really, really think needs to be done is the ability to track your abnormal results over time”) and using lay terms to explain the result (“make it like a 10-year-old would understand”) could be very useful in improving the user experience of patient portals.</p>
        <p>The design of patient portals should also take into consideration marginalized user groups, including patients with disability and older adult patients, to minimize the disparities in the use of patient portals [<xref ref-type="bibr" rid="ref40">40</xref>]. One participant explained why this issue is important to address:</p>
        <disp-quote>
          <p>I noticed with patient portals is they’re not very accessible to people with vision issues. It doesn't have to be fancy, but a little button that says “please push F to read this for them”, and then they could hear it. [...] or maybe the doctor gives a specialized URL and he like “hey, go to this URL, and it will be for blind assist” [...] I would say it needs to be a little bit more use-friendly as far as older folks go. [...] So since the world is having a national aging problem, it should be geared more for grandma to understand it.</p>
          <attrib>P5</attrib>
        </disp-quote>
        <p>Finally, it is interesting to see that several participants described the potential of incorporating artificial intelligence (AI) technology into patient portals so that their data can be used to generate more personalized medical information:</p>
        <disp-quote>
          <p>I know right now a big part of medicine is going to be AI where you actually interact with artificial intelligent agents. They’ll probably have your information right there and it’ll probably make it a lot easier for you to get a lot of your answers because they’ll be accessing your information right there and giving you the answers to your questions.</p>
          <attrib>P2</attrib>
        </disp-quote>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>Although health care providers are expanding patients’ access to laboratory test results through patient portals, less than one-third of Americans had accessed this information online [<xref ref-type="bibr" rid="ref7">7</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. We conducted this study to better understand lay people’s perceptions about reviewing and comprehending laboratory test results through patient portals, including their perceived challenges and confusions as well as informational, emotional, and technological needs. The challenges faced by patients when reviewing laboratory test results (eg, lack of explanatory information from physicians and lack of useful information on patient portals) inevitably caused confusion about different aspects of their laboratory report, such as the meaning of laboratory values and the indications on their health care. To cope with these challenges and confusion, our participants expressed the urgent need to obtain a variety of information to comprehend laboratory results, including treatment options, what to do or ask, prognosis, and lifestyle changes. The desired enhancements of patient portals include providing timely explanations and educational resources (eg, a health encyclopedia), increasing usability and accessibility, and incorporating AI-based technology to provide personalized recommendations. Finally, we found that several intrinsic factors (eg, laboratory result normality, health literacy, and technology proficiency) had significant impacts on people’s perceptions of using portals to view and interpret laboratory results. For example, participants with relatively high health literacy were more likely to understand the meaning of test results and use resources provided by patient portals to make sense of their data, while people with relatively high technology proficiency tended to agree that the patient portal was easy to use and very helpful in reviewing and understanding their laboratory values. These findings highlight significant disparities in laboratory result comprehension and patient portal use among certain groups of the population.</p>
      </sec>
      <sec>
        <title>Design Implications</title>
        <p>At present, the design of test results in patient portals seems to assume that patients have sufficient medical knowledge about their test results. In addition, patients often did not receive explanatory information at the time they received the result. One primary reason is that many physicians would prefer explaining results during a face-to-face clinical encounter to avoid any potential miscommunications [<xref ref-type="bibr" rid="ref41">41</xref>]. However, on the other hand, patients are increasingly interested in accessing and interpreting their results. Many of them would search online to conduct their research [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref25">25</xref>,<xref ref-type="bibr" rid="ref26">26</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. Our participants articulated that internet searches were helpful in clarifying medical terms; however, the online information was not always reliable, and they even found contradicting or disturbing information on the internet, which caused further confusion and anxiety [<xref ref-type="bibr" rid="ref28">28</xref>]. It is therefore crucial to provide more useful, credible, and actionable information to aid patient understanding, something current patient portals do not provide [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
        <p>With regard to the types of information to provide, our results indicate that people have different information needs, including generic information (eg, reference range and diagnostic abilities of a specific test) and personalized or contextual information (eg, treatment options, prognosis, and what to do or ask next). This variety of information needs may be due to different levels of health literacy and numeracy that people have and whether patients receive normal or abnormal results [<xref ref-type="bibr" rid="ref19">19</xref>]. For example, some patients may not be literate enough to understand the medical terminologies or normal ranges of a test, whereas others need help with more complicated issues, such as interpreting the laboratory results in the context of their medical history. It is, therefore, important to assess the patient’s knowledge level and provide corresponding support based on patients’ characteristics [<xref ref-type="bibr" rid="ref42">42</xref>,<xref ref-type="bibr" rid="ref43">43</xref>]. One way to accomplish this is to allow patients to add their own notes about their background (eg, health literacy, medical condition, etc) to a specific section in their patient portals to enable automatic assessment and customized support. It may also be useful to provide links to consumer-friendly and trusted information sources (eg, entries in MedlinePlus) to assist those who do not have high levels of health literacy and numeracy to read and interpret laboratory content [<xref ref-type="bibr" rid="ref42">42</xref>]. We also noticed that people with abnormal results were more likely to ask for prognosis information. This indicates that providing interpretation and other advanced information (eg, prognosis, treatment options) to patients who received abnormal results at the time of portal release should be considered a best practice [<xref ref-type="bibr" rid="ref15">15</xref>], as it can support patients’ information needs and reduce any anxiety they may experience.</p>
        <p>We are not arguing that we should provide all kinds of information and support to patients at one time because they may overwhelm patients and create even more confusion. Instead, we believe that the delivery of additional information should be tailored to patients’ individual preferences, that is, patients should have options to decide whether they want to receive additional information in patient portals and what the information is at the time of ordering the test. Those who do not want to receive additional information can opt out and wait until their physician to provide explanations. This practice aligns with the SPIKES guideline (S: setting, P: perception, I: invitation, K: knowledge, E: empathy, S: summarize) of delivering health information to patients [<xref ref-type="bibr" rid="ref44">44</xref>]. We also see design examples where patients can click buttons or hyperlinked texts to see additional information [<xref ref-type="bibr" rid="ref29">29</xref>]. Presenting additional information in <italic>pop-ups</italic> is expected to not only ease the navigation of the interface but also empower patients to decide whether they would like to see further details.</p>
        <p>A lot of the challenges in result comprehension faced by patients can, in part, be attributed to the interface design of patient portals. We examined how patient portals can be improved to support understanding. In line with previous work [<xref ref-type="bibr" rid="ref29">29</xref>], we conclude that merely providing access to abundant data is insufficient; instead, the patient portal should employ user-centered design strategies to help patients better interpret and manage their test results. For example, the test results should be displayed to patients in ways they can comprehend and grasp the meaning of each test result to inform their decision making and subsequent actions. For example, it would be much easier for patients to interpret their results by delivering intuitive <italic>gist</italic> messages (eg, “your result was normal,” “your cholesterol levels were optimal”) when presenting numerical values [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref35">35</xref>]. Furthermore, as our participants described, they struggled to map a particular abnormal test result to its clinical meaning. Although most patient portals provide patients with a standard reference range to discriminate between normal and abnormal test results, patients cannot necessarily understand how bad the result is and if any urgent action is needed. The literature has suggested that it might be useful to provide an additional reference point to indicate how far outside the standard range that values become clinically concerning [<xref ref-type="bibr" rid="ref21">21</xref>]. By doing so, it will help patients better distinguish different types of abnormal test results, that is, slightly out-of-range results versus extreme values, and in turn reduce the perceived urgency of abnormal results that do not require immediate clinical action [<xref ref-type="bibr" rid="ref21">21</xref>].</p>
        <p>Furthermore, our participants discussed the possibility of incorporating AI technology into patient portals to make use of their clinical data to generate more personalized medical information. Indeed, with the recent rise in the capabilities of AI, it is expected that AI-empowered health care systems can intelligently search, retrieve, and present pertinent information to patients within the context of their medical history [<xref ref-type="bibr" rid="ref45">45</xref>]. For example, although many hospitals and physicians are still using the 2.5 or 3.0 mU/L cutoff (2.5 mU/L in the first trimester and 3.0 mU/L in the second and third trimesters) for the thyroid-stimulating hormone during pregnancy, a recent study pointed out that these cutoffs are too low and may lead to overdiagnosis or even overtreatment [<xref ref-type="bibr" rid="ref46">46</xref>]. The AI-driven patient portals should be able to recommend such relevant and up-to-date medical evidence to patients based on their test results. Patient portals may embody AI technology to change the landscape of how people receive laboratory results and seek medical advice.</p>
        <p>Finally, although our participants wanted to be informed about the meaning of diagnostic results, needless anxiety and negative emotions may arise when communicating <italic>abnormal</italic> diagnostic results without the presence of a medical professional. In particular, we found that patients with abnormal results were more likely to experience negative emotions. Therefore, it is necessary to provide additional support or resources for patients to mitigate emotional stress when communicating sensitive results through patient portals. For example, patients should have options to determine whether they want to receive concerning results via the portal. This could ensure that patient portals deliver potentially stressful health information in a more empathic manner [<xref ref-type="bibr" rid="ref47">47</xref>]. In addition, previous work has shown that patient navigators can reduce negative emotions, such as, anxiety, for patients who receive abnormal mammography while waiting for follow-up testing [<xref ref-type="bibr" rid="ref48">48</xref>].</p>
        <p>However, implementing these design suggestions is challenging. This is because current patient portals lack consistency in the design for communicating laboratory test results across different portal vendors, which not only frustrates patients but also creates barriers for incorporating <italic>best practices</italic> and new features to different portals [<xref ref-type="bibr" rid="ref23">23</xref>,<xref ref-type="bibr" rid="ref49">49</xref>]. Therefore, an urgent research agenda is determining how to develop standards and guidelines and encourage portal providers to incorporate them to enhance patients’ understanding of their test results.</p>
        <p>In brief, when communicating test results via portals, it is necessary to include information about the test itself (eg, test capability, the purpose of test), easy-to-understand explanations of medical jargon, the results in the context of the patient’s health, directions for next steps, and specific educational resources [<xref ref-type="bibr" rid="ref35">35</xref>,<xref ref-type="bibr" rid="ref50">50</xref>,<xref ref-type="bibr" rid="ref51">51</xref>]. Furthermore, designers and developers of patient portals should embody user-centered approaches to significantly improve the design of patient portals to present information in a more meaningful way [<xref ref-type="bibr" rid="ref29">29</xref>,<xref ref-type="bibr" rid="ref52">52</xref>]. Finally, national test result notification and interface design standards for patient portals should be developed to ensure interoperability and consistency in features across portal vendors. Health policies should be enacted to support these strategies [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
      </sec>
      <sec>
        <title>Limitations and Future Research</title>
        <p>A few limitations should be noted. First, our survey and interview participants self-reported their experiences with using patient portals to view laboratory results. Some of our respondents may have difficulties recalling such experiences. As described in the Methods section, we provided a commonly seen laboratory test to participants to cope with this issue. Second, our results may not generalize to all types of patients because we recruited our participants online and our sample inevitably consisted of a large majority of participants who self-reported to have medium to high levels of technology proficiency and health literacy. This limitation of participant recruitment could affect the generalizability of the results because there is a lack of a great representative of marginalized populations who can benefit most from this research. In our future work, we will include more marginalized populations (eg, less literate people, older adults) and examine how to improve their understanding of laboratory results. Third, this study did not investigate which patient portal our participants used. However, we believe it has a limited impact on our results as many patient portals share similar characteristics, such as displaying results in a tabular format with a reference range. Finally, we will design and prototype a new interface of patient portals for test results based on our findings. We will involve different types of patients with various characteristics and use a multiphased, user-centered approach combined with rapid prototyping and formative evaluation.</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>In conclusion, our findings suggest that there are challenges for patients to comprehend their laboratory test results through the patient portal. This is mainly because of the lack of communication with physicians and the lack of support and useful information in current patient portals. Several factors, including participants’ health literacy and technology proficiency as well as laboratory result normality, have impacts on people’s perceptions of using patient portals to understand the laboratory results. Our participants have a variety of information needs, ranging from general information (eg, reference range and diagnostic abilities of a specific test) to personalized information (eg, treatment options, prognosis, and what to do or ask next). Participants also emphasized the importance of improving patient portals to better meet their needs. This study contributes to an empirical, in-depth understanding of the challenges and needs of patients in comprehending laboratory test results and informs strategies and design implications for informatics tools to promote patients’ comprehension of test results.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Survey questionnaire.</p>
        <media xlink:href="jmir_v22i12e18725_app1.pdf" xlink:title="PDF File  (Adobe PDF File), 181 KB"/>
      </supplementary-material>
      <supplementary-material id="app2">
        <label>Multimedia Appendix 2</label>
        <p>Interview protocol.</p>
        <media xlink:href="jmir_v22i12e18725_app2.pdf" xlink:title="PDF File  (Adobe PDF File), 91 KB"/>
      </supplementary-material>
      <supplementary-material id="app3">
        <label>Multimedia Appendix 3</label>
        <p>Patient perceptions of interpreting test results and the association to health literacy and normality of lab test and patient perceptions of using patient portal and the association to health literacy and technology proficiency.</p>
        <media xlink:href="jmir_v22i12e18725_app3.docx" xlink:title="DOCX File , 38 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AI</term>
          <def>
            <p>artificial intelligence</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">CMA</term>
          <def>
            <p>chromosomal microarray</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">MTurk</term>
          <def>
            <p>Amazon Mechanical Turk</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">NIH</term>
          <def>
            <p>National Institutes of Health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">RQ</term>
          <def>
            <p>research question</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This study was supported by the Pace University Scholarly Research Award, the National Institute on Aging of the National Institutes of Health (NIH) under award number R21AG061431 and the Florida State University-University of Florida Clinical and Translational Science Award funded by the National Center for Advancing Translational Sciences under award number UL1TR001427. The content is solely the responsibility of the authors and does not necessarily represent the official views of the NIH.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lippi</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Mattiuzzi</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Critical laboratory values communication: summary recommendations from available guidelines</article-title>
          <source>Ann Transl Med</source>
          <year>2016</year>
          <month>10</month>
          <volume>4</volume>
          <issue>20</issue>
          <fpage>400</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.21037/atm.2016.09.36"/>
          </comment>
          <pub-id pub-id-type="doi">10.21037/atm.2016.09.36</pub-id>
          <pub-id pub-id-type="medline">27867952</pub-id>
          <pub-id pub-id-type="pii">atm-04-20-400</pub-id>
          <pub-id pub-id-type="pmcid">PMC5107409</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pillemer</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Price</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Paone</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Martich</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Albert</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Haidari</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Updike</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Rudin</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>DA</given-names>
            </name>
            <name name-style="western">
              <surname>Mehrotra</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Direct release of test results to patients increases patient engagement and utilization of care</article-title>
          <source>PLoS One</source>
          <year>2016</year>
          <volume>11</volume>
          <issue>6</issue>
          <fpage>e0154743</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.plos.org/10.1371/journal.pone.0154743"/>
          </comment>
          <pub-id pub-id-type="doi">10.1371/journal.pone.0154743</pub-id>
          <pub-id pub-id-type="medline">27337092</pub-id>
          <pub-id pub-id-type="pii">PONE-D-14-55360</pub-id>
          <pub-id pub-id-type="pmcid">PMC4919031</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Baldwin</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Quintela</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Duclos</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Staton</surname>
              <given-names>EW</given-names>
            </name>
            <name name-style="western">
              <surname>Pace</surname>
              <given-names>WD</given-names>
            </name>
          </person-group>
          <article-title>Patient preferences for notification of normal laboratory test results: a report from the ASIPS Collaborative</article-title>
          <source>BMC Fam Pract</source>
          <year>2005</year>
          <month>03</month>
          <day>8</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>11</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcfampract.biomedcentral.com/articles/10.1186/1471-2296-6-11"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1471-2296-6-11</pub-id>
          <pub-id pub-id-type="medline">15755328</pub-id>
          <pub-id pub-id-type="pii">1471-2296-6-11</pub-id>
          <pub-id pub-id-type="pmcid">PMC555570</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ahern</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Woods</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Lightowler</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Finley</surname>
              <given-names>ST</given-names>
            </name>
            <name name-style="western">
              <surname>Houston</surname>
              <given-names>TK</given-names>
            </name>
          </person-group>
          <article-title>Promise of and potential for patient-facing technologies to enable meaningful use</article-title>
          <source>Am J Prev Med</source>
          <year>2011</year>
          <month>05</month>
          <volume>40</volume>
          <issue>5 Suppl 2</issue>
          <fpage>S162</fpage>
          <lpage>72</lpage>
          <pub-id pub-id-type="doi">10.1016/j.amepre.2011.01.005</pub-id>
          <pub-id pub-id-type="medline">21521591</pub-id>
          <pub-id pub-id-type="pii">S0749-3797(11)00069-9</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="web">
          <article-title>Opennotes and AMIA Join Forces to Improve Patient Access to Health Records</article-title>
          <source>MedCity News</source>
          <access-date>2020-11-28</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medcitynews.com/2017/11/opennotes-amia/">https://medcitynews.com/2017/11/opennotes-amia/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Berlin</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Communicating results of all radiologic examinations directly to patients: has the time come?</article-title>
          <source>Am J Roentgenol</source>
          <year>2007</year>
          <month>12</month>
          <volume>189</volume>
          <issue>6</issue>
          <fpage>1275</fpage>
          <lpage>82</lpage>
          <pub-id pub-id-type="doi">10.2214/ajr.07.2740</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Peacock</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Reddy</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Leveille</surname>
              <given-names>SG</given-names>
            </name>
            <name name-style="western">
              <surname>Walker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Payne</surname>
              <given-names>TH</given-names>
            </name>
            <name name-style="western">
              <surname>Oster</surname>
              <given-names>NV</given-names>
            </name>
            <name name-style="western">
              <surname>Elmore</surname>
              <given-names>JG</given-names>
            </name>
          </person-group>
          <article-title>Patient portals and personal health information online: perception, access, and use by US adults</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2017</year>
          <month>04</month>
          <day>1</day>
          <volume>24</volume>
          <issue>e1</issue>
          <fpage>e173</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1093/jamia/ocw095</pub-id>
          <pub-id pub-id-type="medline">27413120</pub-id>
          <pub-id pub-id-type="pii">ocw095</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>O'Kane</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Explaining laboratory test results to patients: what the clinician needs to know</article-title>
          <source>Br Med J</source>
          <year>2015</year>
          <month>12</month>
          <day>3</day>
          <volume>351</volume>
          <fpage>h5552</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.h5552</pub-id>
          <pub-id pub-id-type="medline">26634382</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Johnson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Easterling</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Nelson</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>MR</given-names>
            </name>
            <name name-style="western">
              <surname>Frankel</surname>
              <given-names>RM</given-names>
            </name>
          </person-group>
          <article-title>Access to radiologic reports via a patient portal: clinical simulations to investigate patient preferences</article-title>
          <source>J Am Coll Radiol</source>
          <year>2012</year>
          <month>04</month>
          <volume>9</volume>
          <issue>4</issue>
          <fpage>256</fpage>
          <lpage>63</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jacr.2011.12.023</pub-id>
          <pub-id pub-id-type="medline">22469376</pub-id>
          <pub-id pub-id-type="pii">S1546-1440(11)00780-0</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Walker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Meltsner</surname>
              <given-names>MT</given-names>
            </name>
            <name name-style="western">
              <surname>Delbanco</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>US experience with doctors and patients sharing clinical notes</article-title>
          <source>Br Med J</source>
          <year>2015</year>
          <month>03</month>
          <day>10</day>
          <volume>350</volume>
          <fpage>g7785</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.g7785</pub-id>
          <pub-id pub-id-type="medline">25670182</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ross</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Moore</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Earnest</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Wittevrongel</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Lin</surname>
              <given-names>CT</given-names>
            </name>
          </person-group>
          <article-title>Providing a web-based online medical record with electronic communication capabilities to patients with congestive heart failure: randomized trial</article-title>
          <source>J Med Internet Res</source>
          <year>2004</year>
          <month>05</month>
          <day>14</day>
          <volume>6</volume>
          <issue>2</issue>
          <fpage>e12</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2004/2/e12/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.6.2.e12</pub-id>
          <pub-id pub-id-type="medline">15249261</pub-id>
          <pub-id pub-id-type="pii">v6e12</pub-id>
          <pub-id pub-id-type="pmcid">PMC1550594</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>O'Kane</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Freedman</surname>
              <given-names>DB</given-names>
            </name>
            <name name-style="western">
              <surname>Zikmund-Fisher</surname>
              <given-names>BJ</given-names>
            </name>
          </person-group>
          <article-title>Can patients use test results effectively if they have direct access?</article-title>
          <source>Br Med J</source>
          <year>2015</year>
          <month>03</month>
          <day>11</day>
          <volume>350</volume>
          <fpage>h673</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.h673</pub-id>
          <pub-id pub-id-type="medline">25673132</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Witteman</surname>
              <given-names>HO</given-names>
            </name>
            <name name-style="western">
              <surname>Zikmund-Fisher</surname>
              <given-names>BJ</given-names>
            </name>
          </person-group>
          <article-title>Communicating laboratory results to patients and families</article-title>
          <source>Clin Chem Lab Med</source>
          <year>2019</year>
          <month>02</month>
          <day>25</day>
          <volume>57</volume>
          <issue>3</issue>
          <fpage>359</fpage>
          <lpage>64</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.degruyter.com/doi/10.1515/cclm-2018-0634"/>
          </comment>
          <pub-id pub-id-type="doi">10.1515/cclm-2018-0634</pub-id>
          <pub-id pub-id-type="medline">30407910</pub-id>
          <pub-id pub-id-type="pii">/j/cclm.ahead-of-print/cclm-2018-0634/cclm-2018-0634.xml</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hickner</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Wilkinson</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Epner</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Sheehan</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Pollock</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Duke</surname>
              <given-names>CC</given-names>
            </name>
            <name name-style="western">
              <surname>Jackson</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>JR</given-names>
            </name>
          </person-group>
          <article-title>Primary care physicians' challenges in ordering clinical laboratory tests and interpreting results</article-title>
          <source>J Am Board Fam Med</source>
          <year>2014</year>
          <volume>27</volume>
          <issue>2</issue>
          <fpage>268</fpage>
          <lpage>74</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jabfm.org/cgi/pmidlookup?view=long&#38;pmid=24610189"/>
          </comment>
          <pub-id pub-id-type="doi">10.3122/jabfm.2014.02.130104</pub-id>
          <pub-id pub-id-type="medline">24610189</pub-id>
          <pub-id pub-id-type="pii">27/2/268</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zikmund-Fisher</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Exe</surname>
              <given-names>NL</given-names>
            </name>
            <name name-style="western">
              <surname>Witteman</surname>
              <given-names>HO</given-names>
            </name>
          </person-group>
          <article-title>Numeracy and literacy independently predict patients' ability to identify out-of-range test results</article-title>
          <source>J Med Internet Res</source>
          <year>2014</year>
          <month>08</month>
          <day>8</day>
          <volume>16</volume>
          <issue>8</issue>
          <fpage>e187</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2014/8/e187/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.3241</pub-id>
          <pub-id pub-id-type="medline">25135688</pub-id>
          <pub-id pub-id-type="pii">v16i8e187</pub-id>
          <pub-id pub-id-type="pmcid">PMC4137189</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McCray</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Promoting health literacy</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2004</year>
          <month>11</month>
          <day>23</day>
          <volume>12</volume>
          <issue>2</issue>
          <fpage>152</fpage>
          <lpage>63</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1197/jamia.M1687"/>
          </comment>
          <pub-id pub-id-type="doi">10.1197/jamia.m1687</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alpert</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Krist</surname>
              <given-names>AH</given-names>
            </name>
            <name name-style="western">
              <surname>Aycock</surname>
              <given-names>RA</given-names>
            </name>
            <name name-style="western">
              <surname>Kreps</surname>
              <given-names>GL</given-names>
            </name>
          </person-group>
          <article-title>Applying multiple methods to comprehensively evaluate a patient portal's effectiveness to convey information to patients</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>05</month>
          <day>17</day>
          <volume>18</volume>
          <issue>5</issue>
          <fpage>e112</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/5/e112/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.5451</pub-id>
          <pub-id pub-id-type="medline">27188953</pub-id>
          <pub-id pub-id-type="pii">v18i5e112</pub-id>
          <pub-id pub-id-type="pmcid">PMC4887660</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hong</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Feustel</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Agnihotri</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Silverman</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Simoneaux</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Wilcox</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Supporting Families in Reviewing and Communicating about Radiology Imaging Studies</article-title>
          <source>Proceedings of the  CHI Conference on Human Factors in Computing Systems</source>
          <year>2017</year>
          <month>05</month>
          <conf-name>CHI'17</conf-name>
          <conf-date>May 12-14, 2017</conf-date>
          <conf-loc>Denver, CO</conf-loc>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/28920107"/>
          </comment>
          <pub-id pub-id-type="doi">10.1145/3025453.3025754</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sarkar</surname>
              <given-names>U</given-names>
            </name>
            <name name-style="western">
              <surname>Karter</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Liu</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Adler</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Nguyen</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lopez</surname>
              <given-names>AD</given-names>
            </name>
            <name name-style="western">
              <surname>Schillinger</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>The literacy divide: health literacy and the use of an internet-based patient portal in an integrated health system-results from the diabetes study of northern California (DISTANCE)</article-title>
          <source>J Health Commun</source>
          <year>2010</year>
          <volume>15</volume>
          <issue>Suppl 2</issue>
          <fpage>183</fpage>
          <lpage>96</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/20845203"/>
          </comment>
          <pub-id pub-id-type="doi">10.1080/10810730.2010.499988</pub-id>
          <pub-id pub-id-type="medline">20845203</pub-id>
          <pub-id pub-id-type="pii">926943837</pub-id>
          <pub-id pub-id-type="pmcid">PMC3014858</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Overhage</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Brown</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Aghighi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Entwistle</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Hui</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Hyde</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Klieman</surname>
              <given-names>LC</given-names>
            </name>
            <name name-style="western">
              <surname>Mitchell</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Perkins</surname>
              <given-names>AJ</given-names>
            </name>
            <name name-style="western">
              <surname>Qureshi</surname>
              <given-names>LS</given-names>
            </name>
            <name name-style="western">
              <surname>Waltimyer</surname>
              <given-names>TA</given-names>
            </name>
            <name name-style="western">
              <surname>Winters</surname>
              <given-names>LJ</given-names>
            </name>
            <name name-style="western">
              <surname>Young</surname>
              <given-names>CY</given-names>
            </name>
          </person-group>
          <article-title>Online disease management of diabetes: engaging and motivating patients online with enhanced resources-diabetes (EMPOWER-D), a randomized controlled trial</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2013</year>
          <month>05</month>
          <day>1</day>
          <volume>20</volume>
          <issue>3</issue>
          <fpage>526</fpage>
          <lpage>34</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/23171659"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/amiajnl-2012-001263</pub-id>
          <pub-id pub-id-type="medline">23171659</pub-id>
          <pub-id pub-id-type="pii">amiajnl-2012-001263</pub-id>
          <pub-id pub-id-type="pmcid">PMC3628059</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zikmund-Fisher</surname>
              <given-names>BJ</given-names>
            </name>
            <name name-style="western">
              <surname>Scherer</surname>
              <given-names>AM</given-names>
            </name>
            <name name-style="western">
              <surname>Witteman</surname>
              <given-names>HO</given-names>
            </name>
            <name name-style="western">
              <surname>Solomon</surname>
              <given-names>JB</given-names>
            </name>
            <name name-style="western">
              <surname>Exe</surname>
              <given-names>NL</given-names>
            </name>
            <name name-style="western">
              <surname>Fagerlin</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Effect of harm anchors in visual displays of test results on patient perceptions of urgency about near-normal values: experimental study</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>03</month>
          <day>26</day>
          <volume>20</volume>
          <issue>3</issue>
          <fpage>e98</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2018/3/e98/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.8889</pub-id>
          <pub-id pub-id-type="medline">29581088</pub-id>
          <pub-id pub-id-type="pii">v20i3e98</pub-id>
          <pub-id pub-id-type="pmcid">PMC5891666</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Peters</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Dieckmann</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Dixon</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hibbard</surname>
              <given-names>JC</given-names>
            </name>
            <name name-style="western">
              <surname>Mertz</surname>
              <given-names>CK</given-names>
            </name>
          </person-group>
          <article-title>Less is more in presenting quality information to consumers</article-title>
          <source>Med Care Res Rev</source>
          <year>2007</year>
          <month>04</month>
          <volume>64</volume>
          <issue>2</issue>
          <fpage>169</fpage>
          <lpage>90</lpage>
          <pub-id pub-id-type="doi">10.1177/10775587070640020301</pub-id>
          <pub-id pub-id-type="medline">17406019</pub-id>
          <pub-id pub-id-type="pii">64/2/169</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Giardina</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Baldwin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Nystrom</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Sittig</surname>
              <given-names>DH</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Patient perceptions of receiving test results via online portals: a mixed-methods study</article-title>
          <source>J Am Med Inform Assoc</source>
          <year>2018</year>
          <month>04</month>
          <day>1</day>
          <volume>25</volume>
          <issue>4</issue>
          <fpage>440</fpage>
          <lpage>6</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/29240899"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/jamia/ocx140</pub-id>
          <pub-id pub-id-type="medline">29240899</pub-id>
          <pub-id pub-id-type="pii">4689172</pub-id>
          <pub-id pub-id-type="pmcid">PMC5885801</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Christensen</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Sue</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Viewing laboratory test results online: patients' actions and reactions</article-title>
          <source>J Particip Med</source>
          <year>2013</year>
          <volume>5</volume>
          <fpage>e38</fpage>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Reynolds</surname>
              <given-names>TL</given-names>
            </name>
            <name name-style="western">
              <surname>Ali</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>McGregor</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>O'Brien</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Longhurst</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Rosenberg</surname>
              <given-names>AL</given-names>
            </name>
            <name name-style="western">
              <surname>Rudkin</surname>
              <given-names>SE</given-names>
            </name>
            <name name-style="western">
              <surname>Zheng</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Understanding patient questions about their medical records in an online health forum: opportunity for patient portal design</article-title>
          <source>AMIA Annu Symp Proc</source>
          <year>2017</year>
          <volume>2017</volume>
          <fpage>1468</fpage>
          <lpage>77</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/29854216"/>
          </comment>
          <pub-id pub-id-type="medline">29854216</pub-id>
          <pub-id pub-id-type="pmcid">PMC5977702</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Lu</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Kou</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Wu</surname>
              <given-names>DT</given-names>
            </name>
            <name name-style="western">
              <surname>Huh-Yoo</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>Z</given-names>
            </name>
          </person-group>
          <source>Stud Health Technol Inform</source>
          <year>2019</year>
          <month>08</month>
          <day>21</day>
          <volume>264</volume>
          <fpage>1403</fpage>
          <lpage>7</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/31438157"/>
          </comment>
          <pub-id pub-id-type="doi">10.3233/SHTI190458</pub-id>
          <pub-id pub-id-type="medline">31438157</pub-id>
          <pub-id pub-id-type="pii">SHTI190458</pub-id>
          <pub-id pub-id-type="pmcid">PMC6857529</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wilcox</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Patel</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Shachak</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Human factors in computing systems: focus on patient-centered health communication at the ACM SIGCHI conference</article-title>
          <source>Patient Educ Couns</source>
          <year>2013</year>
          <month>12</month>
          <volume>93</volume>
          <issue>3</issue>
          <fpage>532</fpage>
          <lpage>4</lpage>
          <pub-id pub-id-type="doi">10.1016/j.pec.2013.09.017</pub-id>
          <pub-id pub-id-type="medline">24184039</pub-id>
          <pub-id pub-id-type="pii">S0738-3991(13)00392-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Reiff</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Bernhardt</surname>
              <given-names>BA</given-names>
            </name>
            <name name-style="western">
              <surname>Mulchandani</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Soucier</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Cornell</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Pyeritz</surname>
              <given-names>RE</given-names>
            </name>
            <name name-style="western">
              <surname>Spinner</surname>
              <given-names>NB</given-names>
            </name>
          </person-group>
          <article-title>'What does it mean?': uncertainties in understanding results of chromosomal microarray testing</article-title>
          <source>Genet Med</source>
          <year>2012</year>
          <month>03</month>
          <volume>14</volume>
          <issue>2</issue>
          <fpage>250</fpage>
          <lpage>8</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/22241091"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/gim.2011.52</pub-id>
          <pub-id pub-id-type="medline">22241091</pub-id>
          <pub-id pub-id-type="pii">gim201152</pub-id>
          <pub-id pub-id-type="pmcid">PMC3445036</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nystrom</surname>
              <given-names>DT</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Baldwin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sittig</surname>
              <given-names>DF</given-names>
            </name>
            <name name-style="western">
              <surname>Giardina</surname>
              <given-names>TD</given-names>
            </name>
          </person-group>
          <article-title>Methods for patient-centered interface design of test result display in online portals</article-title>
          <source>EGEMS (Wash DC)</source>
          <year>2018</year>
          <month>06</month>
          <day>26</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>15</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/30094287"/>
          </comment>
          <pub-id pub-id-type="doi">10.5334/egems.255</pub-id>
          <pub-id pub-id-type="medline">30094287</pub-id>
          <pub-id pub-id-type="pmcid">PMC6078112</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Denscombe</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Communities of practice</article-title>
          <source>J Mix Method Res</source>
          <year>2008</year>
          <month>07</month>
          <volume>2</volume>
          <issue>3</issue>
          <fpage>270</fpage>
          <lpage>83</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1177/1558689808316807"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/1558689808316807</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mason</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Mixing methods in a qualitatively driven way</article-title>
          <source>Qual Res</source>
          <year>2016</year>
          <month>08</month>
          <day>15</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>9</fpage>
          <lpage>25</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1177/1468794106058866"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/1468794106058866</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Axinn</surname>
              <given-names>WG</given-names>
            </name>
            <name name-style="western">
              <surname>Pearce</surname>
              <given-names>LD</given-names>
            </name>
          </person-group>
          <source>Mixed Method Data Collection Strategies</source>
          <year>2006</year>
          <publisher-loc>Cambridge, England</publisher-loc>
          <publisher-name>Cambridge University Press</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shapiro</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Chandler</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Mueller</surname>
              <given-names>PA</given-names>
            </name>
          </person-group>
          <article-title>Using mechanical Turk to study clinical populations</article-title>
          <source>Clin Psychol Sci</source>
          <year>2013</year>
          <month>01</month>
          <day>31</day>
          <volume>1</volume>
          <issue>2</issue>
          <fpage>213</fpage>
          <lpage>20</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1177/2167702612469015"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/2167702612469015</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Taha</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Sharit</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Czaja</surname>
              <given-names>SJ</given-names>
            </name>
          </person-group>
          <article-title>The impact of numeracy ability and technology skills on older adults' performance of health management tasks using a patient portal</article-title>
          <source>J Appl Gerontol</source>
          <year>2014</year>
          <month>06</month>
          <volume>33</volume>
          <issue>4</issue>
          <fpage>416</fpage>
          <lpage>36</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/24781964"/>
          </comment>
          <pub-id pub-id-type="doi">10.1177/0733464812447283</pub-id>
          <pub-id pub-id-type="medline">24781964</pub-id>
          <pub-id pub-id-type="pii">0733464812447283</pub-id>
          <pub-id pub-id-type="pmcid">PMC5808916</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Elder</surname>
              <given-names>NC</given-names>
            </name>
            <name name-style="western">
              <surname>Barney</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>'But what does it mean for me?' Primary care patients' communication preferences for test results notification</article-title>
          <source>Jt Comm J Qual Patient Saf</source>
          <year>2012</year>
          <month>04</month>
          <volume>38</volume>
          <issue>4</issue>
          <fpage>168</fpage>
          <lpage>76</lpage>
          <pub-id pub-id-type="doi">10.1016/s1553-7250(12)38022-7</pub-id>
          <pub-id pub-id-type="medline">22533129</pub-id>
          <pub-id pub-id-type="pii">S1553-7250(12)38022-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Braun</surname>
              <given-names>VV</given-names>
            </name>
            <name name-style="western">
              <surname>Clarke</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Using thematic analysis in psychology</article-title>
          <source>Qual Res Psychol</source>
          <year>2006</year>
          <month>01</month>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>77</fpage>
          <lpage>101</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1191/1478088706qp063oa"/>
          </comment>
          <pub-id pub-id-type="doi">10.1191/1478088706qp063oa</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Beyer</surname>
              <given-names>HK</given-names>
            </name>
            <name name-style="western">
              <surname>Holtzblatt</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <source>Contextual design: defining customer-centered systems</source>
          <year>1998</year>
          <publisher-loc>New York</publisher-loc>
          <publisher-name>Elsevier</publisher-name>
          <fpage>978</fpage>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Cheng</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Tang</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Siek</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Bardram</surname>
              <given-names>JE</given-names>
            </name>
          </person-group>
          <article-title>Is My Doctor Listening to Me? Impact of Health IT Systems on Patient-provider Interaction</article-title>
          <source>Extended Abstracts on Human Factors in Computing Systems</source>
          <year>2013</year>
          <conf-name>CHI'13</conf-name>
          <conf-date>April 27, 2013</conf-date>
          <conf-loc>Paris, France</conf-loc>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1145/2468356.2468791"/>
          </comment>
          <pub-id pub-id-type="doi">10.1145/2468356.2468791</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Irizarry</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>DeVito Dabbs</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Curran</surname>
              <given-names>CR</given-names>
            </name>
          </person-group>
          <article-title>Patient portals and patient engagement: a state of the science review</article-title>
          <source>J Med Internet Res</source>
          <year>2015</year>
          <month>06</month>
          <day>23</day>
          <volume>17</volume>
          <issue>6</issue>
          <fpage>e148</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2015/6/e148/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.4255</pub-id>
          <pub-id pub-id-type="medline">26104044</pub-id>
          <pub-id pub-id-type="pii">v17i6e148</pub-id>
          <pub-id pub-id-type="pmcid">PMC4526960</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Yamin</surname>
              <given-names>CK</given-names>
            </name>
            <name name-style="western">
              <surname>Emani</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>DH</given-names>
            </name>
            <name name-style="western">
              <surname>Lipsitz</surname>
              <given-names>SR</given-names>
            </name>
            <name name-style="western">
              <surname>Karson</surname>
              <given-names>AS</given-names>
            </name>
            <name name-style="western">
              <surname>Wald</surname>
              <given-names>JS</given-names>
            </name>
            <name name-style="western">
              <surname>Bates</surname>
              <given-names>DW</given-names>
            </name>
          </person-group>
          <article-title>The digital divide in adoption and use of a personal health record</article-title>
          <source>Arch Intern Med</source>
          <year>2011</year>
          <month>03</month>
          <day>28</day>
          <volume>171</volume>
          <issue>6</issue>
          <fpage>568</fpage>
          <lpage>74</lpage>
          <pub-id pub-id-type="doi">10.1001/archinternmed.2011.34</pub-id>
          <pub-id pub-id-type="medline">21444847</pub-id>
          <pub-id pub-id-type="pii">171/6/568</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Miller</surname>
              <given-names>DP</given-names>
            </name>
            <name name-style="western">
              <surname>Latulipe</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Melius</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Quandt</surname>
              <given-names>SA</given-names>
            </name>
            <name name-style="western">
              <surname>Arcury</surname>
              <given-names>TA</given-names>
            </name>
          </person-group>
          <article-title>Primary care providers' views of patient portals: interview study of perceived benefits and consequences</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>01</month>
          <day>15</day>
          <volume>18</volume>
          <issue>1</issue>
          <fpage>e8</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/1/e8/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.4953</pub-id>
          <pub-id pub-id-type="medline">26772771</pub-id>
          <pub-id pub-id-type="pii">v18i1e8</pub-id>
          <pub-id pub-id-type="pmcid">PMC4733220</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref42">
        <label>42</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>MS</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Oh</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bian</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Consumers’ use of UMLS concepts on social media: diabetes-related textual data analysis in blog and social Q&#38;A sites</article-title>
          <source>JMIR Med Inform</source>
          <year>2016</year>
          <month>11</month>
          <day>24</day>
          <volume>4</volume>
          <issue>4</issue>
          <fpage>e41</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://medinform.jmir.org/2016/4/e41/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/medinform.5748</pub-id>
          <pub-id pub-id-type="medline">27884812</pub-id>
          <pub-id pub-id-type="pii">v4i4e41</pub-id>
          <pub-id pub-id-type="pmcid">PMC5146325</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref43">
        <label>43</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Yu</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Xing</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Lustria</surname>
              <given-names>ML</given-names>
            </name>
          </person-group>
          <article-title>An informatics framework to assess consumer health language complexity differences: proof-of-concept study</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>05</month>
          <day>21</day>
          <volume>22</volume>
          <issue>5</issue>
          <fpage>e16795</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/5/e16795/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/16795</pub-id>
          <pub-id pub-id-type="medline">32436849</pub-id>
          <pub-id pub-id-type="pii">v22i5e16795</pub-id>
          <pub-id pub-id-type="pmcid">PMC7273233</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref44">
        <label>44</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Baile</surname>
              <given-names>WF</given-names>
            </name>
            <name name-style="western">
              <surname>Buckman</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Lenzi</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Glober</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Beale</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Kudelka</surname>
              <given-names>AP</given-names>
            </name>
          </person-group>
          <article-title>SPIKES-A six-step protocol for delivering bad news: application to the patient with cancer</article-title>
          <source>Oncologist</source>
          <year>2000</year>
          <volume>5</volume>
          <issue>4</issue>
          <fpage>302</fpage>
          <lpage>11</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://onlinelibrary.wiley.com/resolve/openurl?genre=article&#38;sid=nlm:pubmed&#38;issn=1083-7159&#38;date=2000&#38;volume=5&#38;issue=4&#38;spage=302"/>
          </comment>
          <pub-id pub-id-type="doi">10.1634/theoncologist.5-4-302</pub-id>
          <pub-id pub-id-type="medline">10964998</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref45">
        <label>45</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Curioni-Fontecedro</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>A new era of oncology through artificial intelligence</article-title>
          <source>ESMO Open</source>
          <year>2017</year>
          <volume>2</volume>
          <issue>2</issue>
          <fpage>e000198</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://esmoopen.bmj.com/lookup/pmidlookup?view=long&#38;pmid=28761754"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/esmoopen-2017-000198</pub-id>
          <pub-id pub-id-type="medline">28761754</pub-id>
          <pub-id pub-id-type="pii">esmoopen-2017-000198</pub-id>
          <pub-id pub-id-type="pmcid">PMC5519782</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref46">
        <label>46</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Korevaar</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>The upper limit for TSH during pregnancy: why we should stop using fixed limits of 2.5 or 3.0 mU/l</article-title>
          <source>Thyroid Res</source>
          <year>2018</year>
          <volume>11</volume>
          <fpage>5</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://thyroidresearchjournal.biomedcentral.com/articles/10.1186/s13044-018-0048-7"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13044-018-0048-7</pub-id>
          <pub-id pub-id-type="medline">29942352</pub-id>
          <pub-id pub-id-type="pii">48</pub-id>
          <pub-id pub-id-type="pmcid">PMC5963163</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref47">
        <label>47</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Choe</surname>
              <given-names>EK</given-names>
            </name>
            <name name-style="western">
              <surname>Duarte</surname>
              <given-names>ME</given-names>
            </name>
            <name name-style="western">
              <surname>Suh</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Pratt</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Kientz</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>Communicating bad news: insights for the design of consumer health technologies</article-title>
          <source>JMIR Hum Factors</source>
          <year>2019</year>
          <month>05</month>
          <day>17</day>
          <volume>6</volume>
          <issue>2</issue>
          <fpage>e8885</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://humanfactors.jmir.org/2019/2/e8885/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/humanfactors.8885</pub-id>
          <pub-id pub-id-type="medline">31102374</pub-id>
          <pub-id pub-id-type="pii">v6i2e8885</pub-id>
          <pub-id pub-id-type="pmcid">PMC6543800</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref48">
        <label>48</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ferrante</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>PS</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>The effect of patient navigation on time to diagnosis, anxiety, and satisfaction in urban minority women with abnormal mammograms: a randomized controlled trial</article-title>
          <source>J Urban Health</source>
          <year>2008</year>
          <month>01</month>
          <volume>85</volume>
          <issue>1</issue>
          <fpage>114</fpage>
          <lpage>24</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/17906931"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s11524-007-9228-9</pub-id>
          <pub-id pub-id-type="medline">17906931</pub-id>
          <pub-id pub-id-type="pmcid">PMC2430139</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref49">
        <label>49</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <collab>Centers for Disease ControlPrevention (CDC)‚ HHS</collab>
            <collab>Office for Civil Rights (OCR)‚ HHS</collab>
          </person-group>
          <article-title>CLIA program and HIPAA privacy rule; patients' access to test reports. Final rule</article-title>
          <source>Fed Regist</source>
          <year>2014</year>
          <month>03</month>
          <day>6</day>
          <volume>79</volume>
          <issue>25</issue>
          <fpage>7289</fpage>
          <lpage>316</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.gpo.gov/fdsys/pkg/FR-2014-02-06/pdf/2014-02280.pdf"/>
          </comment>
          <pub-id pub-id-type="medline">24605389</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref50">
        <label>50</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Giardina</surname>
              <given-names>TD</given-names>
            </name>
            <name name-style="western">
              <surname>Modi</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Parrish</surname>
              <given-names>DE</given-names>
            </name>
            <name name-style="western">
              <surname>Singh</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>The patient portal and abnormal test results: an exploratory study of patient experiences</article-title>
          <source>Patient Exp J</source>
          <year>2015</year>
          <volume>2</volume>
          <issue>1</issue>
          <fpage>148</fpage>
          <lpage>54</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/28345018"/>
          </comment>
          <pub-id pub-id-type="medline">28345018</pub-id>
          <pub-id pub-id-type="pmcid">PMC5363705</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref51">
        <label>51</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kelman</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Robinson</surname>
              <given-names>CO</given-names>
            </name>
            <name name-style="western">
              <surname>Cochin</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Ahluwalia</surname>
              <given-names>NJ</given-names>
            </name>
            <name name-style="western">
              <surname>Braverman</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chiauzzi</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Simacek</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Communicating laboratory test results for rheumatoid factor: what do patients and physicians want?</article-title>
          <source>Patient Prefer Adherence</source>
          <year>2016</year>
          <volume>10</volume>
          <fpage>2501</fpage>
          <lpage>17</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dx.doi.org/10.2147/PPA.S104396"/>
          </comment>
          <pub-id pub-id-type="doi">10.2147/PPA.S104396</pub-id>
          <pub-id pub-id-type="medline">28008236</pub-id>
          <pub-id pub-id-type="pii">ppa-10-2501</pub-id>
          <pub-id pub-id-type="pmcid">PMC5171200</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref52">
        <label>52</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Martinez</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Threatt</surname>
              <given-names>AL</given-names>
            </name>
            <name name-style="western">
              <surname>Rosenbloom</surname>
              <given-names>ST</given-names>
            </name>
            <name name-style="western">
              <surname>Wallston</surname>
              <given-names>KA</given-names>
            </name>
            <name name-style="western">
              <surname>Hickson</surname>
              <given-names>GB</given-names>
            </name>
            <name name-style="western">
              <surname>Elasy</surname>
              <given-names>TA</given-names>
            </name>
          </person-group>
          <article-title>A patient-facing diabetes dashboard embedded in a patient web portal: design sprint and usability testing</article-title>
          <source>JMIR Hum Factors</source>
          <year>2018</year>
          <month>09</month>
          <day>24</day>
          <volume>5</volume>
          <issue>3</issue>
          <fpage>e26</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://humanfactors.jmir.org/2018/3/e26/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/humanfactors.9569</pub-id>
          <pub-id pub-id-type="medline">30249579</pub-id>
          <pub-id pub-id-type="pii">v5i3e26</pub-id>
          <pub-id pub-id-type="pmcid">PMC6231745</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
