<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
    <front>
        <journal-meta>
            <journal-id journal-id-type="publisher-id">JMIR</journal-id>
            <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
            <journal-title>Journal of Medical Internet Research</journal-title>
            <issn pub-type="epub">1438-8871</issn>
            <publisher>
                <publisher-name>Gunther Eysenbach</publisher-name>
                <publisher-loc>Centre for Global eHealth Innovation, Toronto, Canada</publisher-loc>
            </publisher>
        </journal-meta>
        <article-meta>
            <article-id pub-id-type="publisher-id">v4i1e1</article-id>
            <article-id pub-id-type="pmid">11956033</article-id>
            <article-id pub-id-type="medline">21953329</article-id>
            <article-id pub-id-type="doi">10.2196/jmir.4.1.e1</article-id>
            <article-categories>
                <subj-group subj-group-type="article-type">
                    <subject>Original Paper</subject>
                </subj-group>
            </article-categories>
            <title-group>
                <article-title>A Framework for the Evaluation of Internet-based Diabetes Management</article-title>
            </title-group>
            <contrib-group>
                <contrib contrib-type="author" corresp="yes" id="contrib1" xlink:type="simple">
                    <name name-style="western">
                        <surname>Mazzi</surname>
                        <given-names>Christian P</given-names>
                    </name>
                    <degrees>MBA</degrees>
                    <address>
                        <institution>Faculty of Medicine</institution>
                        <institution>The University of Sydney</institution>
                        <addr-line>70A Fullers Rd.</addr-line>
                        <addr-line>Chatswood NSW 2067</addr-line>
                        <country>Australia</country>
                        <phone>+61 0404 149326</phone>
                        <email xlink:type="simple">cmazzi@gmp.usyd.edu.au</email>
                    </address>
                    <role>Professor and Head</role>
                    <xref ref-type="aff" rid="aff1">1</xref>
                </contrib>
                <contrib contrib-type="author" id="contrib2" xlink:type="simple">
                    <name name-style="western">
                        <surname>Kidd</surname>
                        <given-names>Michael</given-names>
                    </name>
                    <degrees>MD</degrees>
                    <xref ref-type="aff" rid="aff2">2</xref>
                </contrib>
             </contrib-group>
            <contrib-group>
                  <contrib contrib-type="reviewer" xlink:type="simple">
                    <name name-style="western">
                        <surname>Mack</surname>
                        <given-names>J</given-names>
                    </name>
                </contrib>
            </contrib-group>
            <aff id="aff1" rid="aff1"><sup>1</sup>
                <institution xlink:type="simple">Faculty of Medicine</institution>
                <institution>The University of Sydney</institution>
                <addr-line>Chatswood NSW 2067</addr-line>
                <country>Australia</country>
            </aff>
            <aff id="aff2" rid="aff2"><sup>2</sup>
                <institution>Department of General Practice</institution>
                <institution>The University of Sydney</institution>
                <addr-line>Balmain 2041</addr-line>
                <addr-line>Sydney</addr-line>
                <country>Australia</country>
            </aff>
            <pub-date pub-type="collection">
                <season>Jan-Mar</season>
                <year>2002</year>
            </pub-date>
            <pub-date pub-type="epub">
                <day>10</day>
                <month>1</month>
                <year>2002</year>
            </pub-date>
            <volume>4</volume>
            <issue>1</issue>
            <elocation-id>e1</elocation-id>
            <history>
                <date date-type="received">
                    <day>11</day>
                    <month>11</month>
                    <year>2001</year>
                </date>
                <date date-type="rev-request">
                    <day>3</day>
                    <month>12</month>
                    <year>2001</year>
                </date>
                <date date-type="rev-recd">
                    <day>17</day>
                    <month>12</month>
                    <year>2001</year>
                </date>
                <date date-type="accepted">
                    <day>17</day>
                    <month>12</month>
                    <year>2001</year>
                </date>
            </history>
            <copyright-statement>&#xA9; Christian P Mazzi, Michael Kidd.  Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 10.1.2002.  Except where otherwise noted, articles published in the Journal of Medical Internet Research are distributed under the terms of the Creative Commons Attribution License (http://www.creativecommons.org/licenses/by/2.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work is properly cited, including full bibliographic	details and the URL (see "please cite as" above), and this statement is included.</copyright-statement>
            <copyright-year>2003</copyright-year>
            <self-uri xlink:href="http://www.jmir.org/2002/1/e1/" xlink:type="simple"/>
            <abstract>
                <sec sec-type="background">
                    <title>Background</title>
                    <p>While still in its infancy, Internet-based diabetes management shows great promise for growth. However, the following aspects must be considered: what are the key metrics for the evaluation of a diabetes management site? how should these sites grow in the future and what services should they offer?</p>
                </sec>
                <sec sec-type="objectives">
                    <title>Objectives</title>
                    <p>To examine the needs of the patient and the health care professional in an Internet-based diabetes management solution and how these needs are translated into services offered.</p>
                </sec>
                <sec sec-type="methods">
                    <title>Methods</title>
                    <p>An evaluation framework was constructed based on a literature review that identified the requirements for an Internet-based diabetes management solution. The requirements were grouped into 5 categories: Monitoring, Information, Personalization, Communication, and Technology. Two of the market leaders (myDiabetes and LifeMasters) were selected and were evaluated with the framework. The Web sites were evaluated independently by 5 raters using the evaluation framework. All evaluations were performed from November 1, 2001 through December 15, 2001.</p>
                </sec>
                <sec sec-type="results">
                    <title>Results</title>
                    <p>The agreement level between raters ranged from 60% to 100%. The multi-rater reliability (kappa) was 0.75 for myDiabetes and 0.65 for LifeMasters, indicating substantial agreement. The results of the evaluations indicate that LifeMasters is a more-complete solution than myDiabetes in all dimensions except Information, where both sites were equivalent. LifeMasters satisfied 32 evaluation criteria while myDiabetes satisfied 24 evaluation criteria, out of a possible 40 in the framework.</p>
                </sec>
                <sec sec-type="conclusions">
                    <title>Conclusions</title>
                    <p>The framework is based on the recognition that the management of diabetes via the Internet is based on several integrated dimensions: Monitoring, Information, Personalization, Communication, and Technology. A successful diabetes management system should efficiently integrate all dimensions. The evaluation found that LifeMasters is successful in integrating the health care professional in the management of diabetes and that MyDiabetes is quite effective in providing a communication channel for community creation (however, communication with the health care professional is lacking).</p>
                </sec>
            </abstract>
            <kwd-group>
                <kwd>Diabetes</kwd>
                <kwd>Internet</kwd>
                <kwd>evaluation</kwd>
                <kwd>therapy</kwd>
                <kwd>chronic disease management</kwd>
            </kwd-group>
        </article-meta>
    </front>
    <body>
        <sec sec-type="intro">
            <title>Introduction</title>
            <p>Management of patients with chronic conditions is a long-standing challenge for health care organizations. These conditions include diabetes, chronic heart failure (CHF), chronic obstructive pulmonary disease (COPD), Asthma, HIV/AIDS, and cancer. Patients are required to adopt lifelong exercise, diet, and drug regimens to maintain optimal health and avoid the complications of the disease. These complications can arise suddenly and be life threatening; therefore, patients with chronic diseases must be monitored constantly [<xref ref-type="bibr" rid="ref1">1</xref>].</p>
            <p>In recent years, Internet-based home telemonitoring systems have become available [<xref ref-type="bibr" rid="ref2">2</xref>]. These sites leverage the Internet to record, measure, monitor, manage, and deliver health care. These information-technology solutions are creating a link between patient and caregiver that enables patients to supply a steady stream of valuable health information to caregivers. For example, diabetics can report their blood glucose readings, thus creating a history of their glucose control, which caregivers can use to evaluate the impact of a therapy (eg, short acting insulin) or the need for a different one [<xref ref-type="bibr" rid="ref1">1</xref>]. Conversely, caregivers have the ability to provide their patients with crucial information and feedback on the management of their disease. For example, patients can be notified about screening appointments for the complications of diabetes. Therefore, patients benefit from an improved control and understanding of the disease; the ability to self-monitor from home reduces the burden of the disease. These solutions have resulted in dramatic improvements in disease management as measured by hospitalizations [<xref ref-type="bibr" rid="ref1">1</xref>] and in an overall reduction in costs [<xref ref-type="bibr" rid="ref3">3</xref>]. Further, patients report higher levels of satisfaction and better control of their conditions [<xref ref-type="bibr" rid="ref4">4</xref>].</p>
            <p>Diabetes is a chronic disease that affects 30 million people worldwide [<xref ref-type="bibr" rid="ref5">5</xref>] and is the seventh leading cause of death in the United States [<xref ref-type="bibr" rid="ref6">6</xref>]. The total annual economic cost of diabetes in 1997 was estimated to be US $98 billion. That includes US $44 billion in direct medical and treatment costs and US $54 billion for indirect costs attributed to disability and humanity [<xref ref-type="bibr" rid="ref7">7</xref>] and a significant intrusion in the life of an individual. In managing diabetes, success is measured by positive change in prognostic indicators and outcomes. Below is a list of measurement criteria used in diabetes management [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>,<xref ref-type="bibr" rid="ref10">10</xref>].</p>
            <list list-type="bullet">
                <list-item>
                    <p>Greater patient self-efficacy</p>
                </list-item>
                <list-item>
                    <p>Greater satisfaction with care, continuity, provider, quality of health outcome</p>
                </list-item>
                <list-item>
                    <p>Decreased HbA <sub>1c</sub>and blood glucose levels</p>
                </list-item>
                <list-item>
                    <p>Improved diet and body weight control</p>
                </list-item>
                <list-item>
                    <p>Lowered cholesterol</p>
                </list-item>
                <list-item>
                    <p>Lowered perception of diabetes intrusiveness</p>
                </list-item>
                <list-item>
                    <p>Improved quality of life</p>
                </list-item>
                <list-item>
                    <p>Less depression</p>
                </list-item>
                <list-item>
                    <p>Decreased incidence of diabetic complications.</p>
                </list-item>
            </list>
            <p>Primarily, diabetes must be managed by the patient because it requires adherence to stringent dietary, physical, and medical regimes [<xref ref-type="bibr" rid="ref8">8</xref>]. Internet-based diabetes management systems have the potential of reducing the burden of disease both to the patient and to the health care system. A recent study found that a high proportion of patients are willing to use Internet resources in the management of their disease [<xref ref-type="bibr" rid="ref9">9</xref>]. The driving forces behind the proliferation of technology for disease management is the patients' demands to get real-time help, get real-time information, and keep in contact with their physician [<xref ref-type="bibr" rid="ref1">1</xref>]. Not surprisingly, several diabetes-specific sites have recently appeared [<xref ref-type="bibr" rid="ref10">10</xref>], including myDiabetes, Health Hero Network, LifeChart, LifeMasters, and Medifor.</p>
            <p>The purpose of this paper is to review the patient's and the health care professional's needs in an Internet-based diabetes management solution and to examine how these needs are addressed in practice. An evaluation framework was constructed by grouping the requirements of an Internet-based diabetes management solution into 5 categories: Monitoring, Information, Personalization, Communication, and Technology. Two of the market leaders (myDiabetes and LifeMasters) were selected and evaluated to illustrate the use of the framework.</p>
        </sec>
        <sec sec-type="methods">
            <title>Methods</title>
            <p>A literature search was conducted on medical databases (Medline, Pre-Medline, EMBASE, Cochrane, and PubMed) and a nonmedical database (Expanded Academic ASAP). The articles were identified by <italic>diabetes</italic>, <italic>chronic disease</italic>, <italic>internet</italic>, and <italic>technology</italic>. The searches were based on the following <italic>AND</italic> combinations of these keywords.</p>
            <list list-type="bullet">
                <list-item>
                    <p>
                        <italic>diabetes</italic> AND <italic>internet</italic>
                    </p>
                </list-item>
                <list-item>
                    <p>
                        <italic>diabetes</italic> AND <italic>technology</italic>
                    </p>
                </list-item>
                <list-item>
                    <p>
                        <italic>chronic disease</italic> AND <italic>internet</italic>
                    </p>
                </list-item>
                <list-item>
                    <p>
                        <italic>chronic disease</italic> AND <italic>technology</italic>
                    </p>
                </list-item>
            </list>
            <p>The exact search methodology differed among databases due to differences in their user interfaces. The methodology for each database is summarized in <xref ref-type="table" rid="table1">Table 1</xref>.</p>
            <p>The abstracts of the articles retrieved by the searches were screened for relevance by the authors. The relevant articles were reviewed in order to compile a comprehensive list of requirements for an Internet-based diabetes management solution. These requirements were identified on the following basis:</p>
            <list list-type="bullet">
                <list-item>
                    <p>No interdependence between requirements</p>
                </list-item>
                <list-item>
                    <p>Requirements can be assessed as present or not present</p>
                </list-item>
                <list-item>
                    <p>Equal implementation effort required to satisfy the requirements.</p>
                </list-item>
            </list>
            <p>The implementation effort was quantified by the number of Use Cases as defined by the Universal Modeling Language (UML) [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. The number of Use Cases ranged from 1 to 3 for each requirement. For example, the requirement defined as <italic>User defined parameter-Patient</italic> allows patients to define which health parameter they wish to monitor. This functionality requires 3 Use Cases: <italic>Identify User,</italic>
                <italic>Retrieve Parameters</italic>, and <italic>Save Parameters</italic>.</p>
            <p>The requirements for Internet-based diabetes management were compiled into the criteria of an evaluation framework. The evaluation criteria were grouped into 5 categories: Monitoring, Information, Personalization, Communication, and Technology. The evaluation framework is presented in <xref ref-type="table" rid="table2">Table 2</xref> and the evaluation criteria are discussed in detail in the "Evaluation Criteria" section of the "Results" section.</p>
            <p>To illustrate the use of the evaluation framework, we have applied it to 2 existing Internet-based diabetes management systems: my Diabetes (<ext-link ext-link-type="uri" xlink:href="http://www.myDiabetes.com" xlink:type="simple">www.myDiabetes.com</ext-link>) and LifeMasters (<ext-link ext-link-type="uri" xlink:href="http://www.lifemasters.com" xlink:type="simple">www.lifemasters.com</ext-link>). These 2 sites were selected because they were first movers in the arena of Internet-based diabetes management. MyDiabetes.com was one of the first sites going live in July 1999, shortly followed by LifeMasters.com in October 1999.</p>
            <p>The sites were evaluated from November 1, 2001 through December 15, 2001. The evaluations were performed by 5 independent evaluators who were not aware of each other's ratings. All evaluators are computer literate and are familiar with the use of the Internet. The evaluators included a physician, 3 diabetic patients, and one author [CM]. All the evaluators registered separately with both sites (registration was free). Each evaluator was given a detailed description of the evaluation criteria, as described in the "Results" section, and <xref ref-type="table" rid="table2">Table 2</xref>, which describes the framework. The evaluators were also given an evaluation form to fill out (effectively <xref ref-type="table" rid="table3">Table 3</xref> without results). For each criterion, the evaluators rated the sites as <italic>Yes</italic> if the criterion was satisfied or <italic>No</italic> if it was not satisfied. The evaluations were not supervised.</p>
            <p>
                <xref ref-type="fig" rid="figure1">Figure 1</xref> and <xref ref-type="fig" rid="figure2">Figure 2</xref> are screen shots of the entry forms for the daily glucose measurements forms at myDiabetes and LifeMasters respectively. This basic function of diabetes monitoring requires the user to input his or her blood glucose levels and the time of the readings. The data is stored, effectively creating a log of the glucose control of the patient. LifeMasters records glucose levels based on relative times such as <italic>Bedtime</italic> and asks for symptoms of high and low blood glucose as well as diabetic complications. Mydiabetes records the exact time of the blood glucose measurement but does not screen for any symptoms; this is done in another section of the site.</p>
            <table-wrap id="table1" position="float">
                <label>Table 1</label>
                <caption>
                    <p>Search methodologies for databases</p>
                </caption>
                <table border="1" cellpadding="4" cellspacing="0" rules="groups">
                    <col width="200"/>
                    <col width="600"/>
                    <thead>
                        <tr>
                            <td valign="top">
                                <bold>Database</bold>
                            </td>
                            <td valign="top">
                                <bold>Search Methodology</bold>
                            </td>
                        </tr>
                    </thead>
                    <tbody>
                        <tr>
                            <td valign="top">Medline (1966 to October week 5, 2001)</td>
                            <td valign="top">
                                <italic>AND</italic>
                                <list list-type="order">
                                    <list-item>
                                        <p>
                                            <italic>diabetes</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>chronic disease</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>internet</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>technology</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>1 and 3</p>
                                    </list-item>
                                    <list-item>
                                        <p>1 and 4</p>
                                    </list-item>
                                    <list-item>
                                        <p>2 and 3</p>
                                    </list-item>
                                    <list-item>
                                        <p>2 and 4</p>
                                    </list-item>
                                </list>The 4 terms were searched separately by entering the search string, exploding the subject, and selecting all subheadings. The search results were combined using the AND condition. The search history is described below: diabetes chronic disease internet technology 1 and 3 1 and 4 2 and 3 2 and 4</td>
                        </tr>
                        <tr>
                            <td valign="top">Cochrane and Pre-Medline</td>
                            <td valign="top">
                                <italic>AND</italic>
                                <list list-type="order">
                                    <list-item>
                                        <p>
                                            <italic>diabetes</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>chronic disease</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>internet</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>technology</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>1 and 3</p>
                                    </list-item>
                                    <list-item>
                                        <p>1 and 4</p>
                                    </list-item>
                                    <list-item>
                                        <p>2 and 3</p>
                                    </list-item>
                                    <list-item>
                                        <p>2 and 4</p>
                                    </list-item>
                                </list>The 4 terms were searched separately. The search results were combined using the AND condition. The search history is described below: diabetes chronic disease internet technology 1 and 3 1 and 4 2 and 3 2 and 4</td>
                        </tr>
                        <tr>
                            <td valign="top">EMBASE (via ScienceDirect), Expanded Academic ASAP, PubMed</td>
                            <td valign="top">
                                <italic>AND All Fields All Years</italic>
                                <list list-type="bullet">
                                    <list-item>
                                        <p>
                                            <italic>diabetes</italic> AND <italic>internet</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>diabetes</italic> AND <italic>technology</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>chronic disease</italic> AND <italic>internet</italic>
                                        </p>
                                    </list-item>
                                    <list-item>
                                        <p>
                                            <italic>chronic disease</italic> AND <italic>technology</italic>
                                        </p>
                                    </list-item>
                                </list>The terms were searched in combination using the AND condition. The terms were searched in All Fields and for All Years indexed. diabetes AND internet diabetes AND technology chronic disease AND internet chronic disease AND technology</td>
                        </tr>
                    </tbody>
                </table>
            </table-wrap>
            <fig id="figure1" position="float">
                <label>Figure 1</label>
                <caption>
                    <p>The myDiabetes entry form for the daily glucose measurements</p>
                </caption>
                <p>
                    <graphic xlink:href="jmir_v4i1e1_fig1.jpg" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
                </p>
            </fig>
            <fig id="figure2" position="float">
                <label>Figure 2</label>
                <caption>
                    <p>The LifeMasters entry form for the daily glucose measurements</p>
                </caption>
                <p>
                    <graphic xlink:href="jmir_v4i1e1_fig2.jpg" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
                </p>
            </fig>
            <table-wrap id="table2" position="float">
                <label>Table 2</label>
                <caption>
                    <p>Evaluation framework</p>
                </caption>
                <table border="1" cellpadding="4" cellspacing="0" rules="groups">
                    <col width="300"/>
                    <col width="600"/>
                    <thead>
                        <tr>
                            <td valign="top">
                                <bold>Evaluation Criteria</bold>
                            </td>
                            <td valign="top">
                                <bold>Description</bold>
                            </td>
                        </tr>
                    </thead>
                    <tbody>
                        <tr>
                            <td valign="top">
                                <bold>Monitoring</bold>
                            </td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">User defined parameters</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Health care worker</td>
                            <td valign="top">Health care professionals can specify the parameters to monitor</td>
                        </tr>
                        <tr>
                            <td valign="top">Patient</td>
                            <td valign="top">Patients can specify the parameters to monitor</td>
                        </tr>
                        <tr>
                            <td valign="top">User defined parameter ranges</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Health care worker</td>
                            <td valign="top">Health care professionals can specify the normal ranges for monitored parameters</td>
                        </tr>
                        <tr>
                            <td valign="top">Patient</td>
                            <td valign="top">normal ranges for monitored parameters</td>
                        </tr>
                        <tr>
                            <td valign="top">Automated data collection</td>
                            <td valign="top">Vital data can be downloaded directly from the measurement device (eg, Glucometer)</td>
                        </tr>
                        <tr>
                            <td valign="top">Alert algorithms to avoid false alarms</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Entry validation</td>
                            <td valign="top">Validation that patient data is not the result of mistyping (eg, Realistic glucose levels)</td>
                        </tr>
                        <tr>
                            <td valign="top">Screening of symptoms</td>
                            <td valign="top">Determine if changes in vital data is associated to symptoms indicative of an emergency</td>
                        </tr>
                        <tr>
                            <td valign="top">Patient involvement in alert</td>
                            <td valign="top">Involving the patient in the decision to notify a health care professional</td>
                        </tr>
                        <tr>
                            <td valign="top">Multidisciplinary approach</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Multiple aspects of disease management monitored</td>
                            <td valign="top">The monitoring is based on a multidisciplinary approach to diabetes</td>
                        </tr>
                        <tr>
                            <td valign="top">Physical</td>
                            <td valign="top">Monitoring of physical parameters (blood glucose, weight, blood pressure, etc.)</td>
                        </tr>
                        <tr>
                            <td valign="top">Social</td>
                            <td valign="top">Monitoring of the social aspects of diabetes (stigma, dieting, etc.)</td>
                        </tr>
                        <tr>
                            <td valign="top">Psychological</td>
                            <td valign="top">Monitoring of psychological aspects of diabetes (depression, loss of motivation, etc.)</td>
                        </tr>
                        <tr>
                            <td valign="top">Patient access to multiple specialists</td>
                            <td valign="top">Allowing for communication to multiple experts (dietitians, endocrinologists, etc.)</td>
                        </tr>
                        <tr>
                            <td valign="top">Proactive outreach</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Notification to patients</td>
                            <td valign="top">medications, health care appointments, etc</td>
                        </tr>
                        <tr>
                            <td valign="top">Notifications to health care professionals</td>
                            <td valign="top">are reminded of screening test and visits</td>
                        </tr>
                        <tr>
                            <td valign="top">Feedback</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Retrieve and review medical information</td>
                            <td valign="top">Patients can retrieve their medical data to monitor their progress (tabular or graphical format)</td>
                        </tr>
                        <tr>
                            <td valign="top">Regular Feedback</td>
                            <td valign="top">control of diabetes is administered and stored</td>
                        </tr>
                        <tr>
                            <td valign="top"/>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">
                                <bold>Information</bold>
                            </td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Quality of information</td>
                            <td valign="top">site should conform to an accepted level of standards</td>
                        </tr>
                        <tr>
                            <td valign="top">Pull</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Navigation</td>
                            <td valign="top">Navigation should be based on a logical categorization of data</td>
                        </tr>
                        <tr>
                            <td valign="top">Search</td>
                            <td valign="top">Search Functionality availability</td>
                        </tr>
                        <tr>
                            <td valign="top">Push</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Notifications</td>
                            <td valign="top">The system should notify its users of newly available information of interest based on their profile (eg. New research)</td>
                        </tr>
                        <tr>
                            <td valign="top">Newsletter subscription</td>
                            <td valign="top">Users can subscribe to a specific newsletter that is delivered via e-mail of Web browser</td>
                        </tr>
                        <tr>
                            <td valign="top"/>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">
                                <bold>Personalization</bold>
                            </td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Assessment and feedback</td>
                            <td valign="top">diabetes should be assessed using standard evaluation tools</td>
                        </tr>
                        <tr>
                            <td valign="top">Collaborative goal setting</td>
                            <td valign="top">management should be clearly specified</td>
                        </tr>
                        <tr>
                            <td valign="top">Identification of barriers and supports</td>
                            <td valign="top">Using questionnaires to determine each patient's barrier and the appropriate support measures</td>
                        </tr>
                        <tr>
                            <td valign="top">Follow-up support</td>
                            <td valign="top">Re-iteration of support measures</td>
                        </tr>
                        <tr>
                            <td valign="top">Construction of personalized management plan</td>
                            <td valign="top">Tailored management plan as a central feature of the site (can be represented as schedules)</td>
                        </tr>
                        <tr>
                            <td valign="top">Modification of management plan</td>
                            <td valign="top">The ability for users to modify their plans</td>
                        </tr>
                        <tr>
                            <td valign="top">Language and ethnicity</td>
                            <td valign="top">Multilanguage delivery and culture conscious content</td>
                        </tr>
                        <tr>
                            <td valign="top"/>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">
                                <bold>Communication</bold>
                            </td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Health professional- patient</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Synchronous</td>
                            <td valign="top">A channel for one-to-one synchronous communication (eg, videoconferencing)</td>
                        </tr>
                        <tr>
                            <td valign="top">Asynchronous</td>
                            <td valign="top">A channel for one-to-one asynchronous communication (eg, secure email)</td>
                        </tr>
                        <tr>
                            <td valign="top">Indirect</td>
                            <td valign="top">Technical representation of the health care professional</td>
                        </tr>
                        <tr>
                            <td valign="top">Community creation</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Chat rooms</td>
                            <td valign="top">Synchronous many-to-many communication channels</td>
                        </tr>
                        <tr>
                            <td valign="top">Newsgroups / Forums</td>
                            <td valign="top">Asynchronous many-to-many communication channels</td>
                        </tr>
                        <tr>
                            <td valign="top">Expert moderation</td>
                            <td valign="top">Communication channels are based on the dialogue with an expert</td>
                        </tr>
                        <tr>
                            <td valign="top"/>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">
                                <bold>Technology</bold>
                            </td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Security</td>
                            <td valign="top"/>
                        </tr>
                        <tr>
                            <td valign="top">Authentication</td>
                            <td valign="top">Identification of users (usually username/password)</td>
                        </tr>
                        <tr>
                            <td valign="top">Encryption</td>
                            <td valign="top">Data transmission security level (eg, 128-bit)</td>
                        </tr>
                        <tr>
                            <td valign="top">Usability and user-acceptance</td>
                            <td valign="top">Evaluation of usability and user-acceptance (achieved with questionnaires, usage monitoring etc.)</td>
                        </tr>
                        <tr>
                            <td valign="top">Reliability and availability</td>
                            <td valign="top">Service should be available at all times</td>
                        </tr>
                        <tr>
                            <td valign="top">Open architecture</td>
                            <td valign="top">on open standard technologies</td>
                        </tr>
                    </tbody>
                </table>
            </table-wrap>
            <sec>
                <title>Statistical Analysis</title>
                <p>Cohen's multi-rater kappa [<xref ref-type="bibr" rid="ref13">13</xref>,<xref ref-type="bibr" rid="ref14">14</xref>] was used to evaluate the agreement between raters for the evaluation framework as a whole. The multi-rater kappa was calculated with SPSS statistical software using the mkappasc procedure.</p>
            </sec>
        </sec>
        <sec sec-type="results">
            <title>Results</title>
            <sec>
                <title>Evaluation Criteria</title>
                <p>In this section, we describe in detail the evaluation criteria presented in <xref ref-type="table" rid="table2">Table 2</xref>.</p>
                <sec>
                    <title>Monitoring</title>
                    <p>Successful patient monitoring is reliant on efficiently extracting the relevant information from a patient without excessive intrusiveness to both patient and health care professional. Several parameters can be monitored; some examples are blood glucose, weight, blood pressure, diet, foot care, smoking, and nutrition [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Health care professionals should be able to designate which parameters they want to monitor and specify the ranges for each patient. The health care professional should be able to indicate which course of action the system should take if the readings are outside the ranges (eg, notification, triage).</p>
                    <p>Patients should also be able to designate parameters in an effort to improve self-management and goal setting (addressed in the "Personalization" section of "Evaluation Criteria") [<xref ref-type="bibr" rid="ref17">17</xref>]; these, however, should be in addition to - and clearly differentiated from - the parameters specified by the health care professional. Patient-designated parameters should not be shared with the health care professional unless the patient desires that they be shared.</p>
                    <p>The degree of intrusiveness is a fundamental consideration when designing a diabetes management system. A major problem with many disease-management programs using information technology is that they try to collect too much data too often [<xref ref-type="bibr" rid="ref1">1</xref>]. The desire to collect as much data as possible must be balanced with the disruption it may cause in a patient's life [<xref ref-type="bibr" rid="ref4">4</xref>]. Successful strategies to reduce intrusiveness are based on automatic data gathering such as Glucometers that transmit glucose readings via the Internet and the use of simplified questionnaires for triage and screening. Intrusiveness to the health care provider is also an important consideration. If systems were designed to send alerts each time a patient's blood sugar readings are outside the normal parameters, the result would be many false alarms. Therefore, systems must have processes in place designed to not overwhelm health care professionals. These processes include entry validation, screening with the use of questionnaires, and patient involvement in the decision to launch an alert [<xref ref-type="bibr" rid="ref1">1</xref>].</p>
                    <p>Effective patient monitoring is not limited to the collection of health data, it also requires a multidisciplinary approach, proactive outreach, and feedback.</p>
                </sec>
                <sec>
                    <title>Multidisciplinary Approach</title>
                    <p>The management of diabetes spans multiple medical specialties as evidenced by the use of multidisciplinary diabetes management teams. For example, an endocrinologist will manage medications and glucose levels, a dietitian will design an appropriate diet, and a psychologist will manage the mental aspect of dealing with diabetes. Internet-based diabetes management programs should be based on a multidisciplinary teamwork. This element consistently appears in successful chronic-disease management systems [<xref ref-type="bibr" rid="ref18">18</xref>]. Patients should have the ability to interact with multiple specialists to manage each facet of their disease and the Internet can provide a communication channel to enhance this interaction. Successful evaluation tools have been created to effectively measure diabetes management outcomes along multiple dimensions (medical, social, psychological, etc.). Some examples of these tools are the Diabetes Quality of Life Measure (DQOL) developed for use in the Diabetes Control and Complications Trial (DCCT) [<xref ref-type="bibr" rid="ref19">19</xref>] and the SF-36 [<xref ref-type="bibr" rid="ref20">20</xref>].</p>
                </sec>
                <sec>
                    <title>Proactive Outreach</title>
                    <p>Proactive outreach and patient tracking are critical success factors for an Internet-based diabetes management system. Proactive outreach consists of notifications sent to patients to take their medication, visit the health care professional, or simply exercise once a day. The benefit of a proactive approach is well documented in the management of other chronic diseases such as chronic heart failure, where increased compliance and monitoring have resulted in a decrease in the number of hospitalizations for cardiovascular diagnoses and hospital days were reduced from 0.6 to 0.2 (P = .09) per patient per year [<xref ref-type="bibr" rid="ref21">21</xref>]. Proactive outreach also applies to health care professionals. Reminders to physicians of routine testing for patients can be implemented in an Internet-based diabetes management system. A study determined that the use of a diabetes management system increases the likelihood of physicians ordering lipid-profile testing (19%) and retinal exams for their patients [<xref ref-type="bibr" rid="ref22">22</xref>].</p>
                </sec>
                <sec>
                    <title>Feedback</title>
                    <p>The role of the patient has become central in the management of chronic disease; therefore, monitoring must integrate the patient [<xref ref-type="bibr" rid="ref22">22</xref>]. A crucial aspect of patient integration is feedback. Patients must have the ability to review their medical data at anytime. On-line graphical tools can allow patients to visualize their medical information in much the same way a physician would. Feedback also provides a valuable motivational tool that improves compliance [<xref ref-type="bibr" rid="ref1">1</xref>] and system usage, both of which are linked to an improved outcome in diabetes management [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
                </sec>
                <sec>
                    <title>Information</title>
                    <p>The Internet has always served as a source of health information; 70 million of the 110 million American Internet users have searched the Web for health information in the past year. Currently they can choose from 20,000 health care sites with 1,500 more coming on-line each month [<xref ref-type="bibr" rid="ref24">24</xref>]. A successful Internet-based diabetes management system should be a source of quality information for the patients who use it. The quality of information on the Internet is a source of great debate. The low barriers to publication on the Internet result in the presence of vast amounts of low-quality and inaccurate information. This misinformation or information that is out of date has the potential of misleading and even harming patients. Consequently, independent agencies such as the Health on the Net Foundation [<xref ref-type="bibr" rid="ref25">25</xref>] were created to certify the content of medical information on the Internet. Information delivery is based on 2 models: pull and push.</p>
                </sec>
                <sec>
                    <title>Pull Model</title>
                    <p>The pull model relies on the patient retrieving the information he or she seeks. Two pathways are provided to this end. The patient can retrieve documents by navigating through the Web site or can retrieve information with a search engine.</p>
                    <p>Navigation requires a clearly-defined information structure. This is effectively implemented with a hierarchical structure that users can follow to retrieve information of increasing level of detail. Navigation should be facilitated by a clear on-screen indication of the user's location in the information hierarchy.</p>
                    <p>Search engines allow users to search for documents based on keywords. Search engine technology is capable of cataloguing documents based on several criteria. In its simplest form, documents will be catalogued based on their text. Therefore, a search will yield all the documents containing the word that was searched for. However, a successful implementation of a search engine will categorize documents based on several criteria such as topic, author, date, and relevance. Users can then use these criteria to refine their searches.</p>
                </sec>
                <sec>
                    <title>Push Model</title>
                    <p>The push model involves presenting the information to the patient who has opted to receive it. Relevant information could include new research or newly-released drugs for patients who have specified an interest. Interest can be formally expressed by the patient or can be inferred by the system in an effort to personalize the service (see the "Personalization" section of "Evaluation Criteria").</p>
                    <p>Information delivery in the push model can be implemented in several ways. Patients can be presented with the relevant information upon logging into the system. Alternatively, technologies such as mobile phones and pagers can be used for delivery. A successful Internet-based management system will implement both models of information delivery.</p>
                </sec>
            </sec>
            <sec>
                <title>Personalization</title>
                <sec>
                    <title>Self-management Plan</title>
                    <p>The management of any chronic disease must be personalized to the individuals, as they are ultimately responsible for its success. Consequently, an Internet-based diabetes management system must allow patients to tailor the intervention to their specific needs. Patients benefit from a proactive approach to their management (in which they are not told what to do) and gain a valuable insight into the management options that may be available to them [<xref ref-type="bibr" rid="ref17">17</xref>]. Patient involvement and contribution to disease management has demonstrated improved results and compliance [<xref ref-type="bibr" rid="ref26">26</xref>].</p>
                    <p>The comprehensive management of diabetes can be based on several models. It is not the purpose of this paper to discuss these management models but rather their successful implementation as Internet-based diabetes management systems. One such model [<xref ref-type="bibr" rid="ref17">17</xref>] (multilevel social-ecological model for self-management and support for behavior change) was implemented as a physical-activity intervention study [<xref ref-type="bibr" rid="ref17">17</xref>]. This model is based on the creation of a <italic>personal action plan</italic> that is the result of both the patient's and health professional's requirements [<xref ref-type="bibr" rid="ref27">27</xref>]. The creation of a personal action plan can be expressed as these self-management action steps: assessment and feedback, collaborative goal setting, identification of barriers and supports, individualized problem solving, follow-up support, and construction of a personal action plan. Glasgow and Bull have identified the strengths and limitations of interactive technologies such as the Internet for Self-Management Action Steps [<xref ref-type="bibr" rid="ref17">17</xref>]. Nonetheless, a successful implementation of an Internet-based diabetes management system should provide the patient with the ability to navigate through each action step towards the creation of a personal action plan or the equivalent (depending on the disease-management model used).</p>
                </sec>
                <sec>
                    <title>Language and Ethnicity</title>
                    <p>Piette et al [<xref ref-type="bibr" rid="ref28">28</xref>] demonstrated that an Automated Telephone Disease Management (ATDM) system produced positive results with an ethnically-diverse diabetic-patient population. Internet-based diabetes systems can reach different ethnicities by offering their services in multiple languages. In some groups where language may be a barrier to medical care, such systems may provide substantial benefits.</p>
                    <p>Inevitably, this opens the discussion of Internet demographics splitting patients between <italic>haves</italic> and <italic>have-nots</italic>. This is particularly relevant for type II Diabetes where some minority groups are disproportionately affected and have limited access to the Internet. However, the report from the National Telecommunications and Information Administration indicates a rapid change in Internet demographics that is reflective of the general population of the United States [<xref ref-type="bibr" rid="ref29">29</xref>].</p>
                </sec>
            </sec>
            <sec>
                <title>Communication</title>
                <sec>
                    <title>Communication Between Health Professional and Patient</title>
                    <p>Most efforts in health care technology focus on assisting the doctor in diagnosing and treating a disease. This approach tends to omit a key component of the health care cycle: the patient. The new trend in medicine favors the inclusion of the patient as an integral part of the healing process. A review of 22 studies by Stewart et al [<xref ref-type="bibr" rid="ref30">30</xref>] indicated a positive effect of communication on actual patient health outcome such as pain, recovery from symptom, anxiety, functional status, and physiologic measures of blood pressure and blood glucose.</p>
                    <p>An Internet-based diabetes management system must be a channel of communication between patients and their health care providers. The communication system can follow 3 models: synchronous, asynchronous, and indirect. Synchronous communication allows the patient and health care provider to communicate directly by using teleconferencing or videoconferencing. Traditionally, these services were in the realm of telemedicine [<xref ref-type="bibr" rid="ref31">31</xref>] where specific technical equipment was installed to allow the communication to happen. However, the advent of multimedia on the Internet does allow for real-time voice-based and image-based communication. Although at its first steps, synchronous communication can be a valuable part of an Internet-based diabetes management system. Equally, the asynchronous communication model is a crucial part of a management system. Simple solutions such as secure text communication between patient and health care provider can be of great benefit in the management of diabetes. A study at the University of Pittsburgh describes a model of asynchronous communication between doctors and patients that reduced some of the differences in communication in terms of expectations, vocabulary used, and other factors [<xref ref-type="bibr" rid="ref32">32</xref>]. This study was based on a communication system that allowed patients to familiarize themselves with the relevant domain terms at their own pace. The system also allowed physicians to request more information of patients while providing contextual information. This allowed patients to understand the underlying reasons for the questions.</p>
                    <p>Lastly, the indirect communication model is based on the concept of representation of the health care professional by technology. Such solutions have been implemented using software agents, a form of artificial intelligence that interacts with its environment and reacts to changes. In this case, the agent can interact with the patient and carry out a basic dialogue - and functions as information search and triage [<xref ref-type="bibr" rid="ref33">33</xref>]. While still experimental, the use of indirect communication in Internet-based diabetes care shows great potential.</p>
                </sec>
                <sec>
                    <title>Community Creation</title>
                    <p>Community creation is based on a many-to-many communication channel compared with the one-to-one communication that occurs between health care professional and patient. Community support is a fundamental aspect of self-management of disease. Diabetes patients benefit from discussing topics that concern management of the disease, anxiety as to what the future holds, and interpersonal and social relationships.</p>
                    <p>The Internet can enable the creation of communities based on the same models of synchronous and asynchronous communication models. One study followed a diabetes chat room for 21 months and found that 79% of all respondents rated participation in the chat as having a positive effect on coping with diabetes [<xref ref-type="bibr" rid="ref34">34</xref>]. Another study established a chat room for adolescents affected by diabetes and moderated by a diabetologist [<xref ref-type="bibr" rid="ref35">35</xref>]. The results indicated a decrease in HbA <sub>1c</sub>and an improved capacity for self-management. Anonymity undoubtedly favors a greater freedom of expression of individual problems. Community creation and maintenance should be an integral part of any Internet-based management systems. The implementation can be as synchronous chat rooms or as newsgroups where users communicate asynchronously by posting their comments. Further, experts can moderate chat rooms.</p>
                </sec>
                <sec>
                    <title>Technology</title>
                    <p>The complex network of human and machine relations involved in managing diabetes via an Internet-based system has strong implications for the design of such a service.</p>
                </sec>
                <sec>
                    <title>Security</title>
                    <p>One of the main concerns with any medical informatics solution is security and privacy of the data. The success of any Internet-based diabetes management system is reliant on the user's trust that the user's data is secure, private, and confidential. This is possible with the recent availability of strong cryptographic tools used for 2 main purposes: authentication and encryption [<xref ref-type="bibr" rid="ref23">23</xref>].</p>
                    <sec>
                        <title>Authentication</title>
                        <p>Identification of users is a crucial step in gaining access to the system. Users are granted access to data based on their security profile. For example, only the treating physician can modify a specific patient's blood glucose ranges. Therefore, authentication is both the identification of a user (usually with a combination of username and password) and the enforcement of the security profile. Naturally, user identification is required for more-advanced functions like personalization as mentioned earlier.</p>
                    </sec>
                    <sec>
                        <title>Encryption</title>
                        <p>All data transmitted between a patient and the system must be secure. Several encryption algorithms exist, with different strengths and speeds. Generally speaking, most Web servers can establish secure communication links using Netscape's Secure Socket Layer (SSL), which is de facto the Internet standard. Recently, 128-bit encryption has been made available worldwide. Any transmission of patient data should be encrypted at the highest level.</p>
                    </sec>
                </sec>
                <sec>
                    <title>Usability and User Acceptance</title>
                    <p>Testing usability and user acceptance is a critical part of any computerized system and should be a continuous process during the life of the system. Typically, evaluation instruments have consisted of on-line questionnaires, on-line commenting (e-mail), telephone interviews, video-based testing, and tracking of system usage [<xref ref-type="bibr" rid="ref36">36</xref>].</p>
                    <p>Many physicians believe that the key success factor in managing diabetes is simplicity [<xref ref-type="bibr" rid="ref1">1</xref>]. Consequently, the implementation of an Internet-based diabetes management system should strive towards simplicity for both patient and health care professional. Internet technologies can be a great supplement but if the implementation is not user-friendly, it can become a real barrier [<xref ref-type="bibr" rid="ref1">1</xref>]. Although the technology has enormous potential, developers should not lose sight of the real purpose of these systems: to collect small amounts of data rapidly and efficiently. Therefore, an Internet-based diabetes management system will only be successful if implemented with a simple user interface used to collect the minimum amount of data from the patient (thus reducing its intrusiveness).</p>
                </sec>
                <sec>
                    <title>Reliability and Availability</title>
                    <p>One of the great advantages of the Internet is that it allows users to access systems anytime and from almost anywhere. This results in a need for systems to always be operational, that is, without downtime. Zero downtime (or close to it) requires fault-tolerant systems. Several technical solutions exist both at the software and hardware level. It is outside the scope of this paper to examine all the solutions; however, it is reasonable to expect an Internet-based diabetes management system to not require downtime for maintenance and to have a fault-tolerant hosting environment.</p>
                </sec>
                <sec>
                    <title>Open Platform</title>
                    <p>Open technologies are based on nonproprietary standards; therefore, a system can be built using technologies from multiple vendors. This is particularly useful for future expansions or medications to accommodate for increased scalability and functionality requirements. An Internet-based diabetes management system should be based on an open platform, particularly for data exchange. Open standards for data representation such as the eXtensible Markup Language (XML) are being adopted by multiple industries. Consequently, a system built using XML will be able to interface with multiple systems and devices. The same system could deliver its services via multiple devices (Internet, mobile phone, handheld computer, etc.) effectively making the Internet open platform the standard.</p>
                </sec>
            </sec>
            <sec>
                <title>Evaluation of 2 Existing Services</title>
                <p>To illustrate the use of the evaluation framework, we have applied it to 2 existing Internet-based diabetes management systems: my Diabetes (<ext-link ext-link-type="uri" xlink:href="http://www.myDiabetes.com" xlink:type="simple">www.myDiabetes.com</ext-link>) and LifeMasters (<ext-link ext-link-type="uri" xlink:href="http://www.lifemasters.com" xlink:type="simple">www.lifemasters.com</ext-link>).</p>
                <p>To produce an overall evaluation, a criterion was considered satisfactory if the majority of the raters evaluated it positively (<italic>Yes</italic> rating). The results of the evaluations were numerically converted by assigning a value of 1 to all positive (<italic>Yes</italic>) ratings and a value of 0 to all negative (<italic>No</italic>) ratings. The results of all the evaluations are compiled in <xref ref-type="table" rid="table3">Table 3</xref>. The agreement level is reported for each individual criterion. This was calculated by dividing the number of ratings consistent with the overall rating (the majority) by the number of raters. For example, if a criterion was rated satisfactory or unsatisfactory by 4 out of the 5 raters, the criterion has an agreement level of 80% (4/5).</p>
                <p>The technology criteria registered the lowest agreement (60%-80%). The different levels of technical expertise of the evaluators may explain this difference. The Personalization criteria also showed lower levels of agreement between evaluators. This is due to the different interpretations of the criteria between evaluators. Personalization remains a difficult dimension to quantify and evaluate. The quality-of-information agreement levels were also low (60%-80%). Both sites displayed the HON code logo and stated that they subscribed to the HONCode principles. However, neither site was HON registered, although - as of December 14, 2001 - LifeMasters was under review process.</p>
                <p>The multi-rater kappa for myDiabetes was 0.75 and for LifeMasters was 0.65, indicating a substantial level of agreement as defined by Landis and Koch [<xref ref-type="bibr" rid="ref37">37</xref>]. There was an important difference between the kappa of MyDiabetes and the kappa of LifeMasters. Further testing is required to clarify the reasons for the difference.</p>
                <table-wrap id="table3" position="float">
                    <label>Table 3</label>
                    <caption>
                        <p>Evaluation Examples</p>
                    </caption>
                    <table border="1" cellpadding="4" cellspacing="0" rules="groups">
                        <thead>
                            <tr>
                                <th valign="top">Evaluation Criteria</th>
                                <th valign="top">myDiabetes.com (Agreement Level)</th>
                                <th valign="top">LifeMasters.com (Agreement Level)</th>
                            </tr>
                        </thead>
                        <tbody>
                            <tr>
                                <td valign="top">
                                    <bold>Monitoring</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>User defined parameters</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Health care worker</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Patient</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>User defined parameter ranges</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Health care worker</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Patient</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Automated data collection</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Alert algorithms to avoid false alarms</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Entry validation</td>
                                <td valign="top">Yes (80%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Screening of symptoms</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Patient involvement in alert</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Multidisciplinary approach</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Multiple aspects of disease management monitored</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Physical</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Social</td>
                                <td valign="top">Yes (100%). Uses DQOL<xref ref-type="table-fn" rid="table3fn1">*</xref></td>
                                <td valign="top">Yes (80%). Uses SF-36</td>
                            </tr>
                            <tr>
                                <td valign="top">Psychological</td>
                                <td valign="top">Yes (100%). Uses DQOL</td>
                                <td valign="top">Yes (80%). Uses SF-36</td>
                            </tr>
                            <tr>
                                <td valign="top">Patient access to multiple specialists</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Proactive outreach</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Notification to patients</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Notifications to health care professionals</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Feedback</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Retrieve and review medical information</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Regular feedback</td>
                                <td valign="top">Yes (80%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Information</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Quality of information</td>
                                <td valign="top">Yes (80%). Uses HON</td>
                                <td valign="top">Yes (60%). Uses HON</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Pull</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Navigation</td>
                                <td valign="top">Yes (80%). Categorized</td>
                                <td valign="top">Yes (80%). Categorized</td>
                            </tr>
                            <tr>
                                <td valign="top">Search</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Push</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Notifications</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Newsletter subscription</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Personalization</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Assessment and feedback</td>
                                <td valign="top">Yes (80%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Collaborative goal setting</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Identification of barriers and supports</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Follow-up support</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Construction of personalized management plan</td>
                                <td valign="top">Yes (80%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Modification of management plan</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Web site personalization</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Language and ethnicity</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Communication</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Health professional - patient</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Synchronous</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Asynchronous</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Indirect</td>
                                <td valign="top">No (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Community creation</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Chat rooms</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">No (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Newsgroups / Forums</td>
                                <td valign="top">Yes (100%)</td>
                                <td valign="top">Yes (100%)</td>
                            </tr>
                            <tr>
                                <td valign="top">Expert moderation</td>
                                <td valign="top">Yes (80%)</td>
                                <td valign="top">Yes (80%)</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Technology</bold>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <italic>Security</italic>
                                </td>
                                <td valign="top"/>
                                <td valign="top"/>
                            </tr>
                            <tr>
                                <td valign="top">Authentication</td>
                                <td valign="top">Yes (100%). User and Password</td>
                                <td valign="top">Yes (100%). User and Password</td>
                            </tr>
                            <tr>
                                <td valign="top">Encryption</td>
                                <td valign="top">Yes (100%). 128-bit</td>
                                <td valign="top">Yes (100%). 128-bit</td>
                            </tr>
                            <tr>
                                <td valign="top">Usability and user acceptance</td>
                                <td valign="top">Yes (60%). Tested with forums</td>
                                <td valign="top">No (80%). Not actively tested</td>
                            </tr>
                            <tr>
                                <td valign="top">Reliability and availability</td>
                                <td valign="top">Netscape compatible</td>
                                <td valign="top">Netscape compatible</td>
                            </tr>
                            <tr>
                                <td valign="top">Open architecture</td>
                                <td valign="top">No (60%). IIS and ASP</td>
                                <td valign="top">No (60%). IIS and ASP</td>
                            </tr>
                            <tr>
                                <td valign="top">
                                    <bold>Total Positive Results</bold>
                                </td>
                                <td valign="top">25 out of 40</td>
                                <td valign="top">32 out of 40</td>
                            </tr>
                        </tbody>
                    </table>
                    <table-wrap-foot>
                        <fn id="table3fn1"><p><sup>*</sup> DQOL = Diabetes Quality of Life Measure</p></fn>
                    </table-wrap-foot>
                </table-wrap>
                <fig id="figure3" position="float">
                    <label>Figure 3</label>
                    <caption>
                        <p>Evaluation of myDiabetes.com and LifeMasters.com. The value of each axis is normalized by conversion to a percentage of the maximum score</p>
                    </caption>
                    <graphic xlink:href="jmir_v4i1e1_fig3.jpg" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
                </fig>
                <sec>
                    <title>Graphical Representation</title>
                    <p>We believe that a graphical representation of the evaluation results is particularly useful for comparing 2 systems and for determining in which direction the systems should expand their services. To this purpose, a radar graph with the 5 axes representing the 5 dimensions of Monitoring, Information, Personalization, Communication, and Technology is a useful representation. The value of each axis is normalized by conversion to a percentage of the maximum score. The evaluation of myDiabetes.com and LifeMasters.com is represented in <xref ref-type="fig" rid="figure3">Figure 3</xref>.</p>
                    <p>The results of the evaluation indicate that LifeMasters is a more-complete solution than myDiabetes in all dimensions - except Information, where both sites were equivalent. This is primarily due to LifeMaster's inclusion of the health care professional in the disease-management cycle. On the other hand, myDiabetes is uniquely interfaced with the patient and is quite good in providing a communication channel for community creation, however, communication with health care professional is lacking, hence the lower score than LifeMasters.</p>
                </sec>
            </sec>
        </sec>
        <sec sec-type="discussion">
            <title>Discussion</title>
            <p>The Internet will undoubtedly change the way we deliver health care services. Chronic disease management, which accounts for 60% of the U.S. medical care costs [<xref ref-type="bibr" rid="ref38">38</xref>], is a desirable target for the efficiencies of the Internet. Chronic-disease management on the Internet is estimated to have a market potential of US $700 billion [<xref ref-type="bibr" rid="ref24">24</xref>]. Already we are seeing several Internet-based chronic-disease-management sites arising; however, there is little evidence as to how these solutions are answering the needs of the consumer (the patient).</p>
            <p>Consumer health informatics research greatly contributes to the health care sector by attempting to systematize and codify consumer's needs, values, and preferences and by trying to build and evaluate information systems that interact directly with consumers and patients [<xref ref-type="bibr" rid="ref39">39</xref>]. In this paper, we have attempted to catalogue the critical success factors for an Internet-based diabetes management system based on the available literature and the authors' experience. The result is a first step towards a comprehensive evaluation framework. The framework is based on the recognition that the management of diabetes via the Internet is based on several integrated dimensions, namely, Monitoring, Information, Personalization, Communication, and Technology. A successful diabetes management system should efficiently integrate all dimensions. Therefore, the framework provides a model for evaluation and, more importantly, for strategic growth planning for existing sites. For example, a site that is deficient in the communication dimension may enhance its offerings by adding a synchronous chat room.</p>
            <p>This paper reports an initial evaluation of 2 sites. The results indicate a high-level inter-rater agreement as measured by Cohen's multi-rater kappa. However, this is based on a small sample of evaluations (5). Future research should focus on validation of the framework by consistency between larger samples of raters and on correlation with the success of the multiple sites available today. Key metrics for success include the number of enrolled patients; length of time managed; clinical, economic, and quality-of-life outcomes; and patient-satisfaction measures [<xref ref-type="bibr" rid="ref24">24</xref>].</p>
        </sec>
    </body>
    <back>
        <fn-group>
            <fn fn-type="conflict">
                
                <p>None declared.</p>
            </fn>
        </fn-group>
        <glossary>
                <title>Abbreviations</title>
                <def-list>
                    <def-item>
                        <term id="abbr1">DQOL</term>
                        <def>
                            <p>Diabetes Quality of Life Measure</p>
                        </def>
                    </def-item>
                    <def-item>
                        <term id="abbr3">XML</term>
                        <def>
                            <p>eXtensible Markup Language</p>
                        </def>
                    </def-item>
                </def-list>
            </glossary>
        <ref-list>
            <ref id="ref1">
                <label>1</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Gillespie</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>
                    <article-title>Deploying an I.T. cure for chronic diseases</article-title>
                    <source>Health Data Manag</source>
                    <year>2000</year>
                    <month>7</month>
                    <volume>8</volume>
                    <issue>7</issue>
                    <fpage>68</fpage>
                    <lpage>70, 72, 74</lpage>
                    <pub-id pub-id-type="medline">20478803</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref2">
                <label>2</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cai</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Johnson</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Hripcsak</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>
                    <article-title>Generic data modeling for home telemonitoring of chronically ill patients</article-title>
                    <source>Proc AMIA Symp</source>
                    <year>2000</year>
                    <fpage>116</fpage>
                    <lpage>20</lpage>
                    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.amia.org/pubs/symposia/D200744.PDF" ext-link-type="uri">http://www.amia.org/pubs/symposia/D200744.PDF</ext-link></comment>
                    <pub-id pub-id-type="medline">21027330</pub-id>
                    <pub-id pub-id-type="pii">D200744</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref3">
                <label>3</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Darves</surname>
                            <given-names>B</given-names>
                        </name>
                    </person-group>
                    <source>Web-enabled disease management program saves big bucks. Apr 17, 200</source>
                    <access-date>2001 Oct 25</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.lifemasters.com/corporate/abou/newsWebMdApril2001.asp" xlink:type="simple">http://www.lifemasters.com/corporate/abou/newsWebMdApril2001.asp</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref4">
                <label>4</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Hagland</surname>
                            <given-names>M</given-names>
                        </name>
                    </person-group>
                    <article-title>Managing diseases by managing IT</article-title>
                    <source>Health Manag Technol</source>
                    <year>1998</year>
                    <month>6</month>
                    <volume>19</volume>
                    <issue>7</issue>
                    <fpage>18</fpage>
                    <lpage>20, 22, 24-5</lpage>
                    <pub-id pub-id-type="medline">98335478</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref5">
                <label>5</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="editor">
                        <name name-style="western">
                            <surname>Kumar</surname>
                            <given-names>Parveen</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Clark</surname>
                            <given-names>Michael</given-names>
                        </name>
                    </person-group>
                    <source>Clinical Medicine</source>
                    <year>1998</year>
                    <month>12</month>
                    <day>1</day>
                    <publisher-loc>London</publisher-loc>
                    <publisher-name>W B Saunders Co</publisher-name>
                    <fpage>978</fpage>
                    <lpage>998</lpage>
                </nlm-citation>
            </ref>
            <ref id="ref6">
                <label>6</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Cotran</surname>
                            <given-names>R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kumar</surname>
                            <given-names>V</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Collins</surname>
                            <given-names>T</given-names>
                        </name>
                    </person-group>
                    <article-title>Robbins Pathologic Basis of Disease</article-title>
                    <year>1998</year>
                    <edition>6</edition>
                    <publisher-loc>Phildelphia</publisher-loc>
                    <publisher-name>W. B. Saunders</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref7">
                <label>7</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <collab xlink:type="simple">American Diabetes Association</collab>
                    <source>Direct and indirect costs of diabetes</source>
                    <access-date>2001 Aug 30</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.diabetes.org/main/info/facts/impact/default2.jsp" xlink:type="simple">http://www.diabetes.org/main/info/facts/impact/default2.jsp</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref8">
                <label>8</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Tattersall</surname>
                            <given-names>Robert B</given-names>
                        </name>
                    </person-group>
                    <person-group person-group-type="editor">
                        <name name-style="western">
                            <surname>Gale</surname>
                            <given-names>Edwin AM</given-names>
                        </name>
                    </person-group>
                    <source>Diabetes: Clinical Management</source>
                    <year>1990</year>
                    <month>5</month>
                    <day>1</day>
                    <publisher-loc>Edinburgh</publisher-loc>
                    <publisher-name>Churchill Livingstone</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref9">
                <label>9</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Feil</surname>
                            <given-names>E G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Glasgow</surname>
                            <given-names>R E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Boles</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Mckay</surname>
                            <given-names>H G</given-names>
                        </name>
                    </person-group>
                    <article-title>Who participates in Internet-based self-management programs? A study among novice computer users in a primary care setting</article-title>
                    <source>Diabetes Educ</source>
                    <year>2000</year>
                    <month>10</month>
                    <volume>26</volume>
                    <issue>5</issue>
                    <fpage>806</fpage>
                    <lpage>11</lpage>
                    <pub-id pub-id-type="medline">21021526</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref10">
                <label>10</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mckenna</surname>
                            <given-names>MK</given-names>
                        </name>
                    </person-group>
                    <source>E-Health and the practicing physician. Medscape TechMed [serial on-line]</source>
                    <year>2001</year>
                    <access-date>2001 Aug 30</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.medscape.com/medscape/TechMed/journal/2001/v01.n01/mtm0709.02.mcke/mtm0709.02.mcke-01.html" xlink:type="simple">http://www.medscape.com/medscape/TechMed/journal/2001/v01.n01/mtm0709.02.mcke/mtm0709.02.mcke-01.html</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref11">
                <label>11</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Jacobson</surname>
                            <given-names>I</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Griss</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jonsson</surname>
                            <given-names>P</given-names>
                        </name>
                    </person-group>
                    <source>Software Reuse : Architecture, Process and Organization for Business Success</source>
                    <year>1997</year>
                    <month>5</month>
                    <day>22</day>
                    <publisher-loc>Reading, MA</publisher-loc>
                    <publisher-name>Addison-Wesley Professional</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref12">
                <label>12</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Jacobson</surname>
                            <given-names>Ivar</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Booch</surname>
                            <given-names>Grady</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Rumbaugh</surname>
                            <given-names>James</given-names>
                        </name>
                    </person-group>
                    <source>The Unified Software Development Process</source>
                    <year>1999</year>
                    <month>2</month>
                    <day>4</day>
                    <publisher-loc>Philadelphia</publisher-loc>
                    <publisher-name>Addison-Wesley Professional</publisher-name>
                </nlm-citation>
            </ref>
            <ref id="ref13">
                <label>13</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Carletta</surname>
                            <given-names>J</given-names>
                        </name>
                    </person-group>
                    <article-title>Assessing agreement on classification tasks: the Kappa statistic</article-title>
                    <source>Computational Linguistics</source>
                    <year>1996</year>
                    <volume>22</volume>
                    <issue>2</issue>
                    <fpage>249</fpage>
                    <lpage>254</lpage>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.cogsci.ed.ac.uk/~jeanc/squib.pdf" xlink:type="simple">http://www.cogsci.ed.ac.uk/~jeanc/squib.pdf</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref14">
                <label>14</label>
<nlm-citation citation-type="book" xlink:type="simple">
    <person-group person-group-type="author">
        <name name-style="western">
            <surname>Siegel</surname>
            <given-names>Sidney</given-names>
        </name>
        <name name-style="western">
            <surname>Castellan</surname>
            <given-names>N John</given-names>
        </name>
    </person-group>
    <source>Nonparametric Statistics for The Behavioral Sciences</source>
    <year>1988</year>
    <month>1</month>
    <day>1</day>
    <edition>2</edition>
    <publisher-loc>NewYork</publisher-loc>
    <publisher-name>McGraw-Hill Humanities/Social Sciences/Languages</publisher-name>
</nlm-citation>
            </ref>
            <ref id="ref15">
                <label>15</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Piette</surname>
                            <given-names>J D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Weinberger</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kraemer</surname>
                            <given-names>F B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Mcphee</surname>
                            <given-names>S J</given-names>
                        </name>
                    </person-group>
                    <article-title>Impact of automated calls with nurse follow-up on diabetes treatment outcomes in a Department of Veterans Affairs Health Care System: a randomized controlled trial</article-title>
                    <source>Diabetes Care</source>
                    <year>2001</year>
                    <month>2</month>
                    <volume>24</volume>
                    <issue>2</issue>
                    <fpage>202</fpage>
                    <lpage>8</lpage>
                    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11213866" ext-link-type="uri">http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11213866</ext-link></comment>
                    <pub-id pub-id-type="medline">21081568</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref16">
                <label>16</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mckay</surname>
                            <given-names>H G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Feil</surname>
                            <given-names>E G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Glasgow</surname>
                            <given-names>R E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brown</surname>
                            <given-names>J E</given-names>
                        </name>
                    </person-group>
                    <article-title>Feasibility and use of an Internet support service for diabetes self-management</article-title>
                    <source>Diabetes Educ</source>
                    <year>1998</year>
                    <month>4</month>
                    <volume>24</volume>
                    <issue>2</issue>
                    <fpage>174</fpage>
                    <lpage>9</lpage>
                    <pub-id pub-id-type="medline">98216194</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref17">
                <label>17</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mckay</surname>
                            <given-names>H G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>King</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Eakin</surname>
                            <given-names>E G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Seeley</surname>
                            <given-names>J R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Glasgow</surname>
                            <given-names>R E</given-names>
                        </name>
                    </person-group>
                    <article-title>The diabetes network internet-based physical activity intervention: a randomized pilot study</article-title>
                    <source>Diabetes Care</source>
                    <year>2001</year>
                    <month>8</month>
                    <volume>24</volume>
                    <issue>8</issue>
                    <fpage>1328</fpage>
                    <lpage>34</lpage>
                    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11473065" ext-link-type="uri">http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11473065</ext-link></comment>
                    <pub-id pub-id-type="medline">21365601</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref18">
                <label>18</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Boulton</surname>
                            <given-names>A J</given-names>
                        </name>
                    </person-group>
                    <article-title>Why bother educating the multi-disciplinary team and the patient--the example of prevention of lower extremity amputation in diabetes</article-title>
                    <source>Patient Educ Couns</source>
                    <year>1995</year>
                    <month>9</month>
                    <volume>26</volume>
                    <issue>1-3</issue>
                    <fpage>183</fpage>
                    <lpage>8</lpage>
                    <pub-id pub-id-type="medline">96037481</pub-id>
                    <pub-id pub-id-type="doi">10.1016/0738-3991(95)00746-M</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref19">
                <label>19</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <collab xlink:type="simple">The Diabetes Control and Complications Trial Research Group</collab>
                    <article-title>The effect of intensive treatment of diabetes on the development and progression of long-term complications in insulin-dependent diabetes mellitus</article-title>
    <source>N Engl J Med</source>
                    <year>1993</year>
                    <month>9</month>
                    <day>30</day>
                    <volume>329</volume>
                    <issue>14</issue>
                    <fpage>977</fpage>
                    <lpage>986</lpage>
    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://content.nejm.org/cgi/pmidlookup?view=short&amp;pmid=8366922" ext-link-type="uri">http://content.nejm.org/cgi/pmidlookup?view=short&amp;pmid=8366922</ext-link></comment>
                    <pub-id pub-id-type="medline">93376052</pub-id>
                    <pub-id pub-id-type="doi">10.1056/NEJM199309303291401</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref20">
                <label>20</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Ware</surname>
                            <given-names>J E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sherbourne</surname>
                            <given-names>C D</given-names>
                        </name>
                    </person-group>
                    <article-title>The MOS 36-item short-form health survey (SF-36). I. Conceptual framework and item selection</article-title>
                    <source>Med Care</source>
                    <year>1992</year>
                    <month>6</month>
                    <volume>30</volume>
                    <issue>6</issue>
                    <fpage>473</fpage>
                    <lpage>83</lpage>
                    <pub-id pub-id-type="medline">92278120</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref21">
                <label>21</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Shah</surname>
                            <given-names>NB</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Der</surname>
                            <given-names>E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ruggerio</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Heidenreich</surname>
                            <given-names>PA</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Massie</surname>
                            <given-names>BM</given-names>
                        </name>
                    </person-group>
                    <article-title>Prevention of hospitalizations for heart failure with an interactive home monitoring program</article-title>
                    <source>LifeMasters </source>
                    <access-date>2002 Jan 5</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.lifemasters.com/Corporate/clin/carticle.asp#art2" xlink:type="simple">http://www.lifemasters.com/Corporate/clin/carticle.asp#art2</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref22">
                <label>22</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Baker</surname>
                            <given-names>A M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Lafata</surname>
                            <given-names>J E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ward</surname>
                            <given-names>R E</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Whitehouse</surname>
                            <given-names>F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Divine</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>
                    <article-title>A Web-based diabetes care management support system</article-title>
                    <source>Jt Comm J Qual Improv</source>
                    <year>2001</year>
                    <month>4</month>
                    <volume>27</volume>
                    <issue>4</issue>
                    <fpage>179</fpage>
                    <lpage>90</lpage>
                    <pub-id pub-id-type="medline">21190149</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref23">
                <label>23</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Zechman</surname>
                            <given-names>D</given-names>
                        </name>
                    </person-group>
                    <article-title>Realizing your resources. Implementing an enterprise-wide patient monitoring system begins with making the most of what you have</article-title>
                    <source>Healthc Inform</source>
                    <year>2000</year>
                    <month>8</month>
                    <volume>17</volume>
                    <issue>8</issue>
                    <fpage>69</fpage>
                    <lpage>70</lpage>
                    <pub-id pub-id-type="medline">20533887</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref24">
                <label>24</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Bernard</surname>
                            <given-names>S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Wisnicki</surname>
                            <given-names>HJ</given-names>
                        </name>
                    </person-group>
                    <article-title>How health information on the Internet is evolving</article-title>
                    <source>Ophthalmology Times</source>
                    <year>2000</year>
                    <month>12</month>
                    <day>1</day>
                    <volume>25</volume>
                    <issue>23</issue>
                    <fpage>10</fpage>
                    <lpage>14</lpage>
                </nlm-citation>
            </ref>
            <ref id="ref25">
                <label>25</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <collab xlink:type="simple">Health on the Net Foundation</collab>
                    <source>Home page</source>
                    <access-date>2001 Oct 29</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.hon.ch" xlink:type="simple">http://www.hon.ch</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref26">
                <label>26</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Deber</surname>
                            <given-names>R B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kraetschmer</surname>
                            <given-names>N</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Irvine</surname>
                            <given-names>J</given-names>
                        </name>
                    </person-group>
                    <article-title>What role do patients wish to play in treatment decision making?</article-title>
                    <source>Arch Intern Med</source>
                    <year>1996</year>
                    <month>7</month>
                    <day>8</day>
                    <volume>156</volume>
                    <issue>13</issue>
                    <fpage>1414</fpage>
                    <lpage>20</lpage>
                    <pub-id pub-id-type="medline">96268174</pub-id>
                    <pub-id pub-id-type="doi">10.1001/archinte.156.13.1414</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref27">
                <label>27</label>
                <nlm-citation citation-type="book" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Glasgow</surname>
                            <given-names>RE</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Eakin</surname>
                            <given-names>EG</given-names>
                        </name>
                    </person-group>
                    <person-group person-group-type="editor">
                        <name name-style="western">
                            <surname>Snoek</surname>
                            <given-names>FJ</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Skinner</surname>
                            <given-names>TC</given-names>
                        </name>
                    </person-group>
                    <article-title>Medical office-based interventions</article-title>
                    <source>Psychology in Diabetes Care</source>
                    <year>2000</year>
                    <publisher-loc>New York</publisher-loc>
                    <publisher-name>John Wiley &amp; Sons</publisher-name>
                    <fpage>141</fpage>
                    <lpage>168</lpage>
                </nlm-citation>
            </ref>
            <ref id="ref28">
                <label>28</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Piette</surname>
                            <given-names>J D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Weinberger</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Mcphee</surname>
                            <given-names>S J</given-names>
                        </name>
                    </person-group>
                    <article-title>The effect of automated calls with telephone nurse follow-up on patient-centered outcomes of diabetes care: a randomized, controlled trial</article-title>
                    <source>Med Care</source>
                    <year>2000</year>
                    <month>2</month>
                    <volume>38</volume>
                    <issue>2</issue>
                    <fpage>218</fpage>
                    <lpage>30</lpage>
                    <pub-id pub-id-type="medline">20123091</pub-id>
                    <pub-id pub-id-type="doi">10.1097/00005650-200002000-00011</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref29">
                <label>29</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <collab xlink:type="simple">National Telecommunications and Information Administration</collab>
                    <source>Falling through the net, toward digital inclusion</source>
                    <year>2000</year>
                    <month>10</month>
                    <access-date>2001 Aug 30</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.ntia.doc.gov/ntiahome/fttn00/contents00.html" xlink:type="simple">http://www.ntia.doc.gov/ntiahome/fttn00/contents00.html</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref30">
                <label>30</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Stewart</surname>
                            <given-names>M</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brown</surname>
                            <given-names>J B</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Boon</surname>
                            <given-names>H</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Galajda</surname>
                            <given-names>J</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Meredith</surname>
                            <given-names>L</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Sangster</surname>
                            <given-names>M</given-names>
                        </name>
                    </person-group>
                    <article-title>Evidence on patient-doctor communication</article-title>
                    <source>Cancer Prev Control</source>
                    <year>1999</year>
                    <month>2</month>
                    <volume>3</volume>
                    <issue>1</issue>
                    <fpage>25</fpage>
                    <lpage>30</lpage>
                    <pub-id pub-id-type="medline">99403997</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref31">
                <label>31</label>
                <nlm-citation citation-type="web" xlink:type="simple">
                    <collab xlink:type="simple">American Telemedicine Association</collab>
                    <source>ATA Adopts Telehomecare Clinical Guidelines</source>
                    <access-date>2001 Aug 27</access-date>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://www.atmeda.org/news/list.html" xlink:type="simple">http://www.atmeda.org/news/list.html</ext-link></comment>
                </nlm-citation>
            </ref>
            <ref id="ref32">
                <label>32</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Buchanan</surname>
                            <given-names>B G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Carenini</surname>
                            <given-names>G</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Mittal</surname>
                            <given-names>V O</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Moore</surname>
                            <given-names>J D</given-names>
                        </name>
                    </person-group>
                    <article-title>Designing computer-based frameworks that facilitate doctor--patient collaboration</article-title>
                    <source>Artif Intell Med</source>
                    <year>1998</year>
                    <month>2</month>
                    <volume>12</volume>
                    <issue>2</issue>
                    <fpage>169</fpage>
                    <lpage>91</lpage>
                    <pub-id pub-id-type="medline">98179045</pub-id>
                    <pub-id pub-id-type="pii">S0933365797000493</pub-id>
                    <pub-id pub-id-type="doi">10.1016/S0933-3657(97)00049-3</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref33">
                <label>33</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mazzi</surname>
                            <given-names>C</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ganguly</surname>
                            <given-names>P</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kidd</surname>
                            <given-names>M</given-names>
                        </name>
                    </person-group>
                    <article-title>Healthcare applications based on software agents</article-title>
                    <source>Medinfo</source>
                    <year>2001</year>
                    <volume>10</volume>
                    <issue>Pt 1</issue>
                    <fpage>136</fpage>
                    <lpage>40</lpage>
                    <pub-id pub-id-type="medline">21490508</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref34">
                <label>34</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Zrebiec</surname>
                            <given-names>J F</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Jacobson</surname>
                            <given-names>A M</given-names>
                        </name>
                    </person-group>
                    <article-title>What attracts patients with diabetes to an internet support group? A 21-month longitudinal website study</article-title>
                    <source>Diabet Med</source>
                    <year>2001</year>
                    <month>2</month>
                    <volume>18</volume>
                    <issue>2</issue>
                    <fpage>154</fpage>
                    <lpage>8</lpage>
                    <pub-id pub-id-type="pii">dme443</pub-id>
                    <pub-id pub-id-type="medline">21150039</pub-id>
                    <pub-id pub-id-type="doi">10.1046/j.1464-5491.2001.00443.x</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref35">
                <label>35</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Iafusco</surname>
                            <given-names>D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Ingenito</surname>
                            <given-names>N</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Prisco</surname>
                            <given-names>F</given-names>
                        </name>
                    </person-group>
                    <article-title>The chatline as a communication and educational tool in adolescents with insulin-dependent diabetes: preliminary observations</article-title>
                    <source>Diabetes Care</source>
                    <year>2000</year>
                    <month>12</month>
                    <volume>23</volume>
                    <issue>12</issue>
                    <fpage>1853</fpage>
                    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11128367" ext-link-type="uri">http://care.diabetesjournals.org/cgi/pmidlookup?view=long&amp;pmid=11128367</ext-link></comment>
                    <pub-id pub-id-type="medline">21012155</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref36">
                <label>36</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Mandl</surname>
                            <given-names>K D</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Kohane</surname>
                            <given-names>I S</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Brandt</surname>
                            <given-names>A M</given-names>
                        </name>
                    </person-group>
                    <article-title>Electronic patient-physician communication: problems and promise</article-title>
                    <source>Ann Intern Med</source>
                    <year>1998</year>
                    <month>9</month>
                    <day>15</day>
                    <volume>129</volume>
                    <issue>6</issue>
                    <fpage>495</fpage>
                    <lpage>500</lpage>
                    <comment><ext-link xmlns:xlink="http://www.w3.org/1999/xlink" xlink:type="simple" xlink:href="http://www.annals.org/cgi/pmidlookup?view=long&amp;pmid=9735088" ext-link-type="uri">http://www.annals.org/cgi/pmidlookup?view=long&amp;pmid=9735088</ext-link></comment>
                    <pub-id pub-id-type="medline">98396847</pub-id>
                    </nlm-citation>
            </ref>
            <ref id="ref37">
                <label>37</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Landis</surname>
                            <given-names>J R</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Koch</surname>
                            <given-names>G G</given-names>
                        </name>
                    </person-group>
                    <article-title>The measurement of observer agreement for categorical data</article-title>
                    <source>Biometrics</source>
                    <year>1977</year>
                    <month>3</month>
                    <volume>33</volume>
                    <issue>1</issue>
                    <fpage>159</fpage>
                    <lpage>74</lpage>
                    <pub-id pub-id-type="medline">77135196</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref38">
                <label>38</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Kendall</surname>
                            <given-names>DM</given-names>
                        </name>
                        <name name-style="western">
                            <surname>Bergenstal</surname>
                            <given-names>RM</given-names>
                        </name>
                    </person-group>
                    <article-title>Comprehensive management of patients with type 2 diabetes: establishing priorities of care</article-title>
                    <source>Am J Manag Care</source>
                    <year>2001</year>
                    <month>8</month>
                    <volume>7</volume>
                    <issue>10 Suppl</issue>
                    <fpage>S327</fpage>
                    <lpage>S343</lpage>
                    <pub-id pub-id-type="medline">11519398</pub-id>
                </nlm-citation>
            </ref>
            <ref id="ref39">
                <label>39</label>
                <nlm-citation citation-type="journal" xlink:type="simple">
                    <person-group person-group-type="author">
                        <name name-style="western">
                            <surname>Eysenbach</surname>
                            <given-names>G</given-names>
                        </name>
                    </person-group>
                    <article-title>Consumer health informatics</article-title>
                    <source>BMJ</source>
                    <year>2000</year>
                    <month>6</month>
                    <day>24</day>
                    <volume>320</volume>
                    <issue>7251</issue>
                    <fpage>1713</fpage>
                    <lpage>6</lpage>
                    <comment><ext-link ext-link-type="uri" xlink:href="http://bmj.com/cgi/content/full/320/7251/1713?view=full&amp;pmid=10864552" xlink:type="simple">http://bmj.com/cgi/content/full/320/7251/1713?view=full&amp;pmid=10864552</ext-link></comment>
                    <pub-id pub-id-type="medline">20323060</pub-id>
                    <pub-id pub-id-type="doi">10.1136/bmj.320.7251.1713</pub-id>
                    </nlm-citation>
            </ref>
        </ref-list>
    </back>
</article>
