<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<?covid-19-tdm?>
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="review-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v22i11e19375</article-id>
      <article-id pub-id-type="pmid">33035177</article-id>
      <article-id pub-id-type="doi">10.2196/19375</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Review</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Review</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Barriers and Enablers in Implementing Electronic Consultations in Primary Care: Scoping Review</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Schüssler</surname>
            <given-names>Sandra</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Bahrami</surname>
            <given-names>Mohammad Amin</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Baines</surname>
            <given-names>Rebecca</given-names>
          </name>
          <degrees>MSc</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-9857-1976</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Tredinnick-Rowe</surname>
            <given-names>John</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1154-7410</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author">
          <name name-style="western">
            <surname>Jones</surname>
            <given-names>Ray</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-2963-3421</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author" corresp="yes">
          <name name-style="western">
            <surname>Chatterjee</surname>
            <given-names>Arunangsu</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <address>
            <institution>University of Plymouth</institution>
            <addr-line>Room 108 I 3 Endsleigh Place</addr-line>
            <addr-line>Drake Circus</addr-line>
            <addr-line>Plymouth, </addr-line>
            <country>United Kingdom</country>
            <phone>44 1752 586837</phone>
            <email>arunangsu.chatterjee@plymouth.ac.uk</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-9506-6007</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>University of Plymouth</institution>
        <addr-line>Plymouth</addr-line>
        <country>United Kingdom</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Arunangsu Chatterjee <email>arunangsu.chatterjee@plymouth.ac.uk</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>11</month>
        <year>2020</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>12</day>
        <month>11</month>
        <year>2020</year>
      </pub-date>
      <volume>22</volume>
      <issue>11</issue>
      <elocation-id>e19375</elocation-id>
      <history>
        <date date-type="received">
          <day>15</day>
          <month>4</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>12</day>
          <month>6</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>26</day>
          <month>6</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>26</day>
          <month>7</month>
          <year>2020</year>
        </date>
      </history>
      <copyright-statement>©Rebecca Baines, John Tredinnick-Rowe, Ray Jones, Arunangsu Chatterjee. Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 12.11.2020.</copyright-statement>
      <copyright-year>2020</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on http://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://www.jmir.org/2020/11/e19375" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Often promoted as a way to address increasing demands, improve patient accessibility, and improve overall efficiency, electronic consultations are becoming increasingly common in primary care, particularly in light of the current COVID-19 pandemic. However, despite their increasing use, a theoretically informed understanding of the factors that support and inhibit their effective implementation is severely limited.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>With this scoping review, we sought to identify the factors that support and inhibit the implementation of electronic consultations in primary care.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>In total, 5 electronic databases (PubMed, Medline, Embase, CINAHL, and PsycINFO) were systematically searched for studies published in 2009-2019 that explored the impact and/or implementation of electronic consultations in primary care. Database searches were supplemented by reference list and grey literature searches. Data were analyzed using inductive thematic analysis and synthesized using Normalization Process Theory (NPT).</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>In total, 227 articles were initially identified and 13 were included in this review. The main factors found to hinder implementation included awareness and expectations; low levels of engagement; perceived suitability for all patient groups, conditions, and demographics; cost; and other contextual factors. Reports of information technology reliability and clinical workload duplication (as opposed to reduction) also appeared detrimental. Conversely, the development of protocols and guidance; patient and staff education; strategic marketing; and patient and public involvement were all identified as beneficial in facilitating electronic consultation implementation.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>This review highlights the need for proactive engagement with patients and staff to facilitate understanding and awareness, process optimization, and delivery of coherent training and education that maximizes impact and success. Although the necessity to use online methods during the COVID-19 pandemic may have accelerated awareness, concerns over workload duplication and inequality of access may remain. Future research should explore health inequalities in electronic consultations and their economic impacts from multiple perspectives (eg, patient, professional, and commissioner) to determine their potential value. Further work to identify the role of meaningful patient involvement in digital innovation, implementation, and evaluation is also required following the rapid digitization of health and social care.</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>remote consultation</kwd>
        <kwd>COVID-19</kwd>
        <kwd>implementation science</kwd>
        <kwd>primary health care</kwd>
        <kwd>patient participation</kwd>
        <kwd>referral and consultation</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>With a 16% increase in the number of general practitioner (GP) consultations between 2007-2014 in England alone, primary care is considered to be at “saturation point” [<xref ref-type="bibr" rid="ref1">1</xref>]. General practice is often described as facing increasing demand, reduced accessibility, and heightened patient dissatisfaction [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref3">3</xref>]. As a result, primary care providers are being increasingly encouraged to adopt alternative, more digitally focused methods of care provision [<xref ref-type="bibr" rid="ref4">4</xref>], particularly in light of the recent COVID-19 outbreak. This drive toward digital platforms is reflected on an international scale [<xref ref-type="bibr" rid="ref5">5</xref>-<xref ref-type="bibr" rid="ref8">8</xref>], with digital platforms often promoted as a way to relieve pressures on existing services and improve accessibility, efficiency, and cost-effectiveness while simultaneously promoting self-management and patient-centered care [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref9">9</xref>]. However, critical exploration of these assumptions is severely limited.</p>
      <p>Within the United Kingdom, prior to the COVID-19 pandemic, there were two main electronic consultation providers: askmyGP and eConsult, the latter of which was previously known as WebGP [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. These systems are online triage tools designed to provide patients with an alternative way of contacting their GP practice [<xref ref-type="bibr" rid="ref4">4</xref>]. For the purposes of this article, we define the term electronic consultations as an online service that enables patients to access advice and care from a primary care practitioner or staff member by combining preliminary health issues and symptom checking with appointment booking. These products are distinct from video-based consultations that a doctor might use, such as Babylon, AccuRx, or LIVI. In the United Kingdom, electronic consultations are also distinct from their use in North America as a video system to communicate between family doctors and specialists in hospitals.</p>
      <p>While some research has explored the benefits of electronic consultations and the experiences of health care professionals using them, such work is often critiqued for its overreliance on pilot studies [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>]. Other criticisms of existing literature also include a limited understanding of the cost implications for both health care professionals and patients [<xref ref-type="bibr" rid="ref12">12</xref>-<xref ref-type="bibr" rid="ref14">14</xref>], and a predominate focus on professional perspectives [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref15">15</xref>], with limited exploration of patient experiences and expectations [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Furthermore, of the limited research conducted, the majority of it has focused on the experience of using electronic consultations, as opposed to the practicalities of implementing such technology. Finally, in spite of their increasing use, a theoretically informed understanding of the factors that support or inhibit the implementation of electronic consultations is severely limited [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref11">11</xref>], particularly in a UK setting [<xref ref-type="bibr" rid="ref16">16</xref>]. This review seeks to address this gap by identifying the factors that support and inhibit the implementation of electronic consultations in primary care using Normalization Process Theory (NPT) as a theoretical framework. This scoping review is well-timed given the rapidly increasing use of electronic consultations in light of the COVID-19 outbreak and the resulting mandatory shift toward total triage in primary care [<xref ref-type="bibr" rid="ref4">4</xref>]. Although focused on general practice and electronic consultations, the implications of this review may be relevant to other digital forms of health and care technology and their subsequent implementation.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Design</title>
        <p>A scoping review was conducted due to their ability to map existing evidence in an emerging field (such as electronic consultations), identify gaps in existing understanding, and incorporate different study designs that can be grouped together to evaluate a particular topic of interest [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>].</p>
      </sec>
      <sec>
        <title>Search Strategy</title>
        <p>As advised by an information specialist, 5 databases (PubMed, Medline, Embase, CINAHL, and PsycINFO) were systematically searched using the search terms “econsult” OR “electronic consultation” OR “WebGP” OR “non face-to-face consultations” OR “technology mediated consultations” AND “primary care” OR “GP” OR “general practice.” Search terms were designed and reviewed using the Peer Review of Electronic Search Strategies (PRESS) guidance [<xref ref-type="bibr" rid="ref19">19</xref>].</p>
        <p>A grey literature search was also conducted to ensure sufficient inclusivity and coverage. Grey literature was defined as “that which is produced on all levels of government, academics, business and industry in print and electronic formats, but which is not controlled by commercial publishers” [<xref ref-type="bibr" rid="ref20">20</xref>]. The peer-reviewed search strategy was also used in Google Scholar.</p>
      </sec>
      <sec>
        <title>Inclusion and Exclusion Criteria</title>
        <p>Research studies that explored the impact or implementation of electronic consultation platforms designed to be used by a primary care clinician and patient, published in the English language, of any study design (including opinion pieces and editorial letters) were included. Research studies that were not published in the English language or focused on electronic consultation platforms outside of primary care (eg, between non–primary care specialties) were excluded to retain a relevant focus of interest.</p>
        <p>While the researchers aimed to be inclusive, due to limited resources, a sensitive translation of non-English texts could not be provided. Finally, to ensure only the most contemporary literature was included, a time limit was applied (January 1, 2009, to January 31, 2019). Literature searches were conducted on February 1, 2019.</p>
      </sec>
      <sec>
        <title>Screening and Eligibility</title>
        <p>All identified articles were screened using a two-stage process. First, the title and abstract of all identified articles were reviewed independently by two researchers using the inclusion criteria outlined above. This process was facilitated using Rayyan, software for conducting reviews [<xref ref-type="bibr" rid="ref21">21</xref>]. The full texts of potentially relevant articles were then reviewed for inclusion. Any disagreements were resolved through the inclusion of a third researcher. Database searches were also supplemented by reference list searches of included studies.</p>
        <p>Data was extracted from the included studies independently by two reviewers using a pilot tested extraction form. Data extracted included author names, date of publication, setting, study type, sample, analysis method, and reported findings/author interpretations.</p>
      </sec>
      <sec>
        <title>Analysis</title>
        <p>Included studies were initially coded independently by two reviewers using inductive thematic analysis as outlined by Braun and Clarke [<xref ref-type="bibr" rid="ref22">22</xref>]. Identified themes were then synthesized using NPT as outlined below by the same two reviewers.</p>
      </sec>
      <sec>
        <title>Synthesis</title>
        <p>Data was synthesized using NPT [<xref ref-type="bibr" rid="ref23">23</xref>] as an analytical framework. NPT was originally developed to understand the embedding of new, particularly complex technologies in health care systems [<xref ref-type="bibr" rid="ref23">23</xref>], providing a clear rationale for its inclusion in this research. As outlined by Murray [<xref ref-type="bibr" rid="ref23">23</xref>], NPT is underpinned by four constructs that often operate simultaneously:</p>
        <list list-type="order">
          <list-item>
            <p>Coherence: how people make sense of a new technology/system/process.</p>
          </list-item>
          <list-item>
            <p>Cognitive participation: how people engage with a new technology/system/process.</p>
          </list-item>
          <list-item>
            <p>Collective action: how people make new technologies/systems/processes work in practice (or not).</p>
          </list-item>
          <list-item>
            <p>Reflexive monitoring: how people assess the value of new technologies/systems/processes.</p>
          </list-item>
        </list>
        <p>For the purposes of this article, we report findings on the first three domains of NPT (coherence, participation, and collective action). As previously mentioned, of the limited research previously conducted, most has focused on professional experiences (eg, reflexive monitoring). We therefore focus on the three remaining domains of NPT to avoid duplication.</p>
      </sec>
      <sec>
        <title>Quality Appraisal</title>
        <p>In line with scoping review practice [<xref ref-type="bibr" rid="ref18">18</xref>], included studies were not quality appraised.</p>
      </sec>
      <sec>
        <title>Ethical Approval</title>
        <p>Participation in this research was entirely voluntary. All participants gave full informed consent. The University of Plymouth, Faculty of Health and Human Sciences (Reference number 18/19-1060) provided ethical approval.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <sec>
        <title>Overview</title>
        <p>From the 227 articles initially identified, 13 were included for the purposes of this review (<xref rid="figure1" ref-type="fig">Figure 1</xref>). <xref ref-type="table" rid="table1">Table 1</xref> summarizes the characteristics of the included studies.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>PRISMA diagram of included studies. PRISMA: Preferred Reporting Items for Systematic Reviews and Meta-analyses.</p>
          </caption>
          <graphic xlink:href="jmir_v22i11e19375_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>Characteristics of included studies.</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="90"/>
            <col width="90"/>
            <col width="150"/>
            <col width="180"/>
            <col width="150"/>
            <col width="190"/>
            <col width="150"/>
            <thead>
              <tr valign="top">
                <td>Authors</td>
                <td>Country</td>
                <td>Study type</td>
                <td>Intervention</td>
                <td>Setting</td>
                <td>Participants and databases</td>
                <td>Analysis method</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td>Atherton et al, 2018 [<xref ref-type="bibr" rid="ref24">24</xref>]</td>
                <td>England and Scotland</td>
                <td>Mixed methods case design</td>
                <td>Alternatives to face-to-face consultations</td>
                <td>General practices with varied experience of implementing alternatives to face-to-face consultations</td>
                <td>Patients and practice staff</td>
                <td>Descriptive statistics, multivariate analysis, and coding of qualitative data from EMIS Health</td>
              </tr>
              <tr valign="top">
                <td>Banks et al, 2018 [<xref ref-type="bibr" rid="ref4">4</xref>]</td>
                <td>West of England</td>
                <td>Qualitative</td>
                <td>eConsult</td>
                <td>General practices that piloted an e-consultation system for 15 months during 2015 and 2016</td>
                <td>23 semistructured interviews with staff members</td>
                <td>Thematic analysis of interview data</td>
              </tr>
              <tr valign="top">
                <td>Cowie et al, 2018 [<xref ref-type="bibr" rid="ref25">25</xref>]</td>
                <td>Scotland</td>
                <td>Mixed methods</td>
                <td>eConsult</td>
                <td>11 general practices piloting eConsult</td>
                <td>44 semistructured practice staff interviews, 1 focus group (4 staff), and 291 patient survey responses</td>
                <td>Health economics, thematic analysis and coding, patient survey, descriptive statistics</td>
              </tr>
              <tr valign="top">
                <td>Carter et al, 2018 [<xref ref-type="bibr" rid="ref6">6</xref>]</td>
                <td>South West England</td>
                <td>Mixed methods</td>
                <td>WebGP</td>
                <td>6 GP practices in Devon</td>
                <td>Six practices provided consultations data; 20 GPs completed case reports (regarding 61 e-consults); 81 patients completed questionnaires; 5 GPs and 5 administrators were interviewed</td>
                <td>Statistical analysis of surveys, thematic analysis of interview data</td>
              </tr>
              <tr valign="top">
                <td>Farr et al, 2018 [<xref ref-type="bibr" rid="ref3">3</xref>]</td>
                <td>South West England</td>
                <td>Mixed methods</td>
                <td>eConsult</td>
                <td>6 general practices</td>
                <td>23 practice staff interviews, patient survey data for 756 e-consultations from 36 practices</td>
                <td>Economic analysis on usage and costs, Normalization Process Theory, inductive thematic analysis, patient survey and record data statistical analysis</td>
              </tr>
              <tr valign="top">
                <td>Ogden, 2018 [<xref ref-type="bibr" rid="ref26">26</xref>]</td>
                <td>N/A<sup>a</sup></td>
                <td>Opinion piece</td>
                <td>Online consulting</td>
                <td>N/A</td>
                <td>N/A</td>
                <td>Descriptive statistics</td>
              </tr>
              <tr valign="top">
                <td>Marshall et al, 2018 [<xref ref-type="bibr" rid="ref8">8</xref>]</td>
                <td>N/A</td>
                <td>Opinion piece</td>
                <td>Online consulting</td>
                <td>N/A</td>
                <td>N/A</td>
                <td>N/A</td>
              </tr>
              <tr valign="top">
                <td>Casey et al, 2017 [<xref ref-type="bibr" rid="ref1">1</xref>]</td>
                <td>N/A</td>
                <td>Mixed methods case study</td>
                <td>Online consultation system</td>
                <td>Inner-city general practice</td>
                <td>Information technology developers, clinicians, and administrative staff</td>
                <td>Thematic and discourse analysis of interview data</td>
              </tr>
              <tr valign="top">
                <td>Edwards et al, 2017 [<xref ref-type="bibr" rid="ref16">16</xref>]</td>
                <td>South West of England</td>
                <td>Observational study</td>
                <td>eConsult</td>
                <td>South West of England</td>
                <td>eConsult data obtained from 36 general practices</td>
                <td>Economic analysis, website analytics, survey-based statistical analysis</td>
              </tr>
              <tr valign="top">
                <td>Wise, 2017 [<xref ref-type="bibr" rid="ref10">10</xref>]</td>
                <td>N/A</td>
                <td>Opinion piece</td>
                <td>Online consultations</td>
                <td>N/A</td>
                <td>N/A</td>
                <td>N/A</td>
              </tr>
              <tr valign="top">
                <td>Hanna et al, 2012 [<xref ref-type="bibr" rid="ref27">27</xref>]</td>
                <td>Scotland</td>
                <td>Qualitative</td>
                <td>Non–face-to-face consultations</td>
                <td>Scotland</td>
                <td>20 semistructured interviews with general practitioners</td>
                <td>Thematic analysis of interviews</td>
              </tr>
              <tr valign="top">
                <td>Mair et al, 2012 [<xref ref-type="bibr" rid="ref28">28</xref>]</td>
                <td>N/A</td>
                <td>Systematic review</td>
                <td>eHealth systems (only information pertaining to online consultations recorded)</td>
                <td>N/A</td>
                <td>MEDLINE, EMBASE, CINAHL, PsycINFO, and the Cochrane Library were searched for reviews published between January 1, 1995, and March 17, 2009. Studies had to be systematic reviews, narrative reviews, qualitative meta-syntheses or meta-ethnographies of eHealth implementation</td>
                <td>Evidence synthesis</td>
              </tr>
              <tr valign="top">
                <td>Hanna et al, 2011 [<xref ref-type="bibr" rid="ref15">15</xref>]</td>
                <td>Scotland</td>
                <td>Mixed methods</td>
                <td>Non–face-to-face consultations</td>
                <td>Scotland</td>
                <td>600 practice manager questionnaire responses, 20 practice manager interviews</td>
                <td>Chi-squared test from a survey, thematic analysis of interviews</td>
              </tr>
            </tbody>
          </table>
          <table-wrap-foot>
            <fn id="table1fn1">
              <p><sup>a</sup>N/A: not applicable.</p>
            </fn>
          </table-wrap-foot>
        </table-wrap>
        <p>As demonstrated in <xref ref-type="table" rid="table1">Table 1</xref>, most of the research published in the last two years has used a mixed methods design. There has been a particular focus on the introduction of electronic consultations in South West England and Scotland. Our results are presented in line with the three selected domains of NPT as outlined above. We begin by presenting the factors that inhibit implementation, followed by those that facilitate implementation.</p>
      </sec>
      <sec>
        <title>Factors That Inhibit Electronic Consultation Implementation</title>
        <sec>
          <title>Coherence: What Factors Inhibit the Understanding of Electronic Consultations?</title>
          <sec>
            <title>Awareness and Expectations</title>
            <p>A lack of understanding and awareness about the purpose of electronic consultation systems appears particularly problematic in their implementation and impact [<xref ref-type="bibr" rid="ref24">24</xref>]. Patient understanding and awareness was reportedly mixed, with the effectiveness of electronic consultation advertising often called into question [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref25">25</xref>]. Patients were often not aware of alternative methods available to them, and how to access or operate such services [<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. Atherton suggests this is a matter for individual practices rather than policy directives [<xref ref-type="bibr" rid="ref24">24</xref>].</p>
            <p>A difference in staff and patient expectations regarding the appropriate use of electronic consultations was also reported [<xref ref-type="bibr" rid="ref3">3</xref>]. This primarily related to the inappropriate use of electronic consultations to get a timelier face-to-face appointment as later discussed.</p>
          </sec>
        </sec>
        <sec>
          <title>Cognitive Participation: What Factors Inhibit How People Engage With Electronic Consultations?</title>
          <sec>
            <title>Low Uptake</title>
            <p>Studies identified a low uptake as a further barrier to electronic consultation implementation [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. Engagement levels were often reported to be much lower than expected [<xref ref-type="bibr" rid="ref3">3</xref>]. For example, Banks et al [<xref ref-type="bibr" rid="ref4">4</xref>] reported that only around 6% of practices used some form of electronic consultation. Similarly, Edwards et al [<xref ref-type="bibr" rid="ref16">16</xref>] concluded that of the 36 practices reviewed, there was an average of 2 electronic consultations per 1000 patients per month. Such findings mirrored calculations by Cowie and colleagues [<xref ref-type="bibr" rid="ref11">11</xref>]; the highest submission rate reported was almost 0.1 consultations per patient per year.</p>
          </sec>
          <sec>
            <title>Suitability for Certain Patient Groups, Conditions, and Demographics</title>
            <p>The suitability of electronic consultations for certain patient groups, conditions, and demographics was also called into question. Electronic consultations were considered more suitable for certain patient groups with discrete, “straightforward,” or familiar ongoing treatment queries [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref27">27</xref>]. In comparison, face-to-face consultations were reportedly preferred for new, acute, or complex cases that required physical examination and/or regular monitoring [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref26">26</xref>]. As a result, some health care professionals expressed concern that patient groups including those with long-term conditions, multimorbidity, and/or mental health problems would be disadvantaged or unintentionally excluded from electronic consultation opportunities [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. Some GPs repeatedly expressed concerns that introducing a new technology could create or widen existing inequalities in access to health care [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref29">29</xref>]. Atherton acknowledged the purposeful selection of patients by health care professionals, including receptionists and administrators, based on their assumptions of who they felt would be able to use electronic consultations appropriately [<xref ref-type="bibr" rid="ref24">24</xref>], highlighting further concerns regarding unintentional or intentional exclusion.</p>
            <p>In regard to demographics, included articles also suggested that women are almost twice as likely to use electronic consultation systems than men [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Younger patients are also more likely to engage with electronic consultation systems, with levels of engagement typically declining with age [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref24">24</xref>]. Levels of education, language [<xref ref-type="bibr" rid="ref3">3</xref>], and employment were also shown to influence electronic consultation use [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref24">24</xref>,<xref ref-type="bibr" rid="ref27">27</xref>].</p>
          </sec>
          <sec>
            <title>Contextual Factors: Practice Size, Deprivation, and Geographical Location</title>
            <p>Following this, a range of contextual factors including practice size, deprivation, and geographical location appeared to influence electronic consultation implementation and effect [<xref ref-type="bibr" rid="ref29">29</xref>]. Hanna et al [<xref ref-type="bibr" rid="ref29">29</xref>] suggested that the bigger the practice size, the more support available, including a website, an information technology (IT) manager, and a triage team, thereby influencing implementation efforts and sustainability [<xref ref-type="bibr" rid="ref29">29</xref>].</p>
          </sec>
          <sec>
            <title>Cost</title>
            <p>Finally, cost was reported as a significant barrier to electronic consultation engagement and implementation [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>], with limited evidence available to justify its associated expense [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. As suggested by Farr et al [<xref ref-type="bibr" rid="ref3">3</xref>], costs often prohibited long-term engagement with electronic consultation systems.</p>
          </sec>
        </sec>
        <sec>
          <title>Collective Action: What Factors Inhibit How People Work With Electronic Consultations?</title>
          <sec>
            <title>Additional Time and Increased Workload</title>
            <p>In regard to how people work with electronic consultations (collective action), the majority of included studies reported an increase in clinician workload. This was often attributed to additional follow-up and/or face-to-face consultations due to a lack of sufficient information being provided [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref16">16</xref>,<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. Although one study acknowledged a potential benefit of having the clinical issue documented prior to a face-to-face appointment [<xref ref-type="bibr" rid="ref4">4</xref>], this conclusion was contradicted by another article that found that follow-up appointments were typically longer than the national average (14.5 minutes versus 9.2 minutes for face-to-face) [<xref ref-type="bibr" rid="ref16">16</xref>]. However, it is important to note that this disparity may be a result of a lack of GP continuity between electronic and follow-up consultations [<xref ref-type="bibr" rid="ref3">3</xref>].</p>
            <p>GPs also reported significant difficulties in establishing a patient’s primary concern, following an inability to probe for further information [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Where reported, most electronic consultations resulted in either follow-up phone calls or face-to-face appointments, leading GPs to report a duplication in their workload, as opposed to a desired reduction [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>]. The relatively low uptake of electronic consultation systems as previously described appeared to exacerbate this concern [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
            <p>An increase in administrative workload was also reported, particularly the frequent need to contact patients regarding their electronic consultation outcome or to arrange a face-to-face or telephone appointment [<xref ref-type="bibr" rid="ref11">11</xref>]. Cowie et al [<xref ref-type="bibr" rid="ref11">11</xref>] reported that most practices made at least three attempts to contact patients, frequently reporting that contact could not be made. The use of a withheld number was suggested as a possible explanation. Some practices have developed methods to facilitate email contact as a solution to this problem, although issues of data security and protection remained a concern [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
          </sec>
          <sec>
            <title>Medicolegal Concerns</title>
            <p>Linked to concerns of data security and protection, medicolegal issues were also identified as a barrier to electronic consultation implementation and use. Concerns about the negotiation of clinical risk and diagnosis uncertainty were repeatedly expressed within the context of an increasingly litigious culture [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref29">29</xref>].</p>
          </sec>
          <sec>
            <title>Potential to “Game” the System</title>
            <p>As previously mentioned, some practice staff members expressed concern regarding “patients gaming the system” [<xref ref-type="bibr" rid="ref4">4</xref>], with some reportedly using the system to achieve a timelier face-to-face appointment [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. As reported by Banks et al [<xref ref-type="bibr" rid="ref4">4</xref>] and others, staff often felt patients could get an appointment quicker using electronic consultation methods by circumventing traditional, often telephone-based appointment systems. The responsibility of the practice to contact the patient within a specified time frame, often by the end of the next working day [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>], as well as the content of marketing material [<xref ref-type="bibr" rid="ref11">11</xref>], appeared to influence this perception.</p>
          </sec>
          <sec>
            <title>IT Reliability</title>
            <p>Finally, the reliability of technology was seen as a potential barrier to electronic consultation implementation and use. As reported by Hanna et al [<xref ref-type="bibr" rid="ref29">29</xref>], some interviewees felt new IT systems were highly reliable, while others expressed concern about network availability and speed [<xref ref-type="bibr" rid="ref27">27</xref>,<xref ref-type="bibr" rid="ref29">29</xref>]. The interoperability of IT systems was also identified as influential in electronic consultation implementation [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref11">11</xref>], with competing or conflicting IT systems proving to be problematic.</p>
          </sec>
        </sec>
      </sec>
      <sec>
        <title>Factors That Facilitate Implementation</title>
        <p>In addition to the barriers outlined above, a number of facilitating factors were also identified in regard to electronic consultation implementation and use. Each are discussed in turn below.</p>
        <sec>
          <title>Coherence: What Factors Facilitate the Understanding of Electronic Consultations?</title>
          <sec>
            <title>Protocols and Strategies</title>
            <p>First, the provision of protocols, strategies, and/or guidance including medicolegal advice was considered integral to the effective implementation of electronic consultations [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref26">26</xref>-<xref ref-type="bibr" rid="ref28">28</xref>]. As identified by Cowie et al [<xref ref-type="bibr" rid="ref11">11</xref>], the development of a clear strategy for introducing electronic consultations prior to any implementation was considered fundamental. Such protocols should be developed in collaboration with staff, patients, and electronic consultation champions (as discussed below) [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
          </sec>
          <sec>
            <title>Education</title>
            <p>Following this, quality patient and staff education on what electronic consultations are, what they are not, and when to use them was widely encouraged [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Cowie et al [<xref ref-type="bibr" rid="ref11">11</xref>] recommended the construction of practice process diagrams along with clear objectives to facilitate staff understanding and awareness. An exploration of staff expectations was also considered integral [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
          </sec>
          <sec>
            <title>Focused Marketing</title>
            <p>Linked to the provision of education was a desire for more focused marketing [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref26">26</xref>]. Identified ways of promoting and marketing electronic consultations included leaflets, clear website positioning, posters, and recorded telephone messages [<xref ref-type="bibr" rid="ref26">26</xref>]. As suggested by Ogden et al [<xref ref-type="bibr" rid="ref26">26</xref>], recorded messages appeared particularly influential when recorded by a doctor.</p>
          </sec>
          <sec>
            <title>Patient and Public Involvement and Wider Network Development</title>
            <p>Combining the development of protocols, education, and marketing, patient and public involvement was also considered integral prior to and during any electronic consultation implementation to ensure acceptability and understanding [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Despite this, few practices reported engaging with patients in electronic consultation design, implementation, or evaluation [<xref ref-type="bibr" rid="ref3">3</xref>]. This may explain some of the issues identified regarding patient understanding and awareness. Patient participation groups (PPGs) were identified as a beneficial resource for facilitating patient engagement, as were patient and public involvement workshops to provide feedback on electronic consultation systems, protocols, and experiences [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
          </sec>
        </sec>
        <sec>
          <title>Cognitive Participation: What Factors Support How Stakeholders Engage With Electronic Consultation?</title>
          <sec>
            <title>Staff Training</title>
            <p>Moving on to cognitive participation, how stakeholders engaged with electronic consultations, adequate staff training that addressed how electronic consultations operate, how electronic consultations fit with existing practice processes, and individual responsibilities were considered essential for successful implementation [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref26">26</xref>-<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          </sec>
          <sec>
            <title>eConsultation Champions</title>
            <p>An electronic consultation champion was also considered helpful in ensuring effective implementation and engagement by promoting its use among patients and more reticent staff members [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref28">28</xref>]. However, the risk of jeopardizing staff and patient commitment by recruiting a critical champion was also acknowledged [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          </sec>
        </sec>
        <sec>
          <title>Collective Action: What Factors Facilitate How Stakeholders Work With Electronic Consultations?</title>
          <sec>
            <title>Strategic Marketing</title>
            <p>With regard to collective action (how stakeholders work with electronic consultations), strategic marketing or signposting was identified as a facilitating factor [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. For example, signposting patients to use electronic consultations in certain situations where only remote GP access was likely to be required, including follow-up appointments, general administrative queries, repeat prescriptions, and general advice [<xref ref-type="bibr" rid="ref11">11</xref>]. A more focused marketing strategy was considered to help improve overall efficiency, ensuring patients who might benefit from the service most were directly encouraged to engage with it [<xref ref-type="bibr" rid="ref11">11</xref>].</p>
          </sec>
          <sec>
            <title>Notification Alerts</title>
            <p>In response to concerns of engaging in telephone “ping-pong,” a desire for the development of an alerting system to inform patients of an incoming call following an electronic consultation request was also expressed [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>]. Some practices already provide an estimated time for patient call-backs, clearly informing the patient that this may be from an unavailable or unknown number [<xref ref-type="bibr" rid="ref11">11</xref>]. The development and effective incorporation of email contact was also expressed as a desirable solution to improving electronic consultation implementation and effectiveness [<xref ref-type="bibr" rid="ref11">11</xref>], although this is reliant on the smooth integration of electronic consultations into existing IT systems as described below [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          </sec>
          <sec>
            <title>Integration of Technology and Adequate Resourcing</title>
            <p>Mair et al [<xref ref-type="bibr" rid="ref28">28</xref>] suggested clinicians may be deterred or become resistant to working with a system that adds complexity or requires additional effort and time. IT support was therefore considered integral to implementation [<xref ref-type="bibr" rid="ref27">27</xref>], as was adequate resourcing, particularly financial support [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          </sec>
          <sec>
            <title>Pairing of GPs With Patients</title>
            <p>Finally, the possibility of allocating electronic consultations to GPs who had had previous contact with the patient was seen as an effective way to facilitate implementation [<xref ref-type="bibr" rid="ref3">3</xref>]. <xref ref-type="table" rid="table2">Table 2</xref> provides a visual representation of the barriers and facilitators outlined in this review, according to the relevant domains of NPT.</p>
            <table-wrap position="float" id="table2">
              <label>Table 2</label>
              <caption>
                <p>Identified barriers and facilitators to electronic consultation implementation and use.</p>
              </caption>
              <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
                <col width="180"/>
                <col width="380"/>
                <col width="440"/>
                <thead>
                  <tr valign="top">
                    <td>Normalization Process Theory domain</td>
                    <td>Barriers</td>
                    <td>Facilitators</td>
                  </tr>
                </thead>
                <tbody>
                  <tr valign="top">
                    <td>Coherence</td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Lack of understanding regarding its purpose and intended use</p>
                        </list-item>
                      </list>
                    </td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Development of protocols, strategies, and guidance, including medicolegal advice</p>
                        </list-item>
                        <list-item>
                          <p>Patient and staff education</p>
                        </list-item>
                        <list-item>
                          <p>Focused marketing</p>
                        </list-item>
                        <list-item>
                          <p>Wider consultation with patients and staff members prior to implementation</p>
                        </list-item>
                      </list>
                    </td>
                  </tr>
                  <tr valign="top">
                    <td>Participation</td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Low uptake</p>
                        </list-item>
                        <list-item>
                          <p>Mainly administrative requests</p>
                        </list-item>
                        <list-item>
                          <p>Suitability for certain patient groups and conditions</p>
                        </list-item>
                        <list-item>
                          <p>Contextual factors including practice size, deprivation, and geographical location</p>
                        </list-item>
                        <list-item>
                          <p>Cost</p>
                        </list-item>
                        <list-item>
                          <p>Limited patient involvement</p>
                        </list-item>
                      </list>
                    </td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Staff training</p>
                        </list-item>
                        <list-item>
                          <p>Strategic/targeted patient use for those most likely to benefit from electronic consultations</p>
                        </list-item>
                        <list-item>
                          <p>Electronic consultation champion</p>
                        </list-item>
                      </list>
                    </td>
                  </tr>
                  <tr valign="top">
                    <td>Action</td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Purposeful patient selection.</p>
                        </list-item>
                        <list-item>
                          <p>Additional time or increased workload</p>
                        </list-item>
                        <list-item>
                          <p>Medicolegal concerns</p>
                        </list-item>
                        <list-item>
                          <p>The potential to “game” the system</p>
                        </list-item>
                        <list-item>
                          <p>Information technology reliability</p>
                        </list-item>
                      </list>
                    </td>
                    <td>
                      <list list-type="bullet">
                        <list-item>
                          <p>Effective signposting informing patients of when to use electronic consultation and when not</p>
                        </list-item>
                        <list-item>
                          <p>Notification alerts to alleviate administrative issues related to contacting patients</p>
                        </list-item>
                        <list-item>
                          <p>Integration of technology and adequate resourcing</p>
                        </list-item>
                        <list-item>
                          <p>Pairing of general practitioner with prior patient contact</p>
                        </list-item>
                      </list>
                    </td>
                  </tr>
                </tbody>
              </table>
            </table-wrap>
          </sec>
        </sec>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Summary</title>
        <p>This review addressed an identified gap in existing literature by developing a theoretically informed understanding of the factors that support and inhibit the practical implementation of electronic consultations in primary care [<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Review findings suggest limited staff and patient awareness and understandings; low levels of engagement; perceived suitability for all patient groups, conditions, and demographics; cost; and other contextual factors including practice size, levels of deprivation, and geographical location are the main inhibitory factors of effectively implementing electronic consultations. As a result, the majority of included studies reported an increase in clinician and admin time, with many GPs reporting a duplication in workload, as opposed to the desired reduction. Findings from our review also identified a number of factors that could help facilitate the effective implementation of electronic consultations. These primarily focused on the provision of staff training, protocols, strategies, and guidance; enhanced patient awareness and education; strategic marketing; notification alerts, and patient and public involvement in the innovation, implementation, and evaluation stages.</p>
      </sec>
      <sec>
        <title>Comparison With Existing Literature</title>
        <p>Similar to existing research, the included articles reported a shortcoming of electronic consultations in their current form [<xref ref-type="bibr" rid="ref4">4</xref>], with expressed skepticism regarding their financial investment [<xref ref-type="bibr" rid="ref3">3</xref>]. Furthermore, many health care professionals expressed concern that electronic consultation duplicated administrative and clinical workloads. Such findings have been widely reported by a number of studies [<xref ref-type="bibr" rid="ref1">1</xref>,<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref4">4</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. However, it is important to note that this may be due to the absence of supporting factors identified in this review and the relatively new emergence of electronic consultations. Further exploration of whether the presence of facilitating factors improves electronic consultation would be beneficial. Finally, we found a limited amount of research exploring electronic consultation experiences and impact from a patient perspective [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref11">11</xref>], identifying a further area for future research.</p>
      </sec>
      <sec>
        <title>Strengths and Limitations</title>
        <p>Strengths of this review include its exploration of both peer-reviewed and grey literature and novel application of a theoretical framework in the context of general practice and electronic consultations. However, its limitations must also be acknowledged. In line with scoping review practice, included articles were not quality appraised. The exclusion of non–English language texts may also have introduced research bias. Future research may benefit from a wider range of bibliographic databases, including technical databases such as IEEE. Exploration of any differences between private and free at the point of access service implementations/commissioning would also be beneficial<italic>.</italic></p>
      </sec>
      <sec>
        <title>Implications for Practice</title>
        <p>With these in mind, the implications from this review are clear. First, the implementation of electronic consultations appears most effective when both patients and staff members are involved in the design, implementation, and evaluation of their processes and outcomes. Second, the rationale, purpose, and intended use of electronic consultations needs to be effectively communicated to both patients and staff members to ensure appropriate use and implementation. This could be best achieved through targeted marketing as well as meaningful patient involvement to facilitate patient understanding and acceptability. Third, marketing materials should reflect the reality of the product proposed to effectively manage people’s expectations (ie, electronic consultation may not reduce workload to the extent originally promised). Efforts should also be made wherever possible to incorporate the factors identified as beneficial in electronic consultation implementation to achieve maximum success and impact. Finally, further work is required to explore the potential inequalities in electronic consultations, with evidence to suggest some patient groups may be disadvantaged or unintentionally excluded from electronic consultation opportunities [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. Further exploration of the economic costs and benefits of electronic consultations from the perspective of patients, professionals, and commissioners would also be beneficial in informing current debates.</p>
      </sec>
      <sec>
        <title>Conclusion</title>
        <p>In conclusion, the implementation of electronic consultations in primary care can be facilitated by the development of protocols and strategies, patient and staff education, accurate and targeted education, and meaningful patient and public involvement. Efforts should be made wherever possible to incorporate factors identified as beneficial in facilitating electronic consultations to ensure maximum impact and success. Further research exploring the economic impacts of electronic consultations would be beneficial from a patient, professional, and commissioner perspective.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">GP</term>
          <def>
            <p>general practitioner</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">IT</term>
          <def>
            <p>information technology</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">NPT</term>
          <def>
            <p>Normalization Process Theory</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This research was commissioned by NHS Kernow Clinical Commissioning Group &#38; NHS Northern, Eastern, and Western Devon Clinical Commissioning Groups to evaluate the eConsult system in Devon and Cornwall. We would like to thank NHS Kernow Clinical Commissioning Group &#38; NHS Northern, Eastern, and Western Devon Clinical Commissioning Groups for the support and feedback.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Casey</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Swinglehurst</surname>
              <given-names>D</given-names>
            </name>
          </person-group>
          <article-title>Experiences with online consultation systems in primary care: case study of one early adopter site</article-title>
          <source>Br J Gen Pract</source>
          <year>2017</year>
          <month>10</month>
          <day>09</day>
          <volume>67</volume>
          <issue>664</issue>
          <fpage>e736</fpage>
          <lpage>e743</lpage>
          <pub-id pub-id-type="doi">10.3399/bjgp17x693137</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ross</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Stevenson</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Lau</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Factors that influence the implementation of e-health: a systematic review of systematic reviews (an update)</article-title>
          <source>Implement Sci</source>
          <year>2016</year>
          <month>10</month>
          <day>26</day>
          <volume>11</volume>
          <issue>1</issue>
          <fpage>146</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.gov.uk/government/uploads/system/uploads/attachment_data/file/550866/Wachter_Review_Accessible.pdf.%20Accessed%2027%20Sept%202016"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s13012-016-0510-7</pub-id>
          <pub-id pub-id-type="medline">27782832</pub-id>
          <pub-id pub-id-type="pii">10.1186/s13012-016-0510-7</pub-id>
          <pub-id pub-id-type="pmcid">PMC5080780</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Farr</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Banks</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Edwards</surname>
              <given-names>HB</given-names>
            </name>
            <name name-style="western">
              <surname>Northstone</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Bernard</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Salisbury</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Horwood</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Implementing online consultations in primary care: a mixed-method evaluation extending normalisation process theory through service co-production</article-title>
          <source>BMJ Open</source>
          <year>2018</year>
          <month>12</month>
          <day>19</day>
          <volume>8</volume>
          <issue>3</issue>
          <fpage>e019966</fpage>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2017-019966</pub-id>
          <pub-id pub-id-type="medline">29555817</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2017-019966</pub-id>
          <pub-id pub-id-type="pmcid">PMC5875620</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Banks</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Farr</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Salisbury</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Bernard</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Northstone</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Edwards</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Horwood</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Use of an electronic consultation system in primary care: a qualitative interview study</article-title>
          <source>Br J Gen Pract</source>
          <year>2018</year>
          <month>01</month>
          <volume>68</volume>
          <issue>666</issue>
          <fpage>e1</fpage>
          <lpage>e8</lpage>
          <pub-id pub-id-type="doi">10.3399/bjgp17X693509</pub-id>
          <pub-id pub-id-type="medline">29109115</pub-id>
          <pub-id pub-id-type="pii">bjgp17X693509</pub-id>
          <pub-id pub-id-type="pmcid">PMC5737315</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Atherton</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ziebland</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>What do we need to consider when planning, implementing and researching the use of alternatives to face-to-face consultations in primary healthcare?</article-title>
          <source>Digit Health</source>
          <year>2016</year>
          <month>11</month>
          <day>21</day>
          <volume>2</volume>
          <fpage>2055207616675559</fpage>
          <pub-id pub-id-type="doi">10.1177/2055207616675559</pub-id>
          <pub-id pub-id-type="medline">29942570</pub-id>
          <pub-id pub-id-type="pii">10.1177_2055207616675559</pub-id>
          <pub-id pub-id-type="pmcid">PMC6001190</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Carter</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Fletcher</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Sansom</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Warren</surname>
              <given-names>FC</given-names>
            </name>
            <name name-style="western">
              <surname>Campbell</surname>
              <given-names>JL</given-names>
            </name>
          </person-group>
          <article-title>Feasibility, acceptability and effectiveness of an online alternative to face-to-face consultation in general practice: a mixed-methods study of webGP in six Devon practices</article-title>
          <source>BMJ Open</source>
          <year>2018</year>
          <month>02</month>
          <day>15</day>
          <volume>8</volume>
          <issue>2</issue>
          <fpage>e018688</fpage>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2017-018688</pub-id>
          <pub-id pub-id-type="medline">29449293</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2017-018688</pub-id>
          <pub-id pub-id-type="pmcid">PMC5829586</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Joschko</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Keely</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Grant</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Moroz</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Graveline</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Drimer</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Liddy</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Electronic Consultation Services Worldwide: Environmental Scan</article-title>
          <source>J Med Internet Res</source>
          <year>2018</year>
          <month>12</month>
          <day>21</day>
          <volume>20</volume>
          <issue>12</issue>
          <fpage>e11112</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://www.jmir.org/2018/12/e11112/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/11112</pub-id>
          <pub-id pub-id-type="medline">30578187</pub-id>
          <pub-id pub-id-type="pii">v20i12e11112</pub-id>
          <pub-id pub-id-type="pmcid">PMC6320413</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Marshall</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Shah</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Stokes-Lampard</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Online consulting in general practice: making the move from disruptive innovation to mainstream service</article-title>
          <source>BMJ</source>
          <year>2018</year>
          <month>03</month>
          <day>26</day>
          <volume>360</volume>
          <fpage>k1195</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.k1195</pub-id>
          <pub-id pub-id-type="medline">29581174</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Johnston</surname>
              <given-names>DL</given-names>
            </name>
            <name name-style="western">
              <surname>Murto</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kurzawa</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Liddy</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Keely</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Lai</surname>
              <given-names>L</given-names>
            </name>
          </person-group>
          <article-title>Use of Electronic Consultation System to Improve Access to Care in Pediatric Hematology/Oncology</article-title>
          <source>J Pediatr Hematol Oncol</source>
          <year>2017</year>
          <month>10</month>
          <volume>39</volume>
          <issue>7</issue>
          <fpage>e367</fpage>
          <lpage>e369</lpage>
          <pub-id pub-id-type="doi">10.1097/MPH.0000000000000833</pub-id>
          <pub-id pub-id-type="medline">28437292</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wise</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Electronic consultations offer few benefits for GP practices, says study</article-title>
          <source>BMJ</source>
          <year>2017</year>
          <month>11</month>
          <day>06</day>
          <fpage>j5141</fpage>
          <pub-id pub-id-type="doi">10.1136/bmj.j5141</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cowie</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Calveley</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Bowers</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Bowers</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of a Digital Consultation and Self-Care Advice Tool in Primary Care: A Multi-Methods Study</article-title>
          <source>Int J Environ Res Public Health</source>
          <year>2018</year>
          <month>05</month>
          <day>02</day>
          <volume>15</volume>
          <issue>5</issue>
          <fpage>896</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=ijerph15050896"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/ijerph15050896</pub-id>
          <pub-id pub-id-type="medline">29724040</pub-id>
          <pub-id pub-id-type="pii">ijerph15050896</pub-id>
          <pub-id pub-id-type="pmcid">PMC5981935</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="web">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Liddy</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Keely</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Afkham</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Faster Access, Better Care: A Guide to Establishing an eConsult Service</article-title>
          <year>2016</year>
          <access-date>2020-10-28</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://ruor.uottawa.ca/bitstream/10393/34968/1/Faster%20Access%2c%20Better%20Care.pdf">https://ruor.uottawa.ca/bitstream/10393/34968/1/Faster%20Access%2c%20Better%20Care.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Anderson</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Villagra</surname>
              <given-names>VG</given-names>
            </name>
            <name name-style="western">
              <surname>Coman</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Ahmed</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Porto</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Jepeal</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Maci</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Teevan</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Reduced Cost Of Specialty Care Using Electronic Consultations For Medicaid Patients</article-title>
          <source>Health Aff (Millwood)</source>
          <year>2018</year>
          <month>12</month>
          <volume>37</volume>
          <issue>12</issue>
          <fpage>2031</fpage>
          <lpage>2036</lpage>
          <pub-id pub-id-type="doi">10.1377/hlthaff.2018.05124</pub-id>
          <pub-id pub-id-type="medline">30633678</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Anderson</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Villagra</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Coman</surname>
              <given-names>EN</given-names>
            </name>
            <name name-style="western">
              <surname>Zlateva</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Hutchinson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Villagra</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Olayiwola</surname>
              <given-names>JN</given-names>
            </name>
          </person-group>
          <article-title>A cost-effectiveness analysis of cardiology eConsults for Medicaid patients</article-title>
          <source>Am J Manag Care</source>
          <year>2018</year>
          <month>01</month>
          <day>01</day>
          <volume>24</volume>
          <issue>1</issue>
          <fpage>e9</fpage>
          <lpage>e16</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.ajmc.com/pubMed.php?pii=87417"/>
          </comment>
          <pub-id pub-id-type="medline">29350511</pub-id>
          <pub-id pub-id-type="pii">87417</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hanna</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>May</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Fairhurst</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>The place of information and communication technology-mediated consultations in primary care: GPs' perspectives</article-title>
          <source>Fam Pract</source>
          <year>2012</year>
          <month>06</month>
          <volume>29</volume>
          <issue>3</issue>
          <fpage>361</fpage>
          <lpage>6</lpage>
          <pub-id pub-id-type="doi">10.1093/fampra/cmr087</pub-id>
          <pub-id pub-id-type="medline">22006040</pub-id>
          <pub-id pub-id-type="pii">cmr087</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Edwards</surname>
              <given-names>HB</given-names>
            </name>
            <name name-style="western">
              <surname>Marques</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hollingworth</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Horwood</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Farr</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Bernard</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Salisbury</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Northstone</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Use of a primary care online consultation system, by whom, when and why: evaluation of a pilot observational study in 36 general practices in South West England</article-title>
          <source>BMJ Open</source>
          <year>2017</year>
          <month>11</month>
          <day>22</day>
          <volume>7</volume>
          <issue>11</issue>
          <fpage>e016901</fpage>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2017-016901</pub-id>
          <pub-id pub-id-type="medline">29167106</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2017-016901</pub-id>
          <pub-id pub-id-type="pmcid">PMC5701981</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Osman</surname>
              <given-names>MA</given-names>
            </name>
            <name name-style="western">
              <surname>Schick-Makaroff</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Thompson</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Featherstone</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Bialy</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Kurzawa</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Okpechi</surname>
              <given-names>IG</given-names>
            </name>
            <name name-style="western">
              <surname>Habib</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Shojai</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Jindal</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Klarenbach</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bello</surname>
              <given-names>AK</given-names>
            </name>
          </person-group>
          <article-title>Barriers and facilitators for implementation of electronic consultations (eConsult) to enhance specialist access to care: a scoping review protocol</article-title>
          <source>BMJ Open</source>
          <year>2018</year>
          <month>09</month>
          <day>28</day>
          <volume>8</volume>
          <issue>9</issue>
          <fpage>e022733</fpage>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2018-022733</pub-id>
          <pub-id pub-id-type="medline">30269069</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2018-022733</pub-id>
          <pub-id pub-id-type="pmcid">PMC6169744</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tricco</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Lillie</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Zarin</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>O'Brien</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Colquhoun</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kastner</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Levac</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Ng</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Sharpe</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kenny</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Warren</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Wilson</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Stelfox</surname>
              <given-names>HT</given-names>
            </name>
            <name name-style="western">
              <surname>Straus</surname>
              <given-names>SE</given-names>
            </name>
          </person-group>
          <article-title>A scoping review on the conduct and reporting of scoping reviews</article-title>
          <source>BMC Med Res Methodol</source>
          <year>2016</year>
          <month>02</month>
          <day>09</day>
          <volume>16</volume>
          <fpage>15</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedresmethodol.biomedcentral.com/articles/10.1186/s12874-016-0116-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12874-016-0116-4</pub-id>
          <pub-id pub-id-type="medline">26857112</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12874-016-0116-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC4746911</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Sampson</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>McGowan</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Cogo</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Grimshaw</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Moher</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Lefebvre</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>An evidence-based practice guideline for the peer review of electronic search strategies</article-title>
          <source>J Clin Epidemiol</source>
          <year>2009</year>
          <month>09</month>
          <volume>62</volume>
          <issue>9</issue>
          <fpage>944</fpage>
          <lpage>52</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jclinepi.2008.10.012</pub-id>
          <pub-id pub-id-type="medline">19230612</pub-id>
          <pub-id pub-id-type="pii">S0895-4356(08)00320-X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <collab>GreyNet</collab>
          </person-group>
          <article-title>4th International Conference on Grey Literature, New Frontiers in Grey Literature, 4-5 October 1999, Washington DC</article-title>
          <source>Asian Libraries</source>
          <year>1999</year>
          <month>07</month>
          <volume>8</volume>
          <issue>7</issue>
          <fpage>1</fpage>
          <pub-id pub-id-type="doi">10.1108/al.1999.17308gab.010</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="web">
          <source>Rayyan - a web and mobile app for systematic reviews</source>
          <access-date>2020-10-28</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://rayyan.qcri.org/">https://rayyan.qcri.org/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Braun</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Clarke</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>Using thematic analysis in psychology</article-title>
          <source>Qualitative Research in Psychology</source>
          <year>2006</year>
          <month>01</month>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>77</fpage>
          <lpage>101</lpage>
          <pub-id pub-id-type="doi">10.1191/1478088706qp063oa</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Treweek</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Pope</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>MacFarlane</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Ballini</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Dowrick</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Finch</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Kennedy</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mair</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>O'Donnell</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ong</surname>
              <given-names>BN</given-names>
            </name>
            <name name-style="western">
              <surname>Rapley</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Rogers</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>May</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Normalisation process theory: a framework for developing, evaluating and implementing complex interventions</article-title>
          <source>BMC Med</source>
          <year>2010</year>
          <month>10</month>
          <day>20</day>
          <volume>8</volume>
          <fpage>63</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmcmedicine.biomedcentral.com/articles/10.1186/1741-7015-8-63"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/1741-7015-8-63</pub-id>
          <pub-id pub-id-type="medline">20961442</pub-id>
          <pub-id pub-id-type="pii">1741-7015-8-63</pub-id>
          <pub-id pub-id-type="pmcid">PMC2978112</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Atherton</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Brant</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Ziebland</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Bikker</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Campbell</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Gibson</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>McKinstry</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Porqueddu</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Salisbury</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>The potential of alternatives to face-to-face consultation in general practice, and the impact on different patient groups: a mixed-methods case study</article-title>
          <source>Health Serv Deliv Res</source>
          <year>2018</year>
          <month>6</month>
          <volume>6</volume>
          <issue>20</issue>
          <fpage>1</fpage>
          <lpage>200</lpage>
          <pub-id pub-id-type="doi">10.3310/hsdr06200</pub-id>
          <pub-id pub-id-type="medline">29889485</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Cowie</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Calveley</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Bowers</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Bowers</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of a Digital Consultation and Self-Care Advice Tool in Primary Care: A Multi-Methods Study</article-title>
          <source>Int J Environ Res Public Health</source>
          <year>2018</year>
          <month>05</month>
          <day>02</day>
          <volume>15</volume>
          <issue>5</issue>
          <fpage>896</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=ijerph15050896"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/ijerph15050896</pub-id>
          <pub-id pub-id-type="medline">29724040</pub-id>
          <pub-id pub-id-type="pii">ijerph15050896</pub-id>
          <pub-id pub-id-type="pmcid">PMC5981935</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ogden</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Online consultations: the future of NHS GP appointments?</article-title>
          <source>Prescriber</source>
          <year>2018</year>
          <month>09</month>
          <day>19</day>
          <volume>29</volume>
          <issue>9</issue>
          <fpage>27</fpage>
          <lpage>30</lpage>
          <pub-id pub-id-type="doi">10.1002/psb.1703</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hanna</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>May</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Fairhurst</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>The place of information and communication technology-mediated consultations in primary care: GPs' perspectives</article-title>
          <source>Fam Pract</source>
          <year>2012</year>
          <month>06</month>
          <day>17</day>
          <volume>29</volume>
          <issue>3</issue>
          <fpage>361</fpage>
          <lpage>6</lpage>
          <pub-id pub-id-type="doi">10.1093/fampra/cmr087</pub-id>
          <pub-id pub-id-type="medline">22006040</pub-id>
          <pub-id pub-id-type="pii">cmr087</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Mair</surname>
              <given-names>FS</given-names>
            </name>
            <name name-style="western">
              <surname>May</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>O'Donnell</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Finch</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Sullivan</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Factors that promote or inhibit the implementation of e-health systems: an explanatory systematic review</article-title>
          <source>Bull World Health Organ</source>
          <year>2012</year>
          <month>05</month>
          <day>1</day>
          <volume>90</volume>
          <issue>5</issue>
          <fpage>357</fpage>
          <lpage>64</lpage>
          <pub-id pub-id-type="doi">10.2471/BLT.11.099424</pub-id>
          <pub-id pub-id-type="medline">22589569</pub-id>
          <pub-id pub-id-type="pii">BLT.11.099424</pub-id>
          <pub-id pub-id-type="pmcid">PMC3341692</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hanna</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>May</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Fairhurst</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Non-face-to-face consultations and communications in primary care: the role and perspective of general practice managers in Scotland</article-title>
          <source>Inform Prim Care</source>
          <year>2011</year>
          <volume>19</volume>
          <issue>1</issue>
          <fpage>17</fpage>
          <lpage>24</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://hijournal.bcs.org/index.php/jhi/article/view/789"/>
          </comment>
          <pub-id pub-id-type="doi">10.14236/jhi.v19i1.789</pub-id>
          <pub-id pub-id-type="medline">22118332</pub-id>
          <pub-id pub-id-type="pii">789</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
