<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v24i6e31921</article-id>
      <article-id pub-id-type="pmid">35763320</article-id>
      <article-id pub-id-type="doi">10.2196/31921</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Viewpoint</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Viewpoint</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>What are Digital Public Health Interventions? First Steps Toward a Definition and an Intervention Classification Framework</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Mavragani</surname>
            <given-names>Amaryllis</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Gupta</surname>
            <given-names>Latika</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Prietula</surname>
            <given-names>Michael</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Janssen</surname>
            <given-names>Anna</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Wienert</surname>
            <given-names>Julian</given-names>
          </name>
          <degrees>BA, MA, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff2" ref-type="aff">2</xref>
          <xref rid="aff3" ref-type="aff">3</xref>
          <address>
            <institution>Health Psychology</institution>
            <institution>Social Sciences</institution>
            <institution>IU International University for Applied Sciences</institution>
            <addr-line>Kaiserplatz 1</addr-line>
            <addr-line>Bad Reichenhall, 83435</addr-line>
            <country>Germany</country>
            <phone>49 1725858645</phone>
            <email>julian.wienert@iu.org</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1246-7591</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author">
          <name name-style="western">
            <surname>Jahnel</surname>
            <given-names>Tina</given-names>
          </name>
          <degrees>BA, MA, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff4" ref-type="aff">4</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-8367-4574</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Maaß</surname>
            <given-names>Laura</given-names>
          </name>
          <degrees>BA, MA</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff5" ref-type="aff">5</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0001-7354-8120</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Research Cluster Evaluation</institution>
        <institution>Leibniz ScienceCampus Bremen Digital Public Health</institution>
        <addr-line>Bremen</addr-line>
        <country>Germany</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Social Epidemiology</institution>
        <institution>Prevention and Evaluation</institution>
        <institution>Leibniz Institute for Prevention Research and Epidemiology – BIPS</institution>
        <addr-line>Bremen</addr-line>
        <country>Germany</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Health Psychology</institution>
        <institution>Social Sciences</institution>
        <institution>IU International University for Applied Sciences</institution>
        <addr-line>Bad Reichenhall</addr-line>
        <country>Germany</country>
      </aff>
      <aff id="aff4">
        <label>4</label>
        <institution>Department of Health Services Research</institution>
        <institution>Institute for Public Health and Nursing</institution>
        <institution>University of Bremen</institution>
        <addr-line>Bremen</addr-line>
        <country>Germany</country>
      </aff>
      <aff id="aff5">
        <label>5</label>
        <institution>Health, Long-Term Care and Pensions</institution>
        <institution>Research Center on Inequality and Social Policy (SOCIUM)</institution>
        <addr-line>Bremen</addr-line>
        <country>Germany</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Julian Wienert <email>julian.wienert@iu.org</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>6</month>
        <year>2022</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>28</day>
        <month>6</month>
        <year>2022</year>
      </pub-date>
      <volume>24</volume>
      <issue>6</issue>
      <elocation-id>e31921</elocation-id>
      <history>
        <date date-type="received">
          <day>9</day>
          <month>7</month>
          <year>2021</year>
        </date>
        <date date-type="rev-request">
          <day>21</day>
          <month>12</month>
          <year>2021</year>
        </date>
        <date date-type="rev-recd">
          <day>4</day>
          <month>1</month>
          <year>2022</year>
        </date>
        <date date-type="accepted">
          <day>4</day>
          <month>1</month>
          <year>2022</year>
        </date>
      </history>
      <copyright-statement>©Julian Wienert, Tina Jahnel, Laura Maaß. Originally published in the Journal of Medical Internet Research (https://www.jmir.org), 28.06.2022.</copyright-statement>
      <copyright-year>2022</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on https://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="https://www.jmir.org/2022/6/e31921" xlink:type="simple"/>
      <abstract>
        <p>Digital public health is an emerging field in population-based research and practice. The fast development of digital technologies provides a fundamentally new understanding of improving public health by using digitalization, especially in prevention and health promotion. The first step toward a better understanding of digital public health is to conceptualize the subject of the assessment by defining what digital public health interventions are. This is important, as one cannot evaluate tools if one does not know what precisely an intervention in this field can be. Therefore, this study aims to provide the first definition of digital public health interventions. We will merge leading models for public health functions by the World Health Organization, a framework for digital health technologies by the National Institute for Health and Care Excellence, and a user-centered approach to intervention development. Together, they provide an overview of the functions and areas of use for digital public health interventions. Nevertheless, one must keep in mind that public health functions can differ among different health care systems, limiting our new framework’s universal validity. We conclude that a digital public health intervention should address essential public health functions through digital means. Furthermore, it should include members of the target group in the development process to improve social acceptance and achieve a population health impact.</p>
      </abstract>
      <kwd-group>
        <kwd>digital health</kwd>
        <kwd>digital Public Health</kwd>
        <kwd>digital public health interventions</kwd>
        <kwd>digital health technologies</kwd>
        <kwd>mHealth</kwd>
        <kwd>eHealth</kwd>
        <kwd>participatory approach</kwd>
        <kwd>framework</kwd>
        <kwd>mobile phone</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <sec>
        <title>Background</title>
        <p>As digitization plays a large role in an increasing number of health systems, digital public health is an emerging field for population-based research and practice. The fast development of both hardware- and software-based digital technologies provides a fundamentally new understanding of improving public health, which can be achieved through digitalization, especially in prevention and health promotion. For example, digital technologies may improve physical activity levels, dietary intake, posture, and mental well-being via sensors and apps [<xref ref-type="bibr" rid="ref1">1</xref>]. Technological innovations in apps for tracking health-related behavior, monitoring potential health risks, and communication and interaction have rapidly changed many aspects of public health [<xref ref-type="bibr" rid="ref2">2</xref>]. However, not all of these interventions might achieve a health impact at a population level by displaying effectiveness in randomized clinical trials and efficacy under quasi-experimental real-world circumstances.</p>
        <p>Although there is a need for evaluation methods that address the many challenges that arise with digitization (eg, fast-paced development), it is challenging to assess digital public health interventions as these may span from population health surveillance to the prevention of specific diseases, and they develop faster than analog interventions [<xref ref-type="bibr" rid="ref3">3</xref>]. Moreover, companies and institutions often develop digital tools based on market evaluations, expected profits, and technological possibilities but not based on the public’s needs and preferences. To improve digital public health interventions’ effectiveness and efficacy, we first need to understand what they entail and how they are defined. However, to our knowledge, no definition for digital public health interventions exists to date. Only by doing that will we gather meaningful, valid, and reliable results on their effectiveness and efficiency. Thus, the aim of this viewpoint is to offer a definition for digital public health interventions.</p>
        <p>Before defining digital public health interventions, we need to explain the differences between eHealth, mobile health (mHealth), digital health, and digital public health. This is necessary to highlight the differences between digital health interventions (DHIs) and digital public health interventions. Following this, we will build a framework that might help to identify, structure, and classify digital public health interventions. Our definition and framework will rely on existing approaches from public health [<xref ref-type="bibr" rid="ref4">4</xref>], digital health [<xref ref-type="bibr" rid="ref5">5</xref>], and user-centered design [<xref ref-type="bibr" rid="ref6">6</xref>]. The first section of building the framework will explain why we chose the selected models; we decided to use the Essential Public Health Functions (EPHFs) by the World Health Organization (WHO) for the public health level [<xref ref-type="bibr" rid="ref4">4</xref>], the updated version of the <italic>Evidence Standards Framework for Digital Health Technologies</italic> by the National Institute for Health and Care Excellence (NICE) [<xref ref-type="bibr" rid="ref5">5</xref>], and the <italic>Participatory Health Research</italic> approach by Wright [<xref ref-type="bibr" rid="ref6">6</xref>]. After clarifying the reasons for choosing named approaches, we will explain each model and how they are related to digital public health and may be used for digital public health purposes. After setting the theoretical background for our definition, we will illustrate our findings using the German Corona App as an example to validate our digital public health intervention criteria. We will conclude with a definition for digital public health interventions and use this to propose a digital public health intervention classification framework (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>).</p>
      </sec>
      <sec>
        <title>Differences Among eHealth, mHealth, Digital Health, and Digital Public Health</title>
        <p>Terms such as eHealth, mHealth, or digital health are used in the context of the digitization of public health. Since 2019, few papers have also referred to the term digital public health. Given the multitude of terms and definitions in digital health, it is essential to understand the considerable heterogeneity of how such terms interrelate with each other and where digital public health might find its place in the terminological canon of digital health. Therefore, the following section will define the named terms and summarize their core fields of action and the target group’s level, as seen in <xref rid="figure1" ref-type="fig">Figure 1</xref>.</p>
        <fig id="figure1" position="float">
          <label>Figure 1</label>
          <caption>
            <p>Core field of action and target group level of mHealth, eHealth, digital health, and digital public health [<xref ref-type="bibr" rid="ref7">7</xref>-<xref ref-type="bibr" rid="ref15">15</xref>]. mHealth: mobile health.</p>
          </caption>
          <graphic xlink:href="jmir_v24i6e31921_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
        <p>An article on <italic>eHealth</italic> concepts based on an extensive literature search [<xref ref-type="bibr" rid="ref7">7</xref>] confirms the lack of consensus on the meaning of <italic>eHealth</italic> as possibly the first word in this field. A 2005 study found 51 different definitions of <italic>eHealth</italic> [<xref ref-type="bibr" rid="ref8">8</xref>]. This lack of consensus highlights the importance of a shared understanding of terms. More recent studies emphasize that because of its immense dynamics, the field of <italic>eHealth</italic> is challenging to define [<xref ref-type="bibr" rid="ref9">9</xref>]. Most definitions share the use of information and communication technologies (eg, the internet) for health topics. Their focus mostly lies on delivering health services rather than health promotion and disease prevention [<xref ref-type="bibr" rid="ref10">10</xref>-<xref ref-type="bibr" rid="ref12">12</xref>]. Some definitions also highlight the importance of user-centered approaches for facilitating health services in <italic>eHealth</italic> [<xref ref-type="bibr" rid="ref10">10</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. The word <italic>mHealth</italic> aims more directly at a particular technology, namely smartphones and mobile sensors, in their health significance. Thus, <italic>mHealth</italic> is defined more precisely overall, although different technologies are used here [<xref ref-type="bibr" rid="ref13">13</xref>]. As a part of <italic>eHealth</italic>, the focus of <italic>mHealth</italic> lies on wireless and mobile technologies and their use in enhancing health-related science, treatment, and ultimately health status [<xref ref-type="bibr" rid="ref9">9</xref>].</p>
        <p>Fatehi et al [<xref ref-type="bibr" rid="ref14">14</xref>] stated that in 2020, there were &#62;90 different definitions for <italic>digital health</italic>. They concluded that <italic>digital health</italic> includes eHealth, mHealth, self-tracking, wearable devices, artificial intelligence, and information systems in health care, focusing on health and not technology. <italic>Digital health</italic> focuses on the health of individuals (eg, patients) to improve health care with technology [<xref ref-type="bibr" rid="ref14">14</xref>]. This is where <italic>digital health</italic> and digital public health differ, as digital public health aims to improve health and well-being at the population level. Nevertheless, digital public health uses the same technologies that are also used to improve individual health care; however, its purposes change. A recent publication by Zeeb et al [<xref ref-type="bibr" rid="ref15">15</xref>] provides the first overview of what digital public health might be. It serves as a starting point for developing a better understanding of digital public health interventions by providing a short introduction to central terms. Thus<italic>,</italic> following the article by Zeeb et al [<xref ref-type="bibr" rid="ref15">15</xref>], the authors propose the following definition for digital public health in distinction to the other abovementioned fields and outline for which fields they see it relevant (own translation from German):</p>
        <disp-quote>
          <p>DiPH [...] focuses on the development, application, and knowledge interest on Public Health and thus on prevention, health promotion, and the related basic sciences such as epidemiology. Primary clinical and individual patient-related aspects are not in the foreground, unlike, for example, telemedicine with its concrete application in an individual treatment and care context. However, it should be noted that the term DiPH has not yet prevailed over others such as eHealth and mHealth. Also, this is hard to expect given the diversity and dynamics of the terms used to date. Where, however, the focus of digitization and health is on population, prevention, and health promotion, including a conscious analysis of health inequalities, DiPH can offer a clearer classification than some other terms in this field.</p>
        </disp-quote>
        <p>Although the definition by Zeeb et al [<xref ref-type="bibr" rid="ref15">15</xref>] serves as a good starting point for the discussion, it mainly focuses on the primary level of prevention (ie, preventing a disease or injury before it occurs). Although public health, in its essence, comprises 3 levels of prevention, according to this definition, secondary (eg, reducing the impact of a disease or injury after it occurred) and tertiary preventions (eg, rehabilitation) are not explicitly mentioned by Zeeb et al [<xref ref-type="bibr" rid="ref15">15</xref>] as part of digital public health. The central challenge of defining digital public health is the integration of digital development and technologies into public health concepts and use them to achieve public health goals rather than redefining and reconceptualizing public health in the face of technological advancements [<xref ref-type="bibr" rid="ref5">5</xref>,<xref ref-type="bibr" rid="ref15">15</xref>].</p>
        <p>To develop a first working definition and classification framework for digital public health interventions, established models for both digital health [<xref ref-type="bibr" rid="ref5">5</xref>] and public health [<xref ref-type="bibr" rid="ref4">4</xref>] were assessed to combine aspects of these models to develop a more holistic definition of digital public health interventions. Here we suggest that digital public health, as a complex intervention, should be viewed from different perspectives. As such, our proposed classification is a combination of the elements of already existing models. Specifically, the EPHF by the WHO provided us with an overview of the necessary core functions of public health, which might also be addressed by digital means. Our definition will explicitly include the area of health promotion (ie, focusing on health resources) and all three levels of prevention (ie, primary prevention to reduce the risk of disease development, secondary prevention as screening and early diagnosis, and tertiary prevention for rehabilitation), as all these levels are included in the EPHF by the WHO [<xref ref-type="bibr" rid="ref4">4</xref>]. We will then link our definition to the participation approach: a user-centered model for the development of digital interventions to increase the acceptance of digital public health interventions among target groups [<xref ref-type="bibr" rid="ref6">6</xref>]. This will create a framework that follows the concepts of public health (goals).</p>
      </sec>
      <sec>
        <title>Choice of Included Models</title>
        <p>Following a narrative approach and based on the authors’ expertise and experience in the field of Public Health, three layers were identified: (1) overview, which is a larger operational layer where the central functions of digital public health are mapped; (2) structure, which is a layer that focuses on structuring digital public health activities (eg, by functions); and (3) improvement, which is a layer that specifically includes the individual perspective in the development and use of digital public health interventions. For each layer, a framework was identified based on the author’s expertise and previous experience with the frameworks and a nonsystematic literature search for alternative frameworks. The frameworks included are as follows: the EPHF by the WHO, which offers a macro view of public health topics [<xref ref-type="bibr" rid="ref4">4</xref>]; the <italic>Evidence Standards Framework for Digital Health Technologies</italic> by the NICE, which categorizes DHIs for the UK setting [<xref ref-type="bibr" rid="ref5">5</xref>]; and the <italic>Steps of Participation Approach</italic>, which was suggested by Wright [<xref ref-type="bibr" rid="ref6">6</xref>].</p>
        <p>Together, the EPHF and the NICE framework will build the base of mapping and structuring digital public health interventions. We then use the <italic>Steps of Participation Approach</italic> suggested by Wright [<xref ref-type="bibr" rid="ref6">6</xref>]. This is a user-centered approach for intervention development that aims to increase acceptance within the target group. The approach provides well-described and clear-cut categories for target group involvement—participation and nonparticipation alike. Together, with all 3 models aligned, a conceptual pyramid for digital public health intervention classification is formed (<xref rid="figure2" ref-type="fig">Figure 2</xref>).</p>
        <fig id="figure2" position="float">
          <label>Figure 2</label>
          <caption>
            <p>Conceptual pyramid for a framework of digital public health interventions [<xref ref-type="bibr" rid="ref4">4</xref>-<xref ref-type="bibr" rid="ref6">6</xref>]. NICE: National Institute for Health and Care Excellence; WHO: World Health Organization.</p>
          </caption>
          <graphic xlink:href="jmir_v24i6e31921_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
      <sec>
        <title>EPHFs by the WHO</title>
        <p>A way of addressing public health goals to affect population health is using <italic>the EPHF</italic> [<xref ref-type="bibr" rid="ref4">4</xref>]. Following the WHO report <italic>EPHFs, health systems and health security: developing conceptual clarity and a WHO road map for action,</italic> these functions can be separated into cross-cutting, horizontal functions, roughly based on the building blocks approach to health systems, and service-based, vertical functions comprising the traditional public health services provided by modern health systems [<xref ref-type="bibr" rid="ref4">4</xref>]. Although there is no precise definition for each part, as they depend on each health care system or region, the WHO report identified some significant categories that most EPHF share (<xref ref-type="boxed-text" rid="box1">Textbox 1</xref>).</p>
        <boxed-text id="box1" position="float">
          <title>Essential public health functions according to the World Health Organization [<xref ref-type="bibr" rid="ref4">4</xref>].</title>
          <p>
            <bold>Essential public health functions</bold>
          </p>
          <p>
            <bold>Horizontal functions</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>Governance (eg, public health management, policy, and planning or quality assurance in health services)</p>
            </list-item>
            <list-item>
              <p>Financing: establishment of sustainable organizational structures, institutional capacity, and policy making</p>
            </list-item>
            <list-item>
              <p>Human resources: development and management of human resources</p>
            </list-item>
            <list-item>
              <p>Health information systems: population health surveillance and monitoring</p>
            </list-item>
            <list-item>
              <p>Research: development of a national public health research agenda, allocation of resources for research, integration of research activities into public health, capacity building for innovation, and dissemination to translate research findings into policy and practice</p>
            </list-item>
            <list-item>
              <p>Social participation and health communication: social participation, community partnership, community engagement and/or (digital) public health communication, and design public health services around people’s needs</p>
            </list-item>
          </list>
          <p>
            <bold>Vertical functions</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>Health protection: regulations and legal protections (for workers, patients, consumers, and the environment)</p>
            </list-item>
            <list-item>
              <p>Health promotion: community and social participation, intersectoral collaboration, measures to address behavioral risk factors (tobacco, alcohol, diet, and physical activity), and the social determinants of health and health education</p>
            </list-item>
            <list-item>
              <p>Disease prevention: services provided within the health care system and targeting communicable diseases</p>
            </list-item>
            <list-item>
              <p>Health care: With specific functions for quality assurance and access, universal health coverage is a defining feature depending on the World Health Organization region or country (ie, the European region emphatically excludes most health care services from the public health remit because of strong roots in the principles of universal access as opposed to the United States or Western Pacific regions).</p>
            </list-item>
            <list-item>
              <p>Preparedness for public health emergencies: encompass any sudden, large-scale, negative impact on public health arising from outbreaks, natural disasters, severe weather events, migratory flows, accidents, terrorism, or other environmental or human causes</p>
            </list-item>
            <list-item>
              <p>Other vertical functions: A wide variety of specific vertical functions are given importance in different countries. In part, this reflects an overall approach to developing frameworks that list essential services rather than broader functions per se. At the same time, the vertical positions were chosen to reflect national priorities.</p>
            </list-item>
          </list>
        </boxed-text>
        <p>These are just a few examples of fields for action in both (analog) public health interventions and digital public health interventions. However, we could not apply all EPHF to every setting. They depend strongly on specific health care systems, which differ among countries. In general, the WHO regards public health interventions primarily as an effort or policy that attempts to improve mental, social, and physical health at the population level by including and addressing EPHF [<xref ref-type="bibr" rid="ref4">4</xref>].</p>
        <p>Analogous to public health interventions, digital public health interventions have the potential to include and address horizontal as well as vertical functions. The governmental regulation of mHealth apps as medical devices with the possibility of reimbursement, as started in Germany in December 2020 [<xref ref-type="bibr" rid="ref16">16</xref>], may be one way of applying horizontal EPHF of governance to digital public health. Various countries have also developed proximity-tracing apps as tools for population health surveillance and monitoring during the SARS-CoV-2 pandemic [<xref ref-type="bibr" rid="ref17">17</xref>,<xref ref-type="bibr" rid="ref18">18</xref>]. The last example of applying the horizontal EPHF to digital public health is the digitization of health care systems in total. This leads to a redesign around people’s needs and expectations in, for instance, web-based consultation services or telemedicine for people in rural areas who do not have access to health care professionals [<xref ref-type="bibr" rid="ref19">19</xref>]. As for vertical public health functions, apps and wearables for self-monitoring, step counting, and fitness tracking can serve as examples of vertical digital public health functions. Their goal is to promote health and a healthy lifestyle [<xref ref-type="bibr" rid="ref20">20</xref>,<xref ref-type="bibr" rid="ref21">21</xref>].</p>
      </sec>
      <sec>
        <title>Level of Interaction: NICE Framework</title>
        <p>Applying EPHF as a cornerstone for the identification and mapping of digital public health interventions provides an initial overview of the field of digital public health. The next step is to further structure such interventions based on their functional classification proposed by NICE’s <italic>Evidence Standards Framework for Digital Health Technologies</italic> [<xref ref-type="bibr" rid="ref5">5</xref>]. This applied framework describes the types and levels of evidence needed to show the effectiveness and expected economic impact of a DHI. Various publications have used this framework for their digital health technology assessment, which confirmed our resolution that this framework is not only well-known but also well-used in the scientific field of digital health [<xref ref-type="bibr" rid="ref22">22</xref>-<xref ref-type="bibr" rid="ref24">24</xref>]. The NICE framework aims to establish standardized criteria that can assess DHIs by providing a functional classification and stratification into evidence tiers. This separation illustrates the main functions of the types of interventions that we expect to be the most widely developed (<xref ref-type="boxed-text" rid="box2">Textbox 2</xref>).</p>
        <boxed-text id="box2" position="float">
          <title>Functional classification and stratification into evidence tiers [<xref ref-type="bibr" rid="ref5">5</xref>].</title>
          <p>
            <bold>Stratification into evidence tiers and functional classification</bold>
          </p>
          <p>
            <bold>Evidence tier C: interventions</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>Preventive behavior change: address public health issues (eg, smoking, eating, alcohol, sexual health, sleeping, and exercise)</p>
            </list-item>
            <list-item>
              <p>Self-manage: allows people to self-manage a specific condition; may include behavior change techniques</p>
            </list-item>
            <list-item>
              <p>Treat: provides treatment and guides treatment</p>
            </list-item>
            <list-item>
              <p>Active monitoring: tracking patient location, using wearables to measure, record, and/or transmit data about a specific condition</p>
            </list-item>
            <list-item>
              <p>Calculate: a calculator that affects treatment, diagnosis, or care</p>
            </list-item>
            <list-item>
              <p>Diagnose: diagnose a specific condition; guides diagnosis</p>
            </list-item>
          </list>
          <p>
            <bold>Evidence tier B: understanding and communicating</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>Inform: provides information (about a condition or general health and lifestyle), resources, or activities to the public, patients, or clinicians</p>
            </list-item>
            <list-item>
              <p>Simple monitoring: includes general health monitoring using fitness wearables and simple symptom diaries</p>
            </list-item>
            <list-item>
              <p>Communicate: allows 2-way communication among citizens, patients, or health care professionals</p>
            </list-item>
          </list>
          <p>
            <bold>Evidence tier A: system impact</bold>
          </p>
          <list list-type="bullet">
            <list-item>
              <p>System service: digital health interventions with no measurable patient outcomes but which provide services to the health and social care system</p>
            </list-item>
          </list>
        </boxed-text>
        <p>The abovementioned evidence tiers serve as concrete examples in digital health for the EPHF. For instance, level A refers to the vertical EPHF <italic>health care</italic> (digital public health tools in this field could be electronic health records). In contrast, the 3 functions in evidence tier B belong to the horizontal public health function <italic>social participation and health communication</italic>. Recent examples for tier B are proximity-tracing apps (sometimes called contact-tracing apps), which various countries use in epidemic or pandemic outbreaks such as SARS-CoV-2. Such apps usually display level 2 functions (ie, informing, simple monitoring, and communication), which mirrors the underlying EPHF, including disease prevention and health information systems as underlying EPHF. The first 3 functions within evidence tier C serve as a digital example for the vertical functions of <italic>health promotion</italic> as well as <italic>disease prevention</italic> (such as mobile apps on prescription [<xref ref-type="bibr" rid="ref16">16</xref>]). Finally, the last 3 functions in tier C, although focusing more on the medical and individual level than the other tiers and functions, can be seen as a part of <italic>health promotion</italic> and <italic>disease prevention</italic>. Unlike the first 3 functions in tier C, which focus more on the primary prevention area, the last 3 functions are more closely linked to secondary and tertiary prevention. Specifically, the functions of DHIs in tier C include the early diagnosis of specific conditions and rehabilitation and healing, which improves the user’s health (for instance, a national telemedicine service [<xref ref-type="bibr" rid="ref19">19</xref>]).</p>
        <p>As seen, there is an interrelation between the NICE framework and EPHF, supporting the argument that digital public health interventions can address EPHF. The critical part here is that the NICE framework, unlike the WHO EPHF, provides a structure for the degree of complexity (ie, level of interaction) based on the user’s risk. Following the understanding of complex and multicomponent interventions that act and interact on different levels, benefits, and acceptance of digital public health interventions, depending on the users of such interventions and their specific perspectives. Any digital public health intervention can ultimately fail if the population does not accept or use it. Thus, it is essential to involve target groups in the development of these interventions. We propose a participatory and user-centered approach for intervention development as the third cornerstone of digital public health interventions.</p>
      </sec>
      <sec>
        <title>User-Centered Approach in Intervention Development</title>
        <p>Hochmuth et al [<xref ref-type="bibr" rid="ref25">25</xref>] advocated that complex and multicomponent interventions require a user-oriented intervention design because of the varying intricacies of such interventions. This intricacy can be based on the following:</p>
        <list list-type="bullet">
          <list-item>
            <p>Interactions between technological components (eg, sensors for data acquisition)</p>
          </list-item>
          <list-item>
            <p>Different requirements for users in the implementation of the intervention (eg, knowledge of data security)</p>
          </list-item>
          <list-item>
            <p>Involvement of other groups or organizational levels (eg, patients or researchers)</p>
          </list-item>
          <list-item>
            <p>Degree of adaptation or flexibility of the intervention (eg, further agile development through software updates) [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref25">25</xref>].</p>
          </list-item>
        </list>
        <p>To follow a user-centered approach, developers must integrate the users (ie, the target group) in the development process. A way of structuring the involvement of users is the <italic>Steps of Participation Approach</italic> suggested by Wright [<xref ref-type="bibr" rid="ref6">6</xref>]. This model describes the user’s nonparticipation and involvement in the research process. It further differentiates among 9 stages, ranging from instrumentalization to self-organization. The 9 stages provide a hierarchical order not only for participation but also for the nonparticipation of target groups in the development of public health interventions. Although it includes 9 stages, only the last 4 include real participation, according to Wright [<xref ref-type="bibr" rid="ref6">6</xref>], as the first 2 have no target group members involved in the development process. Steps 3 to 5 are the precursors for participation. As stated by this approach, one can only speak of participation in only those areas where people have the power to participate in the decision-making processes [<xref ref-type="bibr" rid="ref26">26</xref>]. The 9 significant steps based on Wright [<xref ref-type="bibr" rid="ref6">6</xref>] are described in the following sections. As shown in <xref ref-type="table" rid="table1">Table 1</xref>, the difference among the 3 groups of nonparticipation is that the first 2 steps completely exclude the target group. Although grades 3 to 5 recognize the target group as advisers, they do not include them in the decision-making process, which occurs in steps 6 to 9 (<xref ref-type="table" rid="table1">Table 1</xref>). The chance to successfully roll out and implement a digital public health intervention increases as the development process includes the target group [<xref ref-type="bibr" rid="ref27">27</xref>]. Therefore, a user-oriented way of developing digital public health interventions to increase acceptance and use of such interventions should be a goal of digital public health.</p>
        <p>A way of including target groups in the development of new digital public health interventions could be to apply user-centered approaches to the development process [<xref ref-type="bibr" rid="ref28">28</xref>]. The aim here is to look at issues from various stakeholder perspectives and create new ideas in an interdisciplinary team to solve potential problems and challenges throughout the development of a digital public health intervention. Ideally, this approach also includes the target group (eg, for an app) to increase acceptance and use. Generally, participatory development processes are iterative and may be designed in various forms depending on the goal. In principle, the following four steps can shape the process: (1) concept generation and ideation; (2) prototype design and system development; (3) Evaluation; and (4) deployment, including various feedback loops (<xref rid="figure3" ref-type="fig">Figure 3</xref>). After an initial analysis of the user’s needs, the developers collect the criteria for functions and design. Then, they convert these recommendations into the functional specifications of a user-centered design. Using walkthroughs and usability testing, prototypes are tested and perfected before deployment, which helps to expose latent practical and interface design weaknesses. The developing team can achieve this by analyzing remotely collected data using automatic data transmission or video use. As usability testing is a pillar of the best practices for medical system architecture [<xref ref-type="bibr" rid="ref28">28</xref>], production teams should also apply this to digital public health interventions’ development.</p>
        <table-wrap position="float" id="table1">
          <label>Table 1</label>
          <caption>
            <p>The 9 different steps for including participant perspectives [<xref ref-type="bibr" rid="ref6">6</xref>].</p>
          </caption>
          <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
            <col width="30"/>
            <col width="270"/>
            <col width="0"/>
            <col width="700"/>
            <thead>
              <tr valign="top">
                <td colspan="3">Participation and steps</td>
                <td>Descriptions</td>
              </tr>
            </thead>
            <tbody>
              <tr valign="top">
                <td colspan="4">
                  <bold>No participation</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 1: instrumentalization</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>1.1 The interests of the target group are not necessary</p>
                    </list-item>
                    <list-item>
                      <p>1.2 Production team makes decisions outside the target group</p>
                    </list-item>
                    <list-item>
                      <p>1.3 The interests of the decision-makers are the focus of attention</p>
                    </list-item>
                    <list-item>
                      <p>1.4 Target group members as decoration</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 2: instruction</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>2.1 The situation of the target group is perceived</p>
                    </list-item>
                    <list-item>
                      <p>2.2 The problem is defined exclusively from the perspective of the decision-makers (professionals)</p>
                    </list-item>
                    <list-item>
                      <p>2.3 The opinion of the target group is not considered</p>
                    </list-item>
                    <list-item>
                      <p>2.4 Communication is direct</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="4">
                  <bold>Precursors for participation</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 3: information</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>3.1 The decision-makers tell the target group what problems the group has and what help they need</p>
                    </list-item>
                    <list-item>
                      <p>3.2 Recommendation of various courses of action</p>
                    </list-item>
                    <list-item>
                      <p>3.3 Explanation and justification of the procedure of the decision-makers</p>
                    </list-item>
                    <list-item>
                      <p>3.4 The point of view of the target group is considered to increase the acceptance of the messages</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 4: consultation</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>4.1 The decision-makers are interested in the view of the target group.</p>
                    </list-item>
                    <list-item>
                      <p>4.2 The members of the target group are listened to</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 5: involvement</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>5.1 The decision-makers are advised by (selected persons from) the target group</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td colspan="4">
                  <bold>Participation</bold>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 6: co-determination</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>6.1 The decision-makers consult with the target group</p>
                    </list-item>
                    <list-item>
                      <p>6.2 Negotiations between target group representatives and decision-makers</p>
                    </list-item>
                    <list-item>
                      <p>6.3 The target group members have a say</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 7: partial transfer of decision-making authority</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>7.1 A right of participation in the decision-making process</p>
                    </list-item>
                    <list-item>
                      <p>7.2 Decision-making authority is limited to certain aspects</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 8: decision-making power</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>8.1 The target group itself determines all essential aspects</p>
                    </list-item>
                    <list-item>
                      <p>8.2 Partnership-based cooperation between all parties involved</p>
                    </list-item>
                    <list-item>
                      <p>8.3 Accompaniment or support of others</p>
                    </list-item>
                  </list>
                </td>
              </tr>
              <tr valign="top">
                <td>
                  <break/>
                </td>
                <td>Step 9: self-organization</td>
                <td colspan="2">
                  <list list-type="bullet">
                    <list-item>
                      <p>9.1 The responsibility for a measure or a project is entirely in the hands of the target group</p>
                    </list-item>
                  </list>
                </td>
              </tr>
            </tbody>
          </table>
        </table-wrap>
        <fig id="figure3" position="float">
          <label>Figure 3</label>
          <caption>
            <p>Schematic representation of the user-centered design process [<xref ref-type="bibr" rid="ref28">28</xref>].</p>
          </caption>
          <graphic xlink:href="jmir_v24i6e31921_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
        </fig>
      </sec>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <sec>
        <title>Principal Findings</title>
        <p>The aim of this viewpoint paper was to define digital public health interventions and provide an exemplary classification framework for digital public health interventions. Such an approach may help identify core areas of digital public health interventions, which in turn might be helpful during the development and evaluation phases of digital public health interventions. We argue that it is crucial to examine digital public health interventions from 3 different perspectives. The first one should be the WHO framework for EPHF [<xref ref-type="bibr" rid="ref4">4</xref>]. This is important as it provides an overview of what kind of activities, which strengthen and maintain health at the population level, belong to public health as a discipline and, therefore, what a public health intervention may be. The second perspective focuses on the digital aspects of an intervention. A suitable framework is the <italic>Evidence Standards Framework for Digital Health Technologies</italic> by NICE, as it classifies digital interventions based on their functions and defines corresponding evidence standards [<xref ref-type="bibr" rid="ref5">5</xref>]. Both frameworks combined enable us to categorize digital public health interventions according to the area of public health and the level of interaction between the user and the digital tool. The last perspective focuses on user involvement in the development of such interventions, as proposed by Wright [<xref ref-type="bibr" rid="ref6">6</xref>]. This is of great importance, as studies suggest that the acceptance of target users increases with more involvement in the process of development, testing, and implementation. Therefore, acknowledging the 9 levels of user participation (and focusing on levels 6 to 9) may enable developers to create even more significant and meaningful digital public health interventions for their target group.</p>
        <p>Our current approach relates to a single and, to the best of our knowledge, the only definition of digital public health. As it is natural for such definitions to evolve over time as the field evolves, our suggestion for a definition of digital public health interventions might also evolve, as one cannot talk about a definition for digital public health interventions without defining the borders of digital public health. Although the suggested EPHF in this perspective piece refers to a summary of the WHO, some readers of this paper might find it hard to apply it to their specific context. This may very much depend on the health care system in which the digital public health intervention is developed. Therefore, the EPHF listed in this study should not be seen as a final list of public health functions or classification frameworks but rather as examples of core functions and goals. Similarly, the NICE framework might not be applied directly in other countries with different health systems and contexts; however, it might provide a helpful starting point for identifying relevant frameworks for such systems or developing their own frameworks that focus on interaction and functional classifications. Possible steps for participation to include user perspectives and methods (eg, user stories) might differ depending on the format and content of a specific intervention. For example, one cannot expect an app that facilitates communication between physicians and patients to unfold its full potential when the development team does not consider both perspectives regarding design, functions, and content [<xref ref-type="bibr" rid="ref29">29</xref>,<xref ref-type="bibr" rid="ref30">30</xref>]. Some effects might be more visible on a public health scale than others, depending on the population’s size and the health system for which the intervention was initially developed. More importantly, digital public health interventions should display their effectiveness beyond the laboratory in the real world. They should do so by providing study results with high internal validity and results with high external validity. This well-known approach within empirical social research ensures that measurable effects transpire from the laboratory to the real world.</p>
        <p>The following example aims to display the connections among the 3 analyzed frameworks and models. Since the beginning of the SARS-CoV-2 pandemic in early 2020, various countries have developed contact-tracing apps for monitoring and surveillance [<xref ref-type="bibr" rid="ref31">31</xref>]. The primary function of such apps is to notify users after contact with someone who was (later) tested positive for the SARS-CoV-2 virus [<xref ref-type="bibr" rid="ref32">32</xref>]. As previously mentioned, contact-tracing apps serve the horizontal EPHF of health information systems. Most countries set the bars for data security in contact-tracing apps high to improve users’ trust. Conversely, a high level of data protection prevents the collection and analysis of epidemiologically relevant data, making it more difficult to assess the effectiveness from a public health perspective. When developing a contact-tracing app, it is necessary to weigh the protection of privacy and the potential public health benefits against each other [<xref ref-type="bibr" rid="ref33">33</xref>]. This constraint of data availability for public health (research) limits contact-tracing apps to evidence tiers 1 and 2 within the NICE framework for DHIs. Although simple monitoring (as level 2 demands) is possible, the apps do not aim to calculate the diagnoses needed for tier 3 (instead, recommendations such as different colors for warning levels in the German app).</p>
        <p>As previously mentioned, participation in the development process is key to a successful intervention. Germany introduced the <italic>Corona Warn App</italic> in June 2020. The code was published as an open-source project in May 2020 on the GitHub coding platform [<xref ref-type="bibr" rid="ref34">34</xref>]. According to the developers, this approach should allow interested target group users to assess the code for themselves and add suggestions to improve the app [<xref ref-type="bibr" rid="ref34">34</xref>]. It is also possible to claim an interest in working on a specific part of the app’s code, which suggests a high level of involvement. Target group users are not just listened to but can also actively participate in the development process. However, although this approach offers a high level of participation for some members of the target group with a background or interest in coding, this approach excludes most other users because of their missing knowledge in information and communication technology. Furthermore, no clear information on the extent to which user groups were included in the actual development process is available. Despite the limited involvement of the target group, the app was downloaded 28.3 million times with 472.960 positive tests shared within the app by June 11, 2021, suggesting at least some success [<xref ref-type="bibr" rid="ref35">35</xref>].</p>
      </sec>
      <sec>
        <title>Conclusions</title>
        <p>This study aimed to provide the first definition and classification framework for digital public health interventions. Here, we suggest that digital public health, as a complex intervention, should be viewed from different perspectives. As such, our proposed classification is a combination of the elements of already existing models, specifically, the EPHF by the WHO, which provided us with an overview of the necessary core functions of public health that might also be addressed by digital means. The NICE framework gave us an overview of different areas for digital technologies and potential evaluation requirements. Both models together form a framework for describing digital public health interventions. However, without the inclusion of target groups in user-centered processes during the development, these interventions may lack efficiency and the acceptance of potential users. Therefore, we propose an established user-centered design process to be included in the development of digital public health interventions. Nevertheless, users of our definition and framework must check the validity of our criteria within their setting (eg, population structure, understanding of public health, and health care system). Taking the different strains of research that together might provide a better understanding of the term <italic>digital public health intervention</italic>, the first definition might be as follows:</p>
        <disp-quote>
          <p>A Digital Public Health Intervention addresses at least one essential Public Health function through digital means. Applying a framework for functional classification and stratification categorizes its interaction level with the user. The developmental process of a digital public health intervention includes the user perspective by applying participatory methods to support its effectiveness and implementation with the goal to achieve a population health impact.</p>
        </disp-quote>
        <p>The first step toward a potential intervention classification framework was developed based on this definition and its underlying frameworks (<xref ref-type="supplementary-material" rid="app1">Multimedia Appendix 1</xref>). The aim of this framework is three-fold: (1) support the future reporting of digital public health intervention functions and effectiveness by providing a framework for classification, (2) identify future requirements (eg, for evaluation) of such interventions, and (3) support the implementation processes of digital public health interventions by linking implementation needs and characteristics with the classification framework (ie, a digital public health intervention addressing active monitoring in health care with high levels of user involvement might have other implementation needs than a digital public health intervention that addresses simple monitoring in health care with low levels of user involvement) [<xref ref-type="bibr" rid="ref36">36</xref>]. We view a combination of all 3 models as a chance to set up a first definition and classification for digital public health interventions and hope that our approach will encourage the uptake and further development of our idea.</p>
      </sec>
    </sec>
  </body>
  <back>
    <app-group>
      <supplementary-material id="app1">
        <label>Multimedia Appendix 1</label>
        <p>Digital public health intervention classification framework.</p>
        <media xlink:href="jmir_v24i6e31921_app1.docx" xlink:title="DOCX File , 80 KB"/>
      </supplementary-material>
    </app-group>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">DHI</term>
          <def>
            <p>digital health intervention</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">EPHF</term>
          <def>
            <p>Essential Public Health Function</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">mHealth</term>
          <def>
            <p>mobile health</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">NICE</term>
          <def>
            <p>National Institute for Health and Care Excellence</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">WHO</term>
          <def>
            <p>World Health Organization</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>The authors gratefully acknowledge Leibniz Science Campus Bremen Digital Public Health (lsc-diph.de) support, jointly funded by the Leibniz Association (W4/2018), the Federal State of Bremen, and the Leibniz Institute for Prevention Research and Epidemiology–BIPS.</p>
    </ack>
    <fn-group>
      <fn fn-type="con">
        <p>JW provided the idea for the manuscript, defined the criteria for digital public health interventions, prepared the first definition for such interventions, came up with the first draft of the manuscript, and revised the final draft of the manuscript. JW and LM contributed equally to the manuscript and further developed the draft for the final paper. TJ provided critical feedback and contributed to reviewing and editing the final draft.</p>
      </fn>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schoeppe</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Alley</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Van Lippevelde</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Bray</surname>
              <given-names>NA</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Duncan</surname>
              <given-names>MJ</given-names>
            </name>
            <name name-style="western">
              <surname>Vandelanotte</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Efficacy of interventions that use apps to improve diet, physical activity and sedentary behaviour: a systematic review</article-title>
          <source>Int J Behav Nutr Phys Act</source>
          <year>2016</year>
          <month>12</month>
          <day>07</day>
          <volume>13</volume>
          <issue>1</issue>
          <fpage>127</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://ijbnpa.biomedcentral.com/articles/10.1186/s12966-016-0454-y"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12966-016-0454-y</pub-id>
          <pub-id pub-id-type="medline">27927218</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12966-016-0454-y</pub-id>
          <pub-id pub-id-type="pmcid">PMC5142356</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zeeb</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Pigeot</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Schüz</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>[Digital public health-rapid technological progress, but many open public health questions]</article-title>
          <source>Bundesgesundheitsblatt Gesundheitsforschung Gesundheitsschutz</source>
          <year>2020</year>
          <month>02</month>
          <volume>63</volume>
          <issue>2</issue>
          <fpage>135</fpage>
          <lpage>6</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/31960072"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s00103-020-03092-0</pub-id>
          <pub-id pub-id-type="medline">31960072</pub-id>
          <pub-id pub-id-type="pii">10.1007/s00103-020-03092-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC8219547</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Hekler</surname>
              <given-names>EB</given-names>
            </name>
            <name name-style="western">
              <surname>Andersson</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Collins</surname>
              <given-names>LM</given-names>
            </name>
            <name name-style="western">
              <surname>Doherty</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Hollis</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Rivera</surname>
              <given-names>DE</given-names>
            </name>
            <name name-style="western">
              <surname>West</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Wyatt</surname>
              <given-names>JC</given-names>
            </name>
          </person-group>
          <article-title>Evaluating digital health interventions: key questions and approaches</article-title>
          <source>Am J Prev Med</source>
          <year>2016</year>
          <month>11</month>
          <volume>51</volume>
          <issue>5</issue>
          <fpage>843</fpage>
          <lpage>51</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/27745684"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.amepre.2016.06.008</pub-id>
          <pub-id pub-id-type="medline">27745684</pub-id>
          <pub-id pub-id-type="pii">S0749-3797(16)30229-X</pub-id>
          <pub-id pub-id-type="pmcid">PMC5324832</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="book">
          <source>Essential Public Health Functions, Health Systems and Health Security Developing Conceptual Clarity and a WHO Roadmap for Action</source>
          <year>2018</year>
          <publisher-loc>Geneva</publisher-loc>
          <publisher-name>World Health Organization</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="web">
          <article-title>Evidence standards framework for digital health technologies</article-title>
          <source>National Institute for Health and Care Excellence</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.nice.org.uk/Media/Default/About/what-we-do/our-programmes/evidence-standards-framework/digital-evidence-standards-framework.pdf">https://www.nice.org.uk/Media/Default/About/what-we-do/our-programmes/evidence-standards-framework/digital-evidence-standards-framework.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wright</surname>
              <given-names>MT</given-names>
            </name>
          </person-group>
          <article-title>[Participatory health research: origins and current trends]</article-title>
          <source>Bundesgesundheitsblatt Gesundheitsforschung Gesundheitsschutz</source>
          <year>2021</year>
          <month>02</month>
          <volume>64</volume>
          <issue>2</issue>
          <fpage>140</fpage>
          <lpage>5</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/33336312"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s00103-020-03264-y</pub-id>
          <pub-id pub-id-type="medline">33336312</pub-id>
          <pub-id pub-id-type="pii">10.1007/s00103-020-03264-y</pub-id>
          <pub-id pub-id-type="pmcid">PMC7843534</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nievas</surname>
              <given-names>SB</given-names>
            </name>
            <name name-style="western">
              <surname>García</surname>
              <given-names>DS</given-names>
            </name>
            <name name-style="western">
              <surname>Fernández</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Bonillo</surname>
              <given-names>PA</given-names>
            </name>
            <name name-style="western">
              <surname>Parrón</surname>
              <given-names>CT</given-names>
            </name>
          </person-group>
          <article-title>eHealth: advantages, disadvantages and guiding principles for the future</article-title>
          <source>JMIR</source>
          <year>2019</year>
          <comment>(forthcoming)</comment>
          <pub-id pub-id-type="doi">10.2196/preprints.15366</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Oh</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Rizo</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Enkin</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Jadad</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>What is eHealth (3): a systematic review of published definitions</article-title>
          <source>J Med Internet Res</source>
          <year>2005</year>
          <month>02</month>
          <day>24</day>
          <volume>7</volume>
          <issue>1</issue>
          <fpage>e1</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2005/1/e1/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.7.1.e1</pub-id>
          <pub-id pub-id-type="medline">15829471</pub-id>
          <pub-id pub-id-type="pii">v7e1</pub-id>
          <pub-id pub-id-type="pmcid">PMC1550636</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shaw</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>McGregor</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Brunner</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Keep</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Janssen</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Barnet</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>What is eHealth (6)? Development of a conceptual model for eHealth: qualitative study with key informants</article-title>
          <source>J Med Internet Res</source>
          <year>2017</year>
          <month>10</month>
          <day>24</day>
          <volume>19</volume>
          <issue>10</issue>
          <fpage>e324</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2017/10/e324/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.8106</pub-id>
          <pub-id pub-id-type="medline">29066429</pub-id>
          <pub-id pub-id-type="pii">v19i10e324</pub-id>
          <pub-id pub-id-type="pmcid">PMC5676031</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Eysenbach</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>What is e-health?</article-title>
          <source>J Med Internet Res</source>
          <year>2001</year>
          <volume>3</volume>
          <issue>2</issue>
          <fpage>E20</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2001/2/e20/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.3.2.e20</pub-id>
          <pub-id pub-id-type="medline">11720962</pub-id>
          <pub-id pub-id-type="pmcid">PMC1761894</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Iacono</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Stagg</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Pearce</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Hulme Chambers</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>A scoping review of Australian allied health research in eHealth</article-title>
          <source>BMC Health Serv Res</source>
          <year>2016</year>
          <month>10</month>
          <day>04</day>
          <volume>16</volume>
          <issue>1</issue>
          <fpage>543</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmchealthservres.biomedcentral.com/articles/10.1186/s12913-016-1791-x"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12913-016-1791-x</pub-id>
          <pub-id pub-id-type="medline">27716325</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12913-016-1791-x</pub-id>
          <pub-id pub-id-type="pmcid">PMC5050606</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pagliari</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Sloan</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Gregor</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Sullivan</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Detmer</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Kahan</surname>
              <given-names>JP</given-names>
            </name>
            <name name-style="western">
              <surname>Oortwijn</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>MacGillivray</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>What is eHealth (4): a scoping exercise to map the field</article-title>
          <source>J Med Internet Res</source>
          <year>2005</year>
          <month>03</month>
          <day>31</day>
          <volume>7</volume>
          <issue>1</issue>
          <fpage>e9</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2005/1/e9/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.7.1.e9</pub-id>
          <pub-id pub-id-type="medline">15829481</pub-id>
          <pub-id pub-id-type="pii">v7e9</pub-id>
          <pub-id pub-id-type="pmcid">PMC1550637</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davis</surname>
              <given-names>TL</given-names>
            </name>
            <name name-style="western">
              <surname>DiClemente</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Prietula</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Taking mHealth forward: examining the core characteristics</article-title>
          <source>JMIR Mhealth Uhealth</source>
          <year>2016</year>
          <month>08</month>
          <day>10</day>
          <volume>4</volume>
          <issue>3</issue>
          <fpage>e97</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://mhealth.jmir.org/2016/3/e97/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/mhealth.5659</pub-id>
          <pub-id pub-id-type="medline">27511612</pub-id>
          <pub-id pub-id-type="pii">v4i3e97</pub-id>
          <pub-id pub-id-type="pmcid">PMC4997001</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Fatehi</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Samadbeik</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kazemi</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>What is digital health? Review of definitions</article-title>
          <source>Stud Health Technol Inform</source>
          <year>2020</year>
          <month>11</month>
          <day>23</day>
          <volume>275</volume>
          <fpage>67</fpage>
          <lpage>71</lpage>
          <pub-id pub-id-type="doi">10.3233/SHTI200696</pub-id>
          <pub-id pub-id-type="medline">33227742</pub-id>
          <pub-id pub-id-type="pii">SHTI200696</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zeeb</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Pigeot</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Schüz</surname>
              <given-names>B</given-names>
            </name>
            <collab>Leibniz-WissenschaftsCampus Digital Public Health Bremen</collab>
          </person-group>
          <article-title>[Digital public health-an overview]</article-title>
          <source>Bundesgesundheitsblatt Gesundheitsforschung Gesundheitsschutz</source>
          <year>2020</year>
          <month>02</month>
          <volume>63</volume>
          <issue>2</issue>
          <fpage>137</fpage>
          <lpage>44</lpage>
          <pub-id pub-id-type="doi">10.1007/s00103-019-03078-7</pub-id>
          <pub-id pub-id-type="medline">31919531</pub-id>
          <pub-id pub-id-type="pii">10.1007/s00103-019-03078-7</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="web">
          <article-title>Digital health applications (DiGA)</article-title>
          <source>Federal Institute for Drugs and Medical Devices</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.bfarm.de/EN/MedicalDevices/DiGA/_node.html">https://www.bfarm.de/EN/MedicalDevices/DiGA/_node.html</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Davalbhakta</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Advani</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Kumar</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Agarwal</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Bhoyar</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Fedirko</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Misra</surname>
              <given-names>DP</given-names>
            </name>
            <name name-style="western">
              <surname>Goel</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gupta</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Agarwal</surname>
              <given-names>V</given-names>
            </name>
          </person-group>
          <article-title>A systematic review of smartphone applications available for corona virus disease 2019 (COVID19) and the assessment of their quality using the mobile application rating scale (MARS)</article-title>
          <source>J Med Syst</source>
          <year>2020</year>
          <month>08</month>
          <day>10</day>
          <volume>44</volume>
          <issue>9</issue>
          <fpage>164</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/32779002"/>
          </comment>
          <pub-id pub-id-type="doi">10.1007/s10916-020-01633-3</pub-id>
          <pub-id pub-id-type="medline">32779002</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10916-020-01633-3</pub-id>
          <pub-id pub-id-type="pmcid">PMC7417101</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Murray</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Alamro</surname>
              <given-names>NM</given-names>
            </name>
            <name name-style="western">
              <surname>Hwang</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>U</given-names>
            </name>
          </person-group>
          <article-title>Digital public health and COVID-19</article-title>
          <source>Lancet Public Health</source>
          <year>2020</year>
          <month>09</month>
          <volume>5</volume>
          <issue>9</issue>
          <fpage>e469</fpage>
          <lpage>70</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S2468-2667(20)30187-0"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/S2468-2667(20)30187-0</pub-id>
          <pub-id pub-id-type="medline">32791051</pub-id>
          <pub-id pub-id-type="pii">S2468-2667(20)30187-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC7417165</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bradford</surname>
              <given-names>NK</given-names>
            </name>
            <name name-style="western">
              <surname>Caffery</surname>
              <given-names>LJ</given-names>
            </name>
            <name name-style="western">
              <surname>Smith</surname>
              <given-names>AC</given-names>
            </name>
          </person-group>
          <article-title>Telehealth services in rural and remote Australia: a systematic review of models of care and factors influencing success and sustainability</article-title>
          <source>Rural Remote Health</source>
          <year>2016</year>
          <volume>16</volume>
          <issue>4</issue>
          <fpage>4268</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.rrh.org.au/articles/subviewnew.asp?ArticleID=4268"/>
          </comment>
          <pub-id pub-id-type="medline">27817199</pub-id>
          <pub-id pub-id-type="pii">4268</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Edwards</surname>
              <given-names>EA</given-names>
            </name>
            <name name-style="western">
              <surname>Lumsden</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rivas</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Steed</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Edwards</surname>
              <given-names>LA</given-names>
            </name>
            <name name-style="western">
              <surname>Thiyagarajan</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sohanpal</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Caton</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Griffiths</surname>
              <given-names>CJ</given-names>
            </name>
            <name name-style="western">
              <surname>Munafò</surname>
              <given-names>MR</given-names>
            </name>
            <name name-style="western">
              <surname>Taylor</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Walton</surname>
              <given-names>RT</given-names>
            </name>
          </person-group>
          <article-title>Gamification for health promotion: systematic review of behaviour change techniques in smartphone apps</article-title>
          <source>BMJ Open</source>
          <year>2016</year>
          <month>10</month>
          <day>04</day>
          <volume>6</volume>
          <issue>10</issue>
          <fpage>e012447</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmjopen.bmj.com/lookup/pmidlookup?view=long&#38;pmid=27707829"/>
          </comment>
          <pub-id pub-id-type="doi">10.1136/bmjopen-2016-012447</pub-id>
          <pub-id pub-id-type="medline">27707829</pub-id>
          <pub-id pub-id-type="pii">bmjopen-2016-012447</pub-id>
          <pub-id pub-id-type="pmcid">PMC5073629</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhao</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Freeman</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Li</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Can mobile phone apps influence people's health behavior change? An evidence review</article-title>
          <source>J Med Internet Res</source>
          <year>2016</year>
          <month>10</month>
          <day>31</day>
          <volume>18</volume>
          <issue>11</issue>
          <fpage>e287</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2016/11/e287/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/jmir.5692</pub-id>
          <pub-id pub-id-type="medline">27806926</pub-id>
          <pub-id pub-id-type="pii">v18i11e287</pub-id>
          <pub-id pub-id-type="pmcid">PMC5295827</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kloc</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Rémuzat</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>François</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Toumi</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>PNS235 assessment of digital health technologies - comparison of evidence frameworks of nice and has</article-title>
          <source>Value Health</source>
          <year>2019</year>
          <month>11</month>
          <volume>22</volume>
          <issue>3</issue>
          <fpage>S801</fpage>
          <pub-id pub-id-type="doi">10.1016/j.jval.2019.09.2135</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Forsyth</surname>
              <given-names>JR</given-names>
            </name>
            <name name-style="western">
              <surname>Chase</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Roberts</surname>
              <given-names>NW</given-names>
            </name>
            <name name-style="western">
              <surname>Armitage</surname>
              <given-names>LC</given-names>
            </name>
            <name name-style="western">
              <surname>Farmer</surname>
              <given-names>AJ</given-names>
            </name>
          </person-group>
          <article-title>Application of the national institute for health and care excellence evidence standards framework for digital health technologies in assessing mobile-delivered technologies for the self-management of type 2 diabetes mellitus: scoping review</article-title>
          <source>JMIR Diabetes</source>
          <year>2021</year>
          <month>02</month>
          <day>16</day>
          <volume>6</volume>
          <issue>1</issue>
          <fpage>e23687</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://diabetes.jmir.org/2021/1/e23687/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/23687</pub-id>
          <pub-id pub-id-type="medline">33591278</pub-id>
          <pub-id pub-id-type="pii">v6i1e23687</pub-id>
          <pub-id pub-id-type="pmcid">PMC7925151</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Nwe</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Larsen</surname>
              <given-names>ME</given-names>
            </name>
            <name name-style="western">
              <surname>Nelissen</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wong</surname>
              <given-names>DC</given-names>
            </name>
          </person-group>
          <article-title>Medical mobile app classification using the national institute for health and care excellence evidence standards framework for digital health technologies: interrater reliability study</article-title>
          <source>J Med Internet Res</source>
          <year>2020</year>
          <month>06</month>
          <day>05</day>
          <volume>22</volume>
          <issue>6</issue>
          <fpage>e17457</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.jmir.org/2020/6/e17457/"/>
          </comment>
          <pub-id pub-id-type="doi">10.2196/17457</pub-id>
          <pub-id pub-id-type="medline">32501271</pub-id>
          <pub-id pub-id-type="pii">v22i6e17457</pub-id>
          <pub-id pub-id-type="pmcid">PMC7305556</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Hochmuth</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Exner</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Dockweiler</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>[Implementation and participatory design of digital health interventions]</article-title>
          <source>Bundesgesundheitsblatt Gesundheitsforschung Gesundheitsschutz</source>
          <year>2020</year>
          <month>02</month>
          <volume>63</volume>
          <issue>2</issue>
          <fpage>145</fpage>
          <lpage>52</lpage>
          <pub-id pub-id-type="doi">10.1007/s00103-019-03079-6</pub-id>
          <pub-id pub-id-type="medline">31938837</pub-id>
          <pub-id pub-id-type="pii">10.1007/s00103-019-03079-6</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bergold</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Thomas</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Participatory research methods: a methodological approach in motion</article-title>
          <source>Forum Qual Soc Res</source>
          <year>2012</year>
          <volume>13</volume>
          <issue>1</issue>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://nbn-resolving.de/urn:nbn:de:0114-fqs1201302"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Vastine</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gittelsohn</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ethelbah</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Anliker</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Caballero</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Formative research and stakeholder participation in intervention development</article-title>
          <source>Am J Health Behav</source>
          <year>2005</year>
          <volume>29</volume>
          <issue>1</issue>
          <fpage>57</fpage>
          <lpage>69</lpage>
          <pub-id pub-id-type="doi">10.5993/ajhb.29.1.5</pub-id>
          <pub-id pub-id-type="medline">15604050</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>McCurdie</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Taneva</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Casselman</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Yeung</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>McDaniel</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ho</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Cafazzo</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>mHealth consumer apps: the case for user-centered design</article-title>
          <source>Biomed Instrum Technol</source>
          <year>2012</year>
          <volume>Suppl</volume>
          <fpage>49</fpage>
          <lpage>56</lpage>
          <pub-id pub-id-type="doi">10.2345/0899-8205-46.s2.49</pub-id>
          <pub-id pub-id-type="medline">23039777</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lippke</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Wienert</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Keller</surname>
              <given-names>FM</given-names>
            </name>
            <name name-style="western">
              <surname>Derksen</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Welp</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kötting</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Hofreuter-Gätgens</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Müller</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Louwen</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Weigand</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Ernst</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Kraft</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Reister</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Polasik</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Huener Nee Seemann</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jennewein</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Scholz</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Hannawa</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Communication and patient safety in gynecology and obstetrics - study protocol of an intervention study</article-title>
          <source>BMC Health Serv Res</source>
          <year>2019</year>
          <month>11</month>
          <day>28</day>
          <volume>19</volume>
          <issue>1</issue>
          <fpage>908</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://bmchealthservres.biomedcentral.com/articles/10.1186/s12913-019-4579-y"/>
          </comment>
          <pub-id pub-id-type="doi">10.1186/s12913-019-4579-y</pub-id>
          <pub-id pub-id-type="medline">31779620</pub-id>
          <pub-id pub-id-type="pii">10.1186/s12913-019-4579-y</pub-id>
          <pub-id pub-id-type="pmcid">PMC6883614</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wienert</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Understanding health information technologies as complex interventions with the need for thorough implementation and monitoring to sustain patient safety</article-title>
          <source>Front ICT</source>
          <year>2019</year>
          <month>5</month>
          <day>17</day>
          <volume>6</volume>
          <pub-id pub-id-type="doi">10.3389/fict.2019.00009</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="web">
          <article-title>A flood of coronavirus apps are tracking us</article-title>
          <source>MIT Technology Review</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.technologyreview.com/2020/05/07/1000961/launching-mittr-covid-tracing-tracker/">https://www.technologyreview.com/2020/05/07/1000961/launching-mittr-covid-tracing-tracker/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="web">
          <article-title>Effective configurations of a digital contact tracing app: a report to NHSX</article-title>
          <source>CDN</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://cdn.theconversation.com/static_files/files/1009/Report_-_Effective_App_Configurations.pdf?1587531217">https://cdn.theconversation.com/static_files/files/1009/Report_-_Effective_App_Configurations.pdf?1587531217</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Jahnel</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Gerhardus</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Wienert</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Digitales contact tracing: dilemma zwischen Datenschutz und Public Health Nutzenbewertung</article-title>
          <source>Datenschutz Datensich</source>
          <year>2020</year>
          <month>11</month>
          <day>05</day>
          <volume>44</volume>
          <issue>12</issue>
          <fpage>786</fpage>
          <lpage>90</lpage>
          <pub-id pub-id-type="doi">10.1007/s11623-020-1367-0</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="web">
          <article-title>Corona Warn-App hompage</article-title>
          <source>Corona Warn-App</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.coronawarn.app/en/community/">https://www.coronawarn.app/en/community/</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="web">
          <article-title>Key figures for the Corona-Warn-App</article-title>
          <source>Robert Koch-Institut</source>
          <access-date>2022-02-17</access-date>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/WarnApp/Archiv_Kennzahlen/Kennzahlen_29012021.pdf">https://www.rki.de/DE/Content/InfAZ/N/Neuartiges_Coronavirus/WarnApp/Archiv_Kennzahlen/Kennzahlen_29012021.pdf</ext-link>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wienert</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Zeeb</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Implementing health apps for digital public health - an implementation science approach adopting the consolidated framework for implementation research</article-title>
          <source>Front Public Health</source>
          <year>2021</year>
          <volume>9</volume>
          <fpage>610237</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.3389/fpubh.2021.610237"/>
          </comment>
          <pub-id pub-id-type="doi">10.3389/fpubh.2021.610237</pub-id>
          <pub-id pub-id-type="medline">34026702</pub-id>
          <pub-id pub-id-type="pmcid">PMC8137849</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
