<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE article PUBLIC "-//NLM//DTD Journal Publishing DTD v2.0 20040830//EN" "http://dtd.nlm.nih.gov/publishing/2.0/journalpublishing.dtd">
<article xmlns:xlink="http://www.w3.org/1999/xlink" article-type="research-article" dtd-version="2.0">
  <front>
    <journal-meta>
      <journal-id journal-id-type="publisher-id">JMIR</journal-id>
      <journal-id journal-id-type="nlm-ta">J Med Internet Res</journal-id>
      <journal-title>Journal of Medical Internet Research</journal-title>
      <issn pub-type="epub">1438-8871</issn>
      <publisher>
        <publisher-name>JMIR Publications</publisher-name>
        <publisher-loc>Toronto, Canada</publisher-loc>
      </publisher>
    </journal-meta>
    <article-meta>
      <article-id pub-id-type="publisher-id">v23i2e23920</article-id>
      <article-id pub-id-type="pmid">33533723</article-id>
      <article-id pub-id-type="doi">10.2196/23920</article-id>
      <article-categories>
        <subj-group subj-group-type="heading">
          <subject>Original Paper</subject>
        </subj-group>
        <subj-group subj-group-type="article-type">
          <subject>Original Paper</subject>
        </subj-group>
      </article-categories>
      <title-group>
        <article-title>Novel Analgesic Index for Postoperative Pain Assessment Based on a Photoplethysmographic Spectrogram and Convolutional Neural Network: Observational Study</article-title>
      </title-group>
      <contrib-group>
        <contrib contrib-type="editor">
          <name>
            <surname>Eysenbach</surname>
            <given-names>Gunther</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Chen</surname>
            <given-names>Ken</given-names>
          </name>
        </contrib>
        <contrib contrib-type="reviewer">
          <name>
            <surname>Demir</surname>
            <given-names>Huriye</given-names>
          </name>
        </contrib>
      </contrib-group>
      <contrib-group>
        <contrib id="contrib1" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Choi</surname>
            <given-names>Byung-Moon</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-6561-8851</ext-link>
        </contrib>
        <contrib id="contrib2" contrib-type="author" equal-contrib="yes">
          <name name-style="western">
            <surname>Yim</surname>
            <given-names>Ji Yeon</given-names>
          </name>
          <degrees>BSc</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0003-1752-8779</ext-link>
        </contrib>
        <contrib id="contrib3" contrib-type="author" corresp="yes" equal-contrib="yes">
          <name name-style="western">
            <surname>Shin</surname>
            <given-names>Hangsik</given-names>
          </name>
          <degrees>PhD</degrees>
          <xref rid="aff2" ref-type="aff">2</xref>
          <address>
            <institution>Department of Biomedical Engineering</institution>
            <institution>Chonnam National University</institution>
            <addr-line>50 Daehak-ro</addr-line>
            <addr-line>Yeosu, 59626</addr-line>
            <country>Republic of Korea</country>
            <phone>82 1068998679</phone>
            <email>hangsik.shin@jnu.ac.kr</email>
          </address>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-3353-0310</ext-link>
        </contrib>
        <contrib id="contrib4" contrib-type="author">
          <name name-style="western">
            <surname>Noh</surname>
            <given-names>Gyujeong</given-names>
          </name>
          <degrees>MD, PhD</degrees>
          <xref rid="aff1" ref-type="aff">1</xref>
          <xref rid="aff3" ref-type="aff">3</xref>
          <ext-link ext-link-type="orcid">https://orcid.org/0000-0002-1964-9294</ext-link>
        </contrib>
      </contrib-group>
      <aff id="aff1">
        <label>1</label>
        <institution>Department of Anaesthesiology and Pain Medicine</institution>
        <institution>Asan Medical Center</institution>
        <institution>University of Ulsan College of Medicine</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff2">
        <label>2</label>
        <institution>Department of Biomedical Engineering</institution>
        <institution>Chonnam National University</institution>
        <addr-line>Yeosu</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <aff id="aff3">
        <label>3</label>
        <institution>Department of Clinical Pharmacology and Therapeutics</institution>
        <institution>Asan Medical Center</institution>
        <institution>University of Ulsan College of Medicine</institution>
        <addr-line>Seoul</addr-line>
        <country>Republic of Korea</country>
      </aff>
      <author-notes>
        <corresp>Corresponding Author: Hangsik Shin <email>hangsik.shin@jnu.ac.kr</email></corresp>
      </author-notes>
      <pub-date pub-type="collection">
        <month>2</month>
        <year>2021</year>
      </pub-date>
      <pub-date pub-type="epub">
        <day>3</day>
        <month>2</month>
        <year>2021</year>
      </pub-date>
      <volume>23</volume>
      <issue>2</issue>
      <elocation-id>e23920</elocation-id>
      <history>
        <date date-type="received">
          <day>27</day>
          <month>8</month>
          <year>2020</year>
        </date>
        <date date-type="rev-request">
          <day>21</day>
          <month>9</month>
          <year>2020</year>
        </date>
        <date date-type="rev-recd">
          <day>21</day>
          <month>9</month>
          <year>2020</year>
        </date>
        <date date-type="accepted">
          <day>18</day>
          <month>1</month>
          <year>2021</year>
        </date>
      </history>
      <copyright-statement>©Byung-Moon Choi, Ji Yeon Yim, Hangsik Shin, Gyujeong Noh. Originally published in the Journal of Medical Internet Research (http://www.jmir.org), 03.02.2021.</copyright-statement>
      <copyright-year>2021</copyright-year>
      <license license-type="open-access" xlink:href="https://creativecommons.org/licenses/by/4.0/">
        <p>This is an open-access article distributed under the terms of the Creative Commons Attribution License (https://creativecommons.org/licenses/by/4.0/), which permits unrestricted use, distribution, and reproduction in any medium, provided the original work, first published in the Journal of Medical Internet Research, is properly cited. The complete bibliographic information, a link to the original publication on http://www.jmir.org/, as well as this copyright and license information must be included.</p>
      </license>
      <self-uri xlink:href="http://www.jmir.org/2021/2/e23920/" xlink:type="simple"/>
      <abstract>
        <sec sec-type="background">
          <title>Background</title>
          <p>Although commercially available analgesic indices based on biosignal processing have been used to quantify nociception during general anesthesia, their performance is low in conscious patients. Therefore, there is a need to develop a new analgesic index with improved performance to quantify postoperative pain in conscious patients.</p>
        </sec>
        <sec sec-type="objective">
          <title>Objective</title>
          <p>This study aimed to develop a new analgesic index using photoplethysmogram (PPG) spectrograms and a convolutional neural network (CNN) to objectively assess pain in conscious patients.</p>
        </sec>
        <sec sec-type="methods">
          <title>Methods</title>
          <p>PPGs were obtained from a group of surgical patients for 6 minutes both in the absence (preoperatively) and in the presence (postoperatively) of pain. Then, the PPG data of the latter 5 minutes were used for analysis. Based on the PPGs and a CNN, we developed a spectrogram–CNN index for pain assessment. The area under the curve (AUC) of the receiver-operating characteristic curve was measured to evaluate the performance of the 2 indices.</p>
        </sec>
        <sec sec-type="results">
          <title>Results</title>
          <p>PPGs from 100 patients were used to develop the spectrogram–CNN index. When there was pain, the mean (95% CI) spectrogram–CNN index value increased significantly—baseline: 28.5 (24.2-30.7) versus recovery area: 65.7 (60.5-68.3); <italic>P</italic>&#60;.01. The AUC and balanced accuracy were 0.76 and 71.4%, respectively. The spectrogram–CNN index cutoff value for detecting pain was 48, with a sensitivity of 68.3% and specificity of 73.8%.</p>
        </sec>
        <sec sec-type="conclusions">
          <title>Conclusions</title>
          <p>Although there were limitations to the study design, we confirmed that the spectrogram–CNN index can efficiently detect postoperative pain in conscious patients. Further studies are required to assess the spectrogram–CNN index’s feasibility and prevent overfitting to various populations, including patients under general anesthesia.</p>
        </sec>
        <sec sec-type="Trial Registration">
          <title>Trial Registration</title>
          <p>Clinical Research Information Service KCT0002080; https://cris.nih.go.kr/cris/search/search_result_st01.jsp?seq=6638</p>
        </sec>
      </abstract>
      <kwd-group>
        <kwd>analgesic index</kwd>
        <kwd>machine learning</kwd>
        <kwd>pain assessment</kwd>
        <kwd>photoplethysmogram</kwd>
        <kwd>postoperative pain</kwd>
        <kwd>spectrogram</kwd>
      </kwd-group>
    </article-meta>
  </front>
  <body>
    <sec sec-type="introduction">
      <title>Introduction</title>
      <p>Efficient management of postoperative pain affecting the prognosis of patients is becoming increasingly important [<xref ref-type="bibr" rid="ref1">1</xref>]. To properly administer analgesics, it is necessary to first objectively assess the patient’s degree of pain. In conscious patients, pain can be assessed by asking the patient directly, but unconscious patients or those with difficulty communicating require an appropriate index to quantify their pain. However, current commercial analgesic indices were developed for the purpose of evaluating nociception in patients under general anesthesia [<xref ref-type="bibr" rid="ref2">2</xref>,<xref ref-type="bibr" rid="ref3">3</xref>]; therefore, there is no standard for the quantification of postoperative pain in conscious patients [<xref ref-type="bibr" rid="ref4">4</xref>]. Thus, developing a new pain index to quantify pain in patients who cannot directly communicate their level of pain may also help in the clinical setting; it will also reduce the need to ask questions each time the patient is conscious when pain must be evaluated frequently.</p>
      <p>A photoplethysmogram (PPG) is a biosignal that can be obtained continuously and noninvasively using a pulse oximeter. Because a PPG conveys much information about a patient’s condition, many attempts have been made to quantify pain by analyzing PPG signals [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref5">5</xref>-<xref ref-type="bibr" rid="ref7">7</xref>]. The surgical pleth index (SPI; GE Healthcare), developed for quantifying nociception during general anesthesia, only considers the amplitude and heartbeat interval of a PPG [<xref ref-type="bibr" rid="ref3">3</xref>]. In addition to these 2 parameters, other pain-related features are present in PPG signals [<xref ref-type="bibr" rid="ref6">6</xref>,<xref ref-type="bibr" rid="ref7">7</xref>]. Therefore, the application of a new analytical method has the potential to improve the performance of analgesic indices.</p>
      <p>Deep learning architectures, such as a convolution neural network (CNN), can be a good solution to elucidate the hidden features in a PPG because they can identify optimal abstracted features that are beyond human comprehension without any manual procedure [<xref ref-type="bibr" rid="ref8">8</xref>]. Furthermore, in determining the presence of pain, machine learning has a strong advantage owing to its nonlinear characteristics compared with the SPI, which assesses pain based on simple linear regression [<xref ref-type="bibr" rid="ref8">8</xref>,<xref ref-type="bibr" rid="ref9">9</xref>], potentially making it possible to effectively predict nonlinear deviations among individuals or situations [<xref ref-type="bibr" rid="ref10">10</xref>]. Therefore, a combination of the extended features of PPG and machine learning–based scoring is expected to overcome the limitations of existing pain assessment techniques. However, because a PPG is a 1D signal, whereas CNNs have the advantage of multidimensional data analysis, a dimensional extension of a PPG without loss of time–frequency characteristics is required to apply it optimally in a CNN. A spectrogram, which is a 2D image including the intact time–frequency characteristics of a PPG, can be a good method for applying a CNN to PPGs.</p>
      <p>This study aimed to develop a new analgesic index using PPG spectrograms and a CNN to objectively assess pain in conscious patients. In addition, the performance of our newly developed index was compared with that of the SPI.</p>
    </sec>
    <sec sec-type="methods">
      <title>Methods</title>
      <sec>
        <title>Patient Population</title>
        <p>The study protocol was approved by the Institutional Review Board of Asan Medical Centre (approval number: 2016-0477) and registered on an international clinical trials registry platform (registration number KCT0002080). Written informed consent was obtained from all patients. All procedures were conducted in accordance with relevant guidelines and regulations. In total, 120 patients (American Society of Anesthesiologists Physical Status 1, 2, or 3) between the ages of 20 and 80, who were scheduled to undergo elective surgery, were included in this observational study. Exclusion criteria were as follows: clinically significant impairment of the cardiovascular, hepatic, or renal function; history of cardiac arrhythmia; use of medication that might affect autonomic function; the presence of presurgical acute or chronic pain (Visual Analog Scale score [VAS] &#62; 0, measured before surgery); clinically significant laboratory findings; and evidence of pregnancy.</p>
      </sec>
      <sec>
        <title>Procedure and Data Acquisition</title>
        <p>All patients fasted from midnight on the day of surgery without premedication. In the operating theater, patients were monitored for their heart activity using electrocardiography, end-tidal carbon dioxide partial pressure, and noninvasive blood pressure measurement. Neuromuscular transmission was monitored using an M-NMT module at the adductor pollicis muscle (CARESCAPE B850; GE Healthcare). A reusable SPI sensor was placed on the index finger of each patient (on the arm not used for blood pressure measurement). Patients were allowed to acclimatize for at least 5 minutes in the supine position in a quiet operating theater, after which baseline data (without pain) were collected for 6 minutes, of which the latter 5 minutes were used for analysis. General anesthesia was performed by administering propofol and remifentanil by a target effect-site concentration–controlled infusion using the Schnider and Minto models [<xref ref-type="bibr" rid="ref11">11</xref>,<xref ref-type="bibr" rid="ref12">12</xref>]. Target effect-site concentrations (<italic>Ces</italic>) of propofol were titrated to maintain the bispectral index (Covidien) at less than 60 during the induction and maintenance of anesthesia. The target <italic>Ces</italic> of remifentanil were adjusted to maintain stable hemodynamics (ie, systolic blood pressure &#62;80 mmHg and heart rate over 45 beats/min). All patients received a bolus dose of oxycodone (0.1 mg/kg) 30 minutes prior to the end of surgery.</p>
        <p>Intravenous patient-controlled analgesia with oxycodone began after the administration of the bolus dose of oxycodone. Neuromuscular blockade was reversed with neostigmine and glycopyrrolate at the end of surgery. Tracheal extubation was performed when the train-of-four ratio was greater than 0.9 and bispectral index value was greater than 80. Patients were then transported to the postanesthesia care unit (PACU). When the patients arrived in the PACU, their state of consciousness was assessed with a modified Aldrete score [<xref ref-type="bibr" rid="ref13">13</xref>]. Electrocardiogram, pulse oximetry, and noninvasive blood pressure were also monitored. Additional PPG and SPI data were obtained for the initial 5 minutes in the PACU. After obtaining the data, patients were assessed for pain using a VAS (0=no pain; 100=the most severe pain). Oxycodone was administered according to postoperative pain intensity. The PPG and SPI values were measured using an S/5 Anesthesia Monitor (Datex-Ohmeda, Inc.) and recorded on a laptop for offline analysis. The PPG data were sampled at 300 Hz, and SPI data were recorded every 10 seconds.</p>
      </sec>
      <sec>
        <title>Pain Assessment Model</title>
        <p>A spectrogram–CNN model was developed and validated through fivefold cross-validation. The developed model outputs the spectrogram–CNN index as a pain score using a PPG spectrogram as input and CNN as a pain scorer. During model development, patients and test sets were separated to prevent intrasubject interference to avoid data overlaps between the development and test sets of each fold. For model training, 90% of the development set was used as a training set and 10% as a validation set. Finally, for each fold, 20, 73, and 7 patients’ data were used as a test set, training set, and validation set, respectively. Pain and nonpain data labels were created based on the VAS, where VAS&#62;0 was defined as pain and VAS=0 was defined as nonpain with labels “1” and “0,” respectively. Detailed descriptions of the spectrograms and CNN used in this study are given below.</p>
        <sec>
          <title>Spectrogram</title>
          <p>Spectrogram creation is a method used for time–frequency analysis of time series signals. A spectrogram reconstructs 2D images while maintaining the information contained in 1D time series data [<xref ref-type="bibr" rid="ref14">14</xref>]. Spectrograms are useful for visually describing changes in the frequency characteristics of nonstationary signals, such as physiological signals over time [<xref ref-type="bibr" rid="ref15">15</xref>,<xref ref-type="bibr" rid="ref16">16</xref>]. Spectrograms can be generated by repeating short-time Fourier transforms that divide a longer time signal into shorter segments of equal length and then computing the Fourier transforms separately on each shorter segment. In this study, 2D spectrogram images generated from 1D PPGs were used as the pain classifier input to reflect the whole waveform and change of waveform, not the specific feature of the PPG waveform. Prior to spectrogram generation, all PPGs were filtered using both a finite impulse response bandpass filter with a 0.5-10-Hz passband and a 30-tap moving average filter. In addition, considering that PPG amplitude is an arbitrary unit, a normalization process was performed to reduce the intersubject and intermeasurement deviations [<xref ref-type="bibr" rid="ref17">17</xref>]. In the normalization process, <italic>z</italic>-scores were obtained by subtracting the mean of the measured values from each measured value and dividing by the standard deviation. The spectrogram generation process is illustrated in <xref rid="figure1" ref-type="fig">Figure 1</xref>.</p>
          <p>Spectrogram images were generated by short-time Fourier transforms of 10-second PPGs every 10 seconds without overlap (<xref rid="figure1" ref-type="fig">Figure 1</xref>A). At the time, to generate a single spectrogram image, each 10-second PPG was divided into 6.3-second segments with a 6.27-second overlap and transformed to the frequency domain using fast Fourier transform after windowing with a Hamming window (<xref rid="figure1" ref-type="fig">Figure 1</xref>B). The frequency range of the spectrogram image was set to 0-10 Hz, and the frequency resolution was set to 0.81 Hz to equalize the number of time frames and frequency bins, that is, to equalize the numbers of horizontal and vertical pixels in the spectrogram image, respectively. Finally, 30 spectrogram patches of size 124 × 124 were generated for each 5-minute PPG. <xref rid="figure1" ref-type="fig">Figure 1</xref>C shows the averaged spectrograms without pain (left) and with pain (right). All preprocessing and spectrogram patches were generated using MATLAB (version 2018a; The MathWorks, Inc.).</p>
          <fig id="figure1" position="float">
            <label>Figure 1</label>
            <caption>
              <p>Process for generation of a single spectrogram (A), process of generating multiple spectrograms over time using a sliding window (B) and the average spectrogram during pain (left) and non-pain (right) conditions (C); FFT: fast Fourier transform; PPG: photoplethysmogram.</p>
            </caption>
            <graphic xlink:href="jmir_v23i2e23920_fig1.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
        <sec>
          <title>Convolutional Neural Network</title>
          <p>CNNs are useful for image analysis because they have the advantage of maintaining the spatial information of 2D or higher inputs. CNNs enable data-driven learning, are highly representative, and effectively combine the spatial information of multidimensional inputs [<xref ref-type="bibr" rid="ref18">18</xref>]; supervised CNNs extract information more effectively due to class-specific information [<xref ref-type="bibr" rid="ref19">19</xref>,<xref ref-type="bibr" rid="ref20">20</xref>]. The CNN used in this study has a 2D PPG spectrogram input and binary-coded labels: “0” is a pain-free state and “1” is pain state. <xref rid="figure2" ref-type="fig">Figure 2</xref> shows the structure of the CNN developed in this study. First, in the convolution-max pooling layer (Conv-Maxpool), 32, 64, and 128 filters are applied to the spectrogram input to perform a convolution process, and the spatial characteristics are simplified through max pooling. In Conv-Maxpool, the filter size of all convolutional layers is 2 × 2 and the stride is 1. The size of the feature map is reduced by setting the filter size of the max pooling layer to 2 × 2 and stride to 2, equal to the filter size. A batch normalization layer and rectified linear unit (ReLU) activation functions are applied to the Conv-Maxpool to increase learning speed and efficiency. Batch normalization is a structure that improves the speed and stability of neural networks by normalizing interlayer input data [<xref ref-type="bibr" rid="ref21">21</xref>], and ReLU improves the expressive power of neural networks based on nonlinear features [<xref ref-type="bibr" rid="ref22">22</xref>,<xref ref-type="bibr" rid="ref23">23</xref>]. The Conv-Maxpool process is repeated 3 times, resulting in a feature map size of 15 × 15. The fully connected layer consists of 2 hidden layers and 1 output layer, and the ReLU activation function and dropout are also applied to the hidden layer to reduce overfitting [<xref ref-type="bibr" rid="ref24">24</xref>]. The dropout rate was set to 0.5 in training, but there was no dropout during testing. Cross-entropy is employed as a cost function [<xref ref-type="bibr" rid="ref25">25</xref>], and adaptive moment estimation (Adam) is used as an optimizer [<xref ref-type="bibr" rid="ref26">26</xref>]. Finally, the result is output to 2 nodes, representing “pain free” and “pain,” and the values are probabilistically expressed using the SoftMax function [<xref ref-type="bibr" rid="ref27">27</xref>]. Because the output of SoftMax gives the probability of the input data being a painful condition with a value between 0 and 1, it is converted into a pain index as the “likelihood of pain.” Consequently, the spectrogram–CNN index is calculated by multiplying the probability value output from the pain node by 100. The CNN model was implemented and trained using Python 3.7 (Python Software Foundation) and TensorFlow 2.0 in the Anaconda environment.</p>
          <fig id="figure2" position="float">
            <label>Figure 2</label>
            <caption>
              <p>Architecture of the convolutional neural network proposed in this study. X: input, F: filter, C: convolution layer, M: max pooling, N: number of filters, BN: batch normalisation, ReLU: rectifier linear unit, Conv-Maxpool: convolution-max pooling.</p>
            </caption>
            <graphic xlink:href="jmir_v23i2e23920_fig2.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
          </fig>
        </sec>
      </sec>
      <sec>
        <title>Statistical Analyses</title>
        <p>Receiver-operating characteristic (ROC) curves were computed to compare the sensitivity and specificity of the spectrogram–CNN index and SPI for detecting pain. Cutoff values used for calculation of sensitivity and specificity were computed as “best fit” (highest combined sensitivity and specificity) [<xref ref-type="bibr" rid="ref28">28</xref>]. The differences between the spectrogram–CNN index and SPI in the ROC curves were calculated using the MedCalc Statistical Software (version 13.3.1; MedCalc Software). Statistical analyses were conducted using IBM SPSS (version 22.0; SPSS Inc.). Data are expressed as mean (SD) for normally distributed continuous variables, median (25%-75%) for non-normally distributed continuous variables, and count SPI for categorical variables. <italic>P</italic> values &#60;.05 were considered to be statistically significant.</p>
      </sec>
    </sec>
    <sec sec-type="results">
      <title>Results</title>
      <p>In total, 120 patients were enrolled, of whom 20 dropped out because of failure of PPG data storage (n=8), failure of SPI data storage (n=2), abnormal SPI data that could not be included in the analysis (n=9), and failure to measure VAS after surgery (n=1). Thus, 100 patients were included in the final analysis. The characteristics of these patients are summarized in <xref ref-type="table" rid="table1">Table 1</xref>. All but 1 patient had consciousness values of 2 points (conscious), as assessed by the modified Aldrete score at the time of PACU arrival. One patient scored 1 (arousable on calling) who later had a value of 2 points upon leaving the PACU.</p>
      <p>Individual changes in the spectrogram–CNN index and SPI without and with pain are presented in <xref rid="figure3" ref-type="fig">Figure 3</xref>. The 7 patients who had no postoperative pain were excluded from this analysis. In the case of pain, the mean spectrogram–CNN index and SPI values increased significantly (baseline spectrogram–CNN index: 28.5 (SD 22.1) versus PACU spectrogram–CNN index: 65.7 (SD 25.4), <italic>P</italic>&#60;.01 in paired <italic>t</italic> test; baseline SPI: 42.5 (SD 16.7) versus PACU SPI: 53.5 (SD 17.8), <italic>P</italic>&#60;.01 in paired <italic>t</italic> test). The area under the curve (AUC) of the ROC and cutoff values for detecting pain in terms of spectrogram–CNN index and SPI are listed in <xref ref-type="table" rid="table2">Table 2</xref>. The spectrogram–CNN index was statistically superior to the SPI (pairwise comparison of ROC curves: spectrogram–CNN index versus SPI, <italic>P</italic>&#60;.01). Moreover, as shown in <xref ref-type="table" rid="table2">Table 2</xref> and <xref rid="figure4" ref-type="fig">Figure 4</xref>, the spectrogram–CNN index showed improved performance measures in terms of balanced accuracy, sensitivity, and especially specificity.</p>
      <table-wrap position="float" id="table1">
        <label>Table 1</label>
        <caption>
          <p>Characteristics of the study population.</p>
        </caption>
        <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
          <col width="30"/>
          <col width="670"/>
          <col width="300"/>
          <thead>
            <tr valign="top">
              <td colspan="2">Characteristic</td>
              <td>Patients (N=100)</td>
            </tr>
          </thead>
          <tbody>
            <tr valign="top">
              <td colspan="2">Male/Female</td>
              <td>44/56</td>
            </tr>
            <tr valign="top">
              <td colspan="2">Age (years), mean (SD)</td>
              <td>53.4 (12.5)</td>
            </tr>
            <tr valign="top">
              <td colspan="2">Height (cm), mean (SD)</td>
              <td>161.8 (8.5)</td>
            </tr>
            <tr valign="top">
              <td colspan="2">Weight (kg), mean (SD)</td>
              <td>62.2 (12.2)</td>
            </tr>
            <tr valign="top">
              <td colspan="2">ASA PS<sup>a</sup> 1/2/3</td>
              <td>22/76/2</td>
            </tr>
            <tr valign="top">
              <td colspan="2">
                <bold>Type of surgery, n</bold>
              </td>
              <td>
                <break/>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Breast</td>
              <td>29</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Colorectal</td>
              <td>19</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Hepatobiliary</td>
              <td>12</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Stomach</td>
              <td>30</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Thyroid</td>
              <td>10</td>
            </tr>
            <tr valign="top">
              <td colspan="2">
                <bold>Postoperative pain intensity at PACU<sup>b</sup>, n</bold>
              </td>
              <td>
                <break/>
              </td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>No (VAS<sup>c</sup>=0)</td>
              <td>7</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Mild (0 &#60; VAS ≤ 30)</td>
              <td>11</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Moderate (30 &#60; VAS ≤ 70)</td>
              <td>58</td>
            </tr>
            <tr valign="top">
              <td>
                <break/>
              </td>
              <td>Severe (70 &#60; VAS ≤ 100)</td>
              <td>24</td>
            </tr>
          </tbody>
        </table>
        <table-wrap-foot>
          <fn id="table1fn1">
            <p><sup>a</sup>ASA PS: American Society of Anesthesiologists Physical Status.</p>
          </fn>
          <fn id="table1fn2">
            <p><sup>b</sup>PACU: postanesthesia care unit.</p>
          </fn>
          <fn id="table1fn3">
            <p><sup>c</sup>VAS: Visual Analog Scale (0=no pain; 100=the most severe pain).</p>
          </fn>
        </table-wrap-foot>
      </table-wrap>
      <fig id="figure3" position="float">
        <label>Figure 3</label>
        <caption>
          <p>Individual changes (n=100) in the spectrogram-convolutional neural network index (SCI, A) and the Surgical Pleth Index (SPI, B) and without and with pain. *<italic>P</italic>&#60;.05 vs. baseline. The black circles represent the average of 5 min of the SCI or SPI observed without and with pain. The red circles indicate mean values for all patients. PACU: postanaesthesia care unit.</p>
        </caption>
        <graphic xlink:href="jmir_v23i2e23920_fig3.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <table-wrap position="float" id="table2">
        <label>Table 2</label>
        <caption>
          <p>Areas under the receiver-operating characteristic curves (AUCs) and cutoff values for assessing pain in the spectrogram-convolutional neural network index and surgical pleth index (SPI) in surgical patients.</p>
        </caption>
        <table width="1000" cellpadding="5" cellspacing="0" border="1" rules="groups" frame="hsides">
          <col width="266"/>
          <col width="196"/>
          <col width="172"/>
          <col width="188"/>
          <col width="178"/>
          <thead>
            <tr valign="top">
              <td rowspan="2">Parameter</td>
              <td colspan="3">Spectrogram–CNN<sup>a</sup> index</td>
              <td>SPI</td>
            </tr>
            <tr valign="top">
              <td>Training set</td>
              <td>Validation set</td>
              <td>Test set</td>
              <td>
                <break/>
              </td>
            </tr>
          </thead>
          <tbody>
            <tr valign="top">
              <td>AUC (95% CI)</td>
              <td>0.992 (0.991-0.993)</td>
              <td>0.932 (0.921-0.942)</td>
              <td>0.757 (0.746-0.768)</td>
              <td>0.659 (0.646-0.671)</td>
            </tr>
            <tr valign="top">
              <td><italic>P</italic> value</td>
              <td>&#60;.01</td>
              <td>&#60;.01</td>
              <td>&#60;.01</td>
              <td>&#60;.01</td>
            </tr>
            <tr valign="top">
              <td>Cutoff value<sup>b</sup></td>
              <td>50</td>
              <td>54</td>
              <td>48</td>
              <td>44</td>
            </tr>
            <tr valign="top">
              <td>Sensitivity, %, specificity, %</td>
              <td>94.6, 96.4</td>
              <td>83.1, 88.2</td>
              <td>68.1, 73.8</td>
              <td>65.2, 59.5</td>
            </tr>
            <tr valign="top">
              <td>Balanced accuracy, %<sup>c</sup></td>
              <td>95.5</td>
              <td>85.7</td>
              <td>71.0</td>
              <td>62.4</td>
            </tr>
          </tbody>
        </table>
        <table-wrap-foot>
          <fn id="table2fn1">
            <p><sup>a</sup>CNN: convolutional neural network.</p>
          </fn>
          <fn id="table2fn2">
            <p><sup>b</sup>Cutoff values used for the calculation of sensitivity and specificity were calculated as “best fit” (highest combined sensitivity and specificity).</p>
          </fn>
          <fn id="table2fn3">
            <p><sup>c</sup>Balanced accuracy is the corrected accuracy of the imbalance of a class set, calculated as (sensitivity + specificity)/2.</p>
          </fn>
        </table-wrap-foot>
      </table-wrap>
      <fig id="figure4" position="float">
        <label>Figure 4</label>
        <caption>
          <p>Performance measures of spectrogram-convolutional neural network index (SCI, red) and the Surgical Pleth Index (SPI, blue) in terms of accuracy, sensitivity and specificity. Accuracy means balanced accuracy. Cut-off value of SCI was 48.</p>
        </caption>
        <graphic xlink:href="jmir_v23i2e23920_fig4.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
      <p>Neither spectrogram–CNN index (mild: 69.5 [53.2-87.7], moderate: 71.8 [40.5-89.3], severe: 74.6 [62.9-84.7], <italic>P</italic>=.78, Kruskal–Wallis one-way ANOVA on ranks) nor SPI (mild: 53.5 [SD 21.0], moderate: 51.8 [SD 18.0], severe: 57.9 [SD 15.7], <italic>P</italic>=.37; one-way ANOVA) could statistically distinguish between mild, moderate, and severe pain. The frequency distributions of the spectrogram–CNN index and SPI values observed without and with pain during the data collection period are shown in <xref rid="figure5" ref-type="fig">Figure 5</xref>. The distribution of SPI values overlapped for with and without pain, suggesting that the SPI shows several false positives/false negatives, whereas the distribution of spectrogram–CNN index values showed a significant difference (<italic>P</italic>&#60;.05) in patients with and without pain, suggesting that the spectrogram–CNN index can distinguish pain more clearly than the SPI.</p>
      <fig id="figure5" position="float">
        <label>Figure 5</label>
        <caption>
          <p>Frequency distribution of the spectrogram-convolutional neural network index (SCI, A and B) and surgical pleth index (SPI, C and D) values observed without (A and C) and with (B and D) pain. During the data collection period (baseline: 5 min, postanaesthesia care unit: 5 min), the SCI and SPI values were observed every 10 sec. The vertical red lines show the median frequency (A: 11, B: 82, C: 32, D: 53).</p>
        </caption>
        <graphic xlink:href="jmir_v23i2e23920_fig5.png" alt-version="no" mimetype="image" position="float" xlink:type="simple"/>
      </fig>
    </sec>
    <sec sec-type="discussion">
      <title>Discussion</title>
      <p>The spectrogram–CNN index proposed in this study outperformed the commercialized SPI pain index in the postoperative pain assessment of conscious patients. One of the main reasons for its outperformance is the spectrogram input containing the PPG’s intact waveform information, making it possible to use hidden pain-related factors that could not be provided only by the peaks. Because existing pain assessment methods, such as the SPI, use only certain features of a PPG that show significant changes in surgical stimuli, numerous pain-related information reflected in the PPG may be overlooked. To overcome these limitations, research has been conducted on new pain-related features derived from sophisticated PPG waveform analysis in addition to the heart rate interval and PPG amplitude reflected in the SPI [<xref ref-type="bibr" rid="ref5">5</xref>-<xref ref-type="bibr" rid="ref7">7</xref>]. However, these pain indicators still depend on complex processes, such as peak detection and feature extraction, which require accurate peak extraction algorithms and are vulnerable to signal quality degradation. The proposed PPG spectrogram input does not require any peak detection or feature extraction process, thus avoiding problems such as peak misdetection during preprocessing. In addition, the spectrogram provides information from almost the whole PPG waveform because only the domain in which information is represented is transformed, while the underlying information is retained. This feature of the proposed model is its differentiating factor from SPI, which requires a complicated process of extracting pain features from biosignals, including PPG pulsation start and systolic maximum detection and verification. These preprocessing steps are necessary in existing pain assessment methods, but they are cumbersome and vulnerable, providing only limited features. Therefore, a simplified preprocessing process that still provides plentiful feature information can significantly improve the robustness of pain assessment.</p>
      <p>Another key technique proposed in this study is the discrimination of postoperative pain using machine learning. Although machine learning–based pain assessment has already been studied [<xref ref-type="bibr" rid="ref29">29</xref>-<xref ref-type="bibr" rid="ref31">31</xref>], it is not suitable for practical clinical situations because it depends on high-dimensional clinical data, such as patient records and electroencephalograms, which are rarely used in postoperative care. However, machine learning from PPGs, which are frequently used in clinical practice during postoperative care, has high practical utility. In this study, we used a spectrogram–CNN combination, which has already been applied to electrocardiograms and electroencephalograms and has shown reasonable performance in predicting seizures and atrial fibrillation [<xref ref-type="bibr" rid="ref32">32</xref>-<xref ref-type="bibr" rid="ref35">35</xref>]. The spectrogram converts the data into 2D, and the CNN has the advantage of extracting the spatial features of multidimensional data. The combination of these techniques thus extends the dimensions of PPG and allows spatiotemporal analysis, maximizing the use of features inherent in the signal. Nonlinear classification may be another important reason for the good performance of the proposed model. While the SPI is derived from a simple linear combination of normalized heartbeat interval and normalized pulse wave amplitude [<xref ref-type="bibr" rid="ref3">3</xref>,<xref ref-type="bibr" rid="ref36">36</xref>], the proposed spectrogram–CNN model performs nonlinear classification using the ReLU activation function. In addition, the nonlinearity is increased because the ReLUs are overlapped with each other in a multilayered structure.</p>
      <p>In this study, patient data used for model development and validation were separated, and fivefold cross-validation was performed to eliminate interindividual interference and to generalize the model. Therefore, the proposed spectrogram–CNN index is expected to show similar performance in other groups of patients of similar age who did not participate in model development. However, there may be a few new variables to consider. In <xref ref-type="table" rid="table2">Table 2</xref>, performance measures were approximately 95%, approximately 85%, and approximately 70% for the training, validation, and test sets, respectively. The decision criteria also differed with 50 in the training set, 54 in the validation set, and 48 in the test set, indicating that overfitting occurred. In a random permutation test on balanced accuracy, the average accuracy was 49.5 (SD 0.7), and the range of values was 47.6-51.8. Considering that all balanced accuracies of the training, validation, and test sets were over 71, it is likely that the overfitting in our result stemmed from large intersubject variability rather than the model itself. Overfitting can be interpreted as degradation of the model’s versatility, but it can also be interpreted as higher performance, at least in terms of validation accuracy, if sufficient data are used. Therefore, further studies are required to improve the reliability and versatility of this index using a large patient population with diverse body characteristics.</p>
      <p>The spectrogram–CNN index may not have been able to distinguish pain intensity because the number of observations was relatively small. A previous study showed that SPI distinguishes postoperative pain intensity [<xref ref-type="bibr" rid="ref28">28</xref>]; however, the authors of this previous study analyzed 1300 observations, whereas we only used 93. Nonetheless, a more fundamental reason is that the severity of pain was not accounted for when developing the spectrogram–CNN index. If sufficient PPG data were provided when learning to classify pain severity, it would be possible to classify pain intensity.</p>
      <p>The database used in this study was shared by 2 research groups. Another group, independent of this study, has already published their results. Their pain classifier, based on a deep belief network using various PPG features, discriminated well between the presence and absence of pain [<xref ref-type="bibr" rid="ref37">37</xref>]. Compared to the other study that extracted various features, ours used a simple spectrogram containing all PPG information without any complicated feature extraction procedure and is based on a CNN optimized for the spectrogram input. CNNs have lead in the machine learning field because they demonstrated performance improvements in image recognition in 2012 [<xref ref-type="bibr" rid="ref9">9</xref>]. Moreover, the previous study evaluated pain based on full-length data, whereas ours assessed pain every 10 seconds, the same data display interval as in the SPI. Therefore, it can be applied to real-time pain assessment, which is in contrast to the other study.</p>
      <p>There are some limitations to this study. First, the SPI was selected as a comparative index to evaluate the performance of the spectrogram–CNN index, but its suitability is somewhat debated. The SPI is neither developed nor recommended for use in conscious patients. However, when a new analgesic index is developed, it is essential to evaluate its performance, and this is commonly done by comparing it with an existing index using the same data. Although the SPI is not recommended for use in conscious patients, some studies on conscious patients suggest that SPI can discriminate between the presence and absence of pain [<xref ref-type="bibr" rid="ref28">28</xref>,<xref ref-type="bibr" rid="ref38">38</xref>]. Between the SPI and another commonly used commercial analgesic index—the Analgesia Nociception Index (PhysioDoloris, MetroDoloris)—the AUC–ROC for detecting postoperative pain in conscious patients was highest for the SPI [<xref ref-type="bibr" rid="ref38">38</xref>]. Hence, the SPI was chosen as the comparative index for this study. Second, neostigmine and glycopyrrolate, when administered to reverse neuromuscular blockade, can contribute to PPG signals. As this study was observational, only data necessary for the development of a new analgesic index were collected during the normal anesthesia process without intervention. Neostigmine and glycopyrrolate were used in all patients because none of them required sugammadex. Neostigmine is known to be rapidly eliminated from the plasma after administration, with an average half-life of approximately 25 minutes [<xref ref-type="bibr" rid="ref39">39</xref>]. We collected postoperative PPG data an average of 29.4 minutes after administration of these 2 agents. A previous study reported that baroreflex sensitivity was restored to its baseline value after approximately 82 minutes of glycopyrrolate administration [<xref ref-type="bibr" rid="ref40">40</xref>]. It is possible that glycopyrrolate has mixed effects on postoperative PPG. However, because sugammadex usage is not common, it may be more beneficial to develop an index to distinguish pain based on data that can be obtained from actual practical conditions. Further studies are required to evaluate the extent of the effect of these 2 drugs on postoperative PPG. Third, it is difficult to determine whether the PPG data collected from the PACU solely reflect pain. In the conscious state, the sympathetic nervous system may be activated for other reasons, such as arousal or anxiety. Anxiety has been associated with reduced heart rate variability and vagal tone [<xref ref-type="bibr" rid="ref41">41</xref>]. As we did not evaluate patient anxiety, we cannot determine its contribution to the PPG data. Considering the condition of patients who arrived in the PACU immediately after surgery, the PPG data mostly reflected immediate postoperative pain without controlling consciousness.</p>
      <p>In conclusion, although there were several limitations to the study design, we confirmed that the newly developed spectrogram–CNN index can effectively detect postoperative pain in conscious patients. Further validation studies are required to assess its feasibility and prevent overfitting to various populations, including patients under general anesthesia.</p>
    </sec>
  </body>
  <back>
    <app-group/>
    <glossary>
      <title>Abbreviations</title>
      <def-list>
        <def-item>
          <term id="abb1">AUC</term>
          <def>
            <p>area under the curve</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb2">CNN</term>
          <def>
            <p>convolutional neural network</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb3">PPG</term>
          <def>
            <p>photoplethysmogram</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb4">ReLU</term>
          <def>
            <p>rectified linear unit</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb5">SPI</term>
          <def>
            <p>surgical pleth index</p>
          </def>
        </def-item>
        <def-item>
          <term id="abb6">VAS</term>
          <def>
            <p>Visual Analog Scale</p>
          </def>
        </def-item>
      </def-list>
    </glossary>
    <ack>
      <p>This research was supported by two grants from the Basic Science Research Program through the National Research Foundation of Korea (NRF), one funded by the Ministry of Science and ICT (NRF-2018R1A4A1025704), and one funded by the Ministry of Education (NRF-2018R1D1A3B07046442), Republic of Korea.</p>
    </ack>
    <fn-group>
      <fn fn-type="conflict">
        <p>None declared.</p>
      </fn>
    </fn-group>
    <ref-list>
      <ref id="ref1">
        <label>1</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Chou</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Gordon</surname>
              <given-names>DB</given-names>
            </name>
            <name name-style="western">
              <surname>de</surname>
              <given-names>LOA</given-names>
            </name>
            <name name-style="western">
              <surname>Rosenberg</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Bickler</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Brennan</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Carter</surname>
              <given-names>T</given-names>
            </name>
            <name name-style="western">
              <surname>Cassidy</surname>
              <given-names>CL</given-names>
            </name>
            <name name-style="western">
              <surname>Chittenden</surname>
              <given-names>EH</given-names>
            </name>
            <name name-style="western">
              <surname>Degenhardt</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Griffith</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Manworren</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>McCarberg</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Montgomery</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Murphy</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Perkal</surname>
              <given-names>MF</given-names>
            </name>
            <name name-style="western">
              <surname>Suresh</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sluka</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Strassels</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Thirlby</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Viscusi</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Walco</surname>
              <given-names>GA</given-names>
            </name>
            <name name-style="western">
              <surname>Warner</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Weisman</surname>
              <given-names>SJ</given-names>
            </name>
            <name name-style="western">
              <surname>Wu</surname>
              <given-names>CL</given-names>
            </name>
          </person-group>
          <article-title>Management of postoperative pain: a clinical practice guideline from the American Pain Society, the American Society of Regional Anesthesia and Pain Medicine, and the American Society of Anesthesiologists' Committee on Regional Anesthesia, Executive Committee, and Administrative Council</article-title>
          <source>J Pain</source>
          <year>2016</year>
          <month>02</month>
          <volume>17</volume>
          <issue>2</issue>
          <fpage>131</fpage>
          <lpage>57</lpage>
          <pub-id pub-id-type="doi">10.1016/j.jpain.2015.12.008</pub-id>
          <pub-id pub-id-type="medline">26827847</pub-id>
          <pub-id pub-id-type="pii">S1526-5900(15)00995-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref2">
        <label>2</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Logier</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Jeanne</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>De Jonckheere</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Dassonneville</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Delecroix</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Tavernier</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>PhysioDoloris: a monitoring device for analgesia / nociception balance evaluation using heart rate variability analysis</article-title>
          <source>Annu Int Conf IEEE Eng Med Biol Soc</source>
          <year>2010</year>
          <volume>2010</volume>
          <fpage>1194</fpage>
          <lpage>7</lpage>
          <pub-id pub-id-type="doi">10.1109/IEMBS.2010.5625971</pub-id>
          <pub-id pub-id-type="medline">21095676</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref3">
        <label>3</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Huiku</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Uutela</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>van Gils</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Korhonen</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Kymäläinen</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Meriläinen</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Paloheimo</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Rantanen</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Takala</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Viertiö-Oja</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Yli-Hankala</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Assessment of surgical stress during general anaesthesia</article-title>
          <source>Br J Anaesth</source>
          <year>2007</year>
          <month>05</month>
          <volume>98</volume>
          <issue>4</issue>
          <fpage>447</fpage>
          <lpage>55</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0007-0912(17)35495-8"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/bja/aem004</pub-id>
          <pub-id pub-id-type="medline">17329347</pub-id>
          <pub-id pub-id-type="pii">S0007-0912(17)35495-8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref4">
        <label>4</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ledowski</surname>
              <given-names>T</given-names>
            </name>
          </person-group>
          <article-title>Objective monitoring of nociception: a review of current commercial solutions</article-title>
          <source>Br J Anaesth</source>
          <year>2019</year>
          <month>08</month>
          <volume>123</volume>
          <issue>2</issue>
          <fpage>e312</fpage>
          <lpage>e321</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/31047645"/>
          </comment>
          <pub-id pub-id-type="doi">10.1016/j.bja.2019.03.024</pub-id>
          <pub-id pub-id-type="medline">31047645</pub-id>
          <pub-id pub-id-type="pii">S0007-0912(19)30228-4</pub-id>
          <pub-id pub-id-type="pmcid">PMC6676047</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref5">
        <label>5</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Park</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jeon</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Shin</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Noh</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Performance of the nasal photoplethysmographic index as an analgesic index during surgery under general anaesthesia</article-title>
          <source>Sci Rep</source>
          <year>2020</year>
          <month>04</month>
          <day>28</day>
          <volume>10</volume>
          <issue>1</issue>
          <fpage>7130</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.1038/s41598-020-64033-0"/>
          </comment>
          <pub-id pub-id-type="doi">10.1038/s41598-020-64033-0</pub-id>
          <pub-id pub-id-type="medline">32346057</pub-id>
          <pub-id pub-id-type="pii">10.1038/s41598-020-64033-0</pub-id>
          <pub-id pub-id-type="pmcid">PMC7188862</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref6">
        <label>6</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>YL</given-names>
            </name>
            <name name-style="western">
              <surname>Seok</surname>
              <given-names>HS</given-names>
            </name>
            <name name-style="western">
              <surname>Noh</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Shin</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Postoperative Pain Assessment Indices Based on Photoplethysmography Waveform Analysis</article-title>
          <source>Front Physiol</source>
          <year>2018</year>
          <volume>9</volume>
          <fpage>1199</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.3389/fphys.2018.01199"/>
          </comment>
          <pub-id pub-id-type="doi">10.3389/fphys.2018.01199</pub-id>
          <pub-id pub-id-type="medline">30210363</pub-id>
          <pub-id pub-id-type="pmcid">PMC6121033</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref7">
        <label>7</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Seok</surname>
              <given-names>HS</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Noh</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Shin</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Postoperative Pain Assessment Model Based on Pulse Contour Characteristics Analysis</article-title>
          <source>IEEE J Biomed Health Inform</source>
          <year>2019</year>
          <month>11</month>
          <volume>23</volume>
          <issue>6</issue>
          <fpage>2317</fpage>
          <lpage>2324</lpage>
          <pub-id pub-id-type="doi">10.1109/JBHI.2018.2890482</pub-id>
          <pub-id pub-id-type="medline">30605112</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref8">
        <label>8</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>LeCun</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Bengio</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Hinton</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Deep learning</article-title>
          <source>Nature</source>
          <year>2015</year>
          <month>05</month>
          <day>28</day>
          <volume>521</volume>
          <issue>7553</issue>
          <fpage>436</fpage>
          <lpage>44</lpage>
          <pub-id pub-id-type="doi">10.1038/nature14539</pub-id>
          <pub-id pub-id-type="medline">26017442</pub-id>
          <pub-id pub-id-type="pii">nature14539</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref9">
        <label>9</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Krizhevsky</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sutskever</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Hinton</surname>
              <given-names>GE</given-names>
            </name>
          </person-group>
          <article-title>ImageNet classification with deep convolutional neural networks</article-title>
          <source>Commun. ACM</source>
          <year>2017</year>
          <month>6</month>
          <volume>60</volume>
          <issue>6</issue>
          <fpage>84</fpage>
          <lpage>90</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dl.acm.org/doi/pdf/10.1145/3065386"/>
          </comment>
          <pub-id pub-id-type="doi">10.1145/3065386</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref10">
        <label>10</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Pham</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <source>Springer Handbook of Engineering Statistics</source>
          <year>2006</year>
          <publisher-loc>Cham, Switzerland</publisher-loc>
          <publisher-name>Springer Science &#38; Business Media</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref11">
        <label>11</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Schnider</surname>
              <given-names>TW</given-names>
            </name>
            <name name-style="western">
              <surname>Minto</surname>
              <given-names>CF</given-names>
            </name>
            <name name-style="western">
              <surname>Gambus</surname>
              <given-names>PL</given-names>
            </name>
            <name name-style="western">
              <surname>Andresen</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Goodale</surname>
              <given-names>DB</given-names>
            </name>
            <name name-style="western">
              <surname>Shafer</surname>
              <given-names>SL</given-names>
            </name>
            <name name-style="western">
              <surname>Youngs</surname>
              <given-names>EJ</given-names>
            </name>
          </person-group>
          <article-title>The influence of method of administration and covariates on the pharmacokinetics of propofol in adult volunteers</article-title>
          <source>Anesthesiology</source>
          <year>1998</year>
          <month>05</month>
          <volume>88</volume>
          <issue>5</issue>
          <fpage>1170</fpage>
          <lpage>82</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pubs.asahq.org/anesthesiology/article-lookup/doi/10.1097/00000542-199805000-00006"/>
          </comment>
          <pub-id pub-id-type="doi">10.1097/00000542-199805000-00006</pub-id>
          <pub-id pub-id-type="medline">9605675</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref12">
        <label>12</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Minto</surname>
              <given-names>CF</given-names>
            </name>
            <name name-style="western">
              <surname>Schnider</surname>
              <given-names>TW</given-names>
            </name>
            <name name-style="western">
              <surname>Egan</surname>
              <given-names>TD</given-names>
            </name>
            <name name-style="western">
              <surname>Youngs</surname>
              <given-names>E</given-names>
            </name>
            <name name-style="western">
              <surname>Lemmens</surname>
              <given-names>HJ</given-names>
            </name>
            <name name-style="western">
              <surname>Gambus</surname>
              <given-names>PL</given-names>
            </name>
            <name name-style="western">
              <surname>Billard</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Hoke</surname>
              <given-names>JF</given-names>
            </name>
            <name name-style="western">
              <surname>Moore</surname>
              <given-names>KH</given-names>
            </name>
            <name name-style="western">
              <surname>Hermann</surname>
              <given-names>DJ</given-names>
            </name>
            <name name-style="western">
              <surname>Muir</surname>
              <given-names>KT</given-names>
            </name>
            <name name-style="western">
              <surname>Mandema</surname>
              <given-names>JW</given-names>
            </name>
            <name name-style="western">
              <surname>Shafer</surname>
              <given-names>SL</given-names>
            </name>
          </person-group>
          <article-title>Influence of age and gender on the pharmacokinetics and pharmacodynamics of remifentanil. I. Model development</article-title>
          <source>Anesthesiology</source>
          <year>1997</year>
          <month>01</month>
          <volume>86</volume>
          <issue>1</issue>
          <fpage>10</fpage>
          <lpage>23</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://pubs.asahq.org/anesthesiology/article-lookup/doi/10.1097/00000542-199701000-00004"/>
          </comment>
          <pub-id pub-id-type="doi">10.1097/00000542-199701000-00004</pub-id>
          <pub-id pub-id-type="medline">9009935</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref13">
        <label>13</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Aldrete</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>Modifications to the postanesthesia score for use in ambulatory surgery</article-title>
          <source>J Perianesth Nurs</source>
          <year>1998</year>
          <month>06</month>
          <volume>13</volume>
          <issue>3</issue>
          <fpage>148</fpage>
          <lpage>55</lpage>
          <pub-id pub-id-type="doi">10.1016/s1089-9472(98)80044-0</pub-id>
          <pub-id pub-id-type="medline">9801540</pub-id>
          <pub-id pub-id-type="pii">S108994729800015X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref14">
        <label>14</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Flanagan</surname>
              <given-names>JL</given-names>
            </name>
          </person-group>
          <source>Speech Analysis Synthesis and Perception</source>
          <year>2013</year>
          <publisher-loc>Cham, Switzerland</publisher-loc>
          <publisher-name>Springer Science &#38; Business Media</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref15">
        <label>15</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>W</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Yang</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Balaji</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Elamaran</surname>
              <given-names>V</given-names>
            </name>
            <name name-style="western">
              <surname>Arunkumar</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Revisiting signal processing with spectrogram analysis on EEG, ECG and speech signals</article-title>
          <source>Future Generation Computer Systems</source>
          <year>2019</year>
          <month>09</month>
          <volume>98</volume>
          <fpage>227</fpage>
          <lpage>232</lpage>
          <pub-id pub-id-type="doi">10.1016/j.future.2018.12.060</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref16">
        <label>16</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Huang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Chen</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Yao</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>He</surname>
              <given-names>W</given-names>
            </name>
          </person-group>
          <article-title>ECG Arrhythmia Classification Using STFT-Based Spectrogram and Convolutional Neural Network</article-title>
          <source>IEEE Access</source>
          <year>2019</year>
          <volume>7</volume>
          <fpage>92871</fpage>
          <lpage>92880</lpage>
          <pub-id pub-id-type="doi">10.1109/access.2019.2928017</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref17">
        <label>17</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Alian</surname>
              <given-names>AA</given-names>
            </name>
            <name name-style="western">
              <surname>Shelley</surname>
              <given-names>KH</given-names>
            </name>
          </person-group>
          <article-title>Photoplethysmography</article-title>
          <source>Best Pract Res Clin Anaesthesiol</source>
          <year>2014</year>
          <month>12</month>
          <volume>28</volume>
          <issue>4</issue>
          <fpage>395</fpage>
          <lpage>406</lpage>
          <pub-id pub-id-type="doi">10.1016/j.bpa.2014.08.006</pub-id>
          <pub-id pub-id-type="medline">25480769</pub-id>
          <pub-id pub-id-type="pii">S1521-6896(14)00075-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref18">
        <label>18</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Shin</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Roth</surname>
              <given-names>HR</given-names>
            </name>
            <name name-style="western">
              <surname>Gao</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Lu</surname>
              <given-names>L</given-names>
            </name>
            <name name-style="western">
              <surname>Xu</surname>
              <given-names>Z</given-names>
            </name>
            <name name-style="western">
              <surname>Nogues</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Yao</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Mollura</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Summers</surname>
              <given-names>RM</given-names>
            </name>
          </person-group>
          <article-title>Deep Convolutional Neural Networks for Computer-Aided Detection: CNN Architectures, Dataset Characteristics and Transfer Learning</article-title>
          <source>IEEE Trans Med Imaging</source>
          <year>2016</year>
          <month>05</month>
          <volume>35</volume>
          <issue>5</issue>
          <fpage>1285</fpage>
          <lpage>98</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26886976"/>
          </comment>
          <pub-id pub-id-type="doi">10.1109/TMI.2016.2528162</pub-id>
          <pub-id pub-id-type="medline">26886976</pub-id>
          <pub-id pub-id-type="pmcid">PMC4890616</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref19">
        <label>19</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Romero</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Gatta</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Camps-Valls</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Unsupervised Deep Feature Extraction for Remote Sensing Image Classification</article-title>
          <source>IEEE Trans. Geosci. Remote Sensing</source>
          <year>2016</year>
          <month>3</month>
          <volume>54</volume>
          <issue>3</issue>
          <fpage>1349</fpage>
          <lpage>1362</lpage>
          <pub-id pub-id-type="doi">10.1109/tgrs.2015.2478379</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref20">
        <label>20</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Soleymani</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dabouei</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kazemi</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Dawson</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Nasrabadi</surname>
              <given-names>N</given-names>
            </name>
          </person-group>
          <article-title>Multi-level feature abstraction from convolutional neural networks for multimodal biometric identification</article-title>
          <year>2018</year>
          <conf-name>24th International Conference on Pattern Recognition (ICPR)</conf-name>
          <conf-date>August 20-24, 2018</conf-date>
          <conf-loc>Beijing, China</conf-loc>
          <publisher-loc>New York, NY</publisher-loc>
          <publisher-name>IEEE</publisher-name>
          <pub-id pub-id-type="doi">10.1109/icpr.2018.8545061</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref21">
        <label>21</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ioffe</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Szegedy</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Batch normalization: accelerating deep network training by reducing internal covariate shift</article-title>
          <source>arXiv</source>
          <year>2021</year>
        </nlm-citation>
      </ref>
      <ref id="ref22">
        <label>22</label>
        <nlm-citation citation-type="confproc">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Raghu</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Poole</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Kleinberg</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ganguli</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Dickstein</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>On the expressive power of deep neural networks</article-title>
          <source>Proceedings of the 34th International Conference on Machine Learning (Volume 70)</source>
          <year>2017</year>
          <conf-name>34th International Conference on Machine Learning</conf-name>
          <conf-date>August 2017</conf-date>
          <conf-loc>Sydney, Australia</conf-loc>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://dl.acm.org/doi/pdf/10.5555/3305890.3305975"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref23">
        <label>23</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Arora</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Basu</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Mianjy</surname>
              <given-names>P</given-names>
            </name>
            <name name-style="western">
              <surname>Mukherjee</surname>
              <given-names>A</given-names>
            </name>
          </person-group>
          <article-title>Understanding deep neural networks with rectified linear units</article-title>
          <source>arXiv</source>
          <year>2021</year>
        </nlm-citation>
      </ref>
      <ref id="ref24">
        <label>24</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Srivastava</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Hinton</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Krizhevsky</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Sutskever</surname>
              <given-names>I</given-names>
            </name>
            <name name-style="western">
              <surname>Salakhutdinov</surname>
              <given-names>R</given-names>
            </name>
          </person-group>
          <article-title>Dropout: a simple way to prevent neural networks from overfitting</article-title>
          <source>Journal of Machine Learning Research</source>
          <year>2014</year>
          <volume>15</volume>
          <issue>1</issue>
          <fpage>1929</fpage>
          <lpage>1958</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://jmlr.org/papers/volume15/srivastava14a/srivastava14a.pdf"/>
          </comment>
        </nlm-citation>
      </ref>
      <ref id="ref25">
        <label>25</label>
        <nlm-citation citation-type="book">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Rubinstein</surname>
              <given-names>RY</given-names>
            </name>
            <name name-style="western">
              <surname>Kroese</surname>
              <given-names>DP</given-names>
            </name>
          </person-group>
          <source>The Cross-Entropy Method: A Unified Approach to Combinatorial Optimization, Monte-Carlo Simulation and Machine Learning</source>
          <year>2013</year>
          <publisher-loc>Cham, Switzerland</publisher-loc>
          <publisher-name>Springer Science &#38; Business Media</publisher-name>
        </nlm-citation>
      </ref>
      <ref id="ref26">
        <label>26</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Kingma</surname>
              <given-names>DP</given-names>
            </name>
            <name name-style="western">
              <surname>Ba</surname>
              <given-names>JA</given-names>
            </name>
          </person-group>
          <article-title>A method for stochastic optimization</article-title>
          <source>arXiv</source>
          <year>2021</year>
        </nlm-citation>
      </ref>
      <ref id="ref27">
        <label>27</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Bridle</surname>
              <given-names>J</given-names>
            </name>
          </person-group>
          <article-title>Training stochastic model recognition algorithms as networks can lead to maximum mutual information estimation of parameters</article-title>
          <source>Adv Neural Inform Process Syst</source>
          <year>1990</year>
          <volume>2</volume>
          <fpage>211</fpage>
          <lpage>217</lpage>
        </nlm-citation>
      </ref>
      <ref id="ref28">
        <label>28</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Thee</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Ilies</surname>
              <given-names>C</given-names>
            </name>
            <name name-style="western">
              <surname>Gruenewald</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Kleinschmidt</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Steinfath</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Bein</surname>
              <given-names>B</given-names>
            </name>
          </person-group>
          <article-title>Reliability of the surgical Pleth index for assessment of postoperative pain: a pilot study</article-title>
          <source>Eur J Anaesthesiol</source>
          <year>2015</year>
          <month>01</month>
          <volume>32</volume>
          <issue>1</issue>
          <fpage>44</fpage>
          <lpage>8</lpage>
          <pub-id pub-id-type="doi">10.1097/EJA.0000000000000095</pub-id>
          <pub-id pub-id-type="medline">24827966</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref29">
        <label>29</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Gram</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Erlenwein</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Petzke</surname>
              <given-names>F</given-names>
            </name>
            <name name-style="western">
              <surname>Falla</surname>
              <given-names>D</given-names>
            </name>
            <name name-style="western">
              <surname>Przemeck</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Emons</surname>
              <given-names>MI</given-names>
            </name>
            <name name-style="western">
              <surname>Reuster</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Olesen</surname>
              <given-names>SS</given-names>
            </name>
            <name name-style="western">
              <surname>Drewes</surname>
              <given-names>AM</given-names>
            </name>
          </person-group>
          <article-title>Prediction of postoperative opioid analgesia using clinical-experimental parameters and electroencephalography</article-title>
          <source>Eur J Pain</source>
          <year>2017</year>
          <month>02</month>
          <volume>21</volume>
          <issue>2</issue>
          <fpage>264</fpage>
          <lpage>277</lpage>
          <pub-id pub-id-type="doi">10.1002/ejp.921</pub-id>
          <pub-id pub-id-type="medline">27470494</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref30">
        <label>30</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lötsch</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Ultsch</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Kalso</surname>
              <given-names>E</given-names>
            </name>
          </person-group>
          <article-title>Prediction of persistent post-surgery pain by preoperative cold pain sensitivity: biomarker development with machine-learning-derived analysis</article-title>
          <source>Br J Anaesth</source>
          <year>2017</year>
          <month>10</month>
          <day>01</day>
          <volume>119</volume>
          <issue>4</issue>
          <fpage>821</fpage>
          <lpage>829</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0007-0912(17)53823-4"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/bja/aex236</pub-id>
          <pub-id pub-id-type="medline">29121286</pub-id>
          <pub-id pub-id-type="pii">S0007-0912(17)53823-4</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref31">
        <label>31</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Tighe</surname>
              <given-names>PJ</given-names>
            </name>
            <name name-style="western">
              <surname>Harle</surname>
              <given-names>CA</given-names>
            </name>
            <name name-style="western">
              <surname>Hurley</surname>
              <given-names>RW</given-names>
            </name>
            <name name-style="western">
              <surname>Aytug</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Boezaart</surname>
              <given-names>AP</given-names>
            </name>
            <name name-style="western">
              <surname>Fillingim</surname>
              <given-names>RB</given-names>
            </name>
          </person-group>
          <article-title>Teaching a Machine to Feel Postoperative Pain: Combining High-Dimensional Clinical Data with Machine Learning Algorithms to Forecast Acute Postoperative Pain</article-title>
          <source>Pain Med</source>
          <year>2015</year>
          <month>07</month>
          <volume>16</volume>
          <issue>7</issue>
          <fpage>1386</fpage>
          <lpage>401</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="http://europepmc.org/abstract/MED/26031220"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/pme.12713</pub-id>
          <pub-id pub-id-type="medline">26031220</pub-id>
          <pub-id pub-id-type="pmcid">PMC4504764</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref32">
        <label>32</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Raghu</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Sriraam</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Temel</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Rao</surname>
              <given-names>SV</given-names>
            </name>
            <name name-style="western">
              <surname>Kubben</surname>
              <given-names>PL</given-names>
            </name>
          </person-group>
          <article-title>EEG based multi-class seizure type classification using convolutional neural network and transfer learning</article-title>
          <source>Neural Netw</source>
          <year>2020</year>
          <month>04</month>
          <volume>124</volume>
          <fpage>202</fpage>
          <lpage>212</lpage>
          <pub-id pub-id-type="doi">10.1016/j.neunet.2020.01.017</pub-id>
          <pub-id pub-id-type="medline">32018158</pub-id>
          <pub-id pub-id-type="pii">S0893-6080(20)30019-8</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref33">
        <label>33</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Zhai</surname>
              <given-names>X</given-names>
            </name>
            <name name-style="western">
              <surname>Jelfs</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>RHM</given-names>
            </name>
            <name name-style="western">
              <surname>Tin</surname>
              <given-names>C</given-names>
            </name>
          </person-group>
          <article-title>Self-Recalibrating Surface EMG Pattern Recognition for Neuroprosthesis Control Based on Convolutional Neural Network</article-title>
          <source>Front Neurosci</source>
          <year>2017</year>
          <volume>11</volume>
          <fpage>379</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://doi.org/10.3389/fnins.2017.00379"/>
          </comment>
          <pub-id pub-id-type="doi">10.3389/fnins.2017.00379</pub-id>
          <pub-id pub-id-type="medline">28744189</pub-id>
          <pub-id pub-id-type="pmcid">PMC5504564</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref34">
        <label>34</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Parvaneh</surname>
              <given-names>S</given-names>
            </name>
            <name name-style="western">
              <surname>Rubin</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Rahman</surname>
              <given-names>A</given-names>
            </name>
            <name name-style="western">
              <surname>Conroy</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Babaeizadeh</surname>
              <given-names>S</given-names>
            </name>
          </person-group>
          <article-title>Analyzing single-lead short ECG recordings using dense convolutional neural networks and feature-based post-processing to detect atrial fibrillation</article-title>
          <source>Physiol Meas</source>
          <year>2018</year>
          <month>08</month>
          <day>23</day>
          <volume>39</volume>
          <issue>8</issue>
          <fpage>084003</fpage>
          <pub-id pub-id-type="doi">10.1088/1361-6579/aad5bd</pub-id>
          <pub-id pub-id-type="medline">30044235</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref35">
        <label>35</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Xia</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Wulan</surname>
              <given-names>N</given-names>
            </name>
            <name name-style="western">
              <surname>Wang</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Zhang</surname>
              <given-names>H</given-names>
            </name>
          </person-group>
          <article-title>Detecting atrial fibrillation by deep convolutional neural networks</article-title>
          <source>Comput Biol Med</source>
          <year>2018</year>
          <month>02</month>
          <day>01</day>
          <volume>93</volume>
          <fpage>84</fpage>
          <lpage>92</lpage>
          <pub-id pub-id-type="doi">10.1016/j.compbiomed.2017.12.007</pub-id>
          <pub-id pub-id-type="medline">29291535</pub-id>
          <pub-id pub-id-type="pii">S0010-4825(17)30404-3</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref36">
        <label>36</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Ahonen</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Jokela</surname>
              <given-names>R</given-names>
            </name>
            <name name-style="western">
              <surname>Uutela</surname>
              <given-names>K</given-names>
            </name>
            <name name-style="western">
              <surname>Huiku</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Surgical stress index reflects surgical stress in gynaecological laparoscopic day-case surgery</article-title>
          <source>Br J Anaesth</source>
          <year>2007</year>
          <month>04</month>
          <volume>98</volume>
          <issue>4</issue>
          <fpage>456</fpage>
          <lpage>61</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://linkinghub.elsevier.com/retrieve/pii/S0007-0912(17)35496-X"/>
          </comment>
          <pub-id pub-id-type="doi">10.1093/bja/aem035</pub-id>
          <pub-id pub-id-type="medline">17350969</pub-id>
          <pub-id pub-id-type="pii">S0007-0912(17)35496-X</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref37">
        <label>37</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lim</surname>
              <given-names>H</given-names>
            </name>
            <name name-style="western">
              <surname>Kim</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Noh</surname>
              <given-names>G</given-names>
            </name>
            <name name-style="western">
              <surname>Yoo</surname>
              <given-names>SK</given-names>
            </name>
          </person-group>
          <article-title>A Deep Neural Network-Based Pain Classifier Using a Photoplethysmography Signal</article-title>
          <source>Sensors (Basel)</source>
          <year>2019</year>
          <month>01</month>
          <day>18</day>
          <volume>19</volume>
          <issue>2</issue>
          <fpage>384</fpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://www.mdpi.com/resolver?pii=s19020384"/>
          </comment>
          <pub-id pub-id-type="doi">10.3390/s19020384</pub-id>
          <pub-id pub-id-type="medline">30669327</pub-id>
          <pub-id pub-id-type="pii">s19020384</pub-id>
          <pub-id pub-id-type="pmcid">PMC6358962</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref38">
        <label>38</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Choi</surname>
              <given-names>B</given-names>
            </name>
            <name name-style="western">
              <surname>Jung</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Lee</surname>
              <given-names>Y</given-names>
            </name>
            <name name-style="western">
              <surname>Bang</surname>
              <given-names>J</given-names>
            </name>
            <name name-style="western">
              <surname>Noh</surname>
              <given-names>G</given-names>
            </name>
          </person-group>
          <article-title>Evaluation of Surgical Pleth Index and Analgesia Nociception Index as surrogate pain measures in conscious postoperative patients: an observational study</article-title>
          <source>J Clin Monit Comput</source>
          <year>2020</year>
          <month>10</month>
          <volume>34</volume>
          <issue>5</issue>
          <fpage>1087</fpage>
          <lpage>1093</lpage>
          <pub-id pub-id-type="doi">10.1007/s10877-019-00399-5</pub-id>
          <pub-id pub-id-type="medline">31628569</pub-id>
          <pub-id pub-id-type="pii">10.1007/s10877-019-00399-5</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref39">
        <label>39</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Calvey</surname>
              <given-names>TN</given-names>
            </name>
            <name name-style="western">
              <surname>Wareing</surname>
              <given-names>M</given-names>
            </name>
            <name name-style="western">
              <surname>Williams</surname>
              <given-names>NE</given-names>
            </name>
            <name name-style="western">
              <surname>Chan</surname>
              <given-names>K</given-names>
            </name>
          </person-group>
          <article-title>Pharmacokinetics and pharmacological effects of neostigmine in man</article-title>
          <source>Br J Clin Pharmacol</source>
          <year>1979</year>
          <month>03</month>
          <volume>7</volume>
          <issue>2</issue>
          <fpage>149</fpage>
          <lpage>55</lpage>
          <comment>
            <ext-link ext-link-type="uri" xlink:type="simple" xlink:href="https://onlinelibrary.wiley.com/resolve/openurl?genre=article&#38;sid=nlm:pubmed&#38;issn=0306-5251&#38;date=1979&#38;volume=7&#38;issue=2&#38;spage=149"/>
          </comment>
          <pub-id pub-id-type="doi">10.1111/j.1365-2125.1979.tb00915.x</pub-id>
          <pub-id pub-id-type="medline">216382</pub-id>
          <pub-id pub-id-type="pmcid">PMC1429427</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref40">
        <label>40</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Parlow</surname>
              <given-names>JL</given-names>
            </name>
            <name name-style="western">
              <surname>van Vlymen</surname>
              <given-names>JM</given-names>
            </name>
            <name name-style="western">
              <surname>Odell</surname>
              <given-names>MJ</given-names>
            </name>
          </person-group>
          <article-title>The duration of impairment of autonomic control after anticholinergic drug administration in humans</article-title>
          <source>Anesth Analg</source>
          <year>1997</year>
          <month>01</month>
          <volume>84</volume>
          <issue>1</issue>
          <fpage>155</fpage>
          <lpage>9</lpage>
          <pub-id pub-id-type="doi">10.1097/00000539-199701000-00028</pub-id>
          <pub-id pub-id-type="medline">8989017</pub-id>
        </nlm-citation>
      </ref>
      <ref id="ref41">
        <label>41</label>
        <nlm-citation citation-type="journal">
          <person-group person-group-type="author">
            <name name-style="western">
              <surname>Miu</surname>
              <given-names>AC</given-names>
            </name>
            <name name-style="western">
              <surname>Heilman</surname>
              <given-names>RM</given-names>
            </name>
            <name name-style="western">
              <surname>Miclea</surname>
              <given-names>M</given-names>
            </name>
          </person-group>
          <article-title>Reduced heart rate variability and vagal tone in anxiety: trait versus state, and the effects of autogenic training</article-title>
          <source>Auton Neurosci</source>
          <year>2009</year>
          <month>01</month>
          <day>28</day>
          <volume>145</volume>
          <issue>1-2</issue>
          <fpage>99</fpage>
          <lpage>103</lpage>
          <pub-id pub-id-type="doi">10.1016/j.autneu.2008.11.010</pub-id>
          <pub-id pub-id-type="medline">19059813</pub-id>
          <pub-id pub-id-type="pii">S1566-0702(08)00250-6</pub-id>
        </nlm-citation>
      </ref>
    </ref-list>
  </back>
</article>
