@article{Bova_Ergas_2017, title={Early effect of validation efforts of Massachusetts syndromic surveillance data}, volume={9}, url={https://ojphi.org/ojs/index.php/ojphi/article/view/7613}, DOI={10.5210/ojphi.v9i1.7613}, abstractNote={<div style="left: 90px; top: 320.758px; font-size: 14.1667px; font-family: sans-serif; transform: scaleX(1.08129);" data-canvas-width="63.778333333333336">Objective</div><div style="left: 105px; top: 335.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.04836);" data-canvas-width="382.40649999999954">To develop a detailed data validation strategy for facilities</div><div style="left: 90px; top: 352.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.999236);" data-canvas-width="393.4324166666667">sending emergency department data to the Massachusetts Syndromic</div><div style="left: 90px; top: 369.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00567);" data-canvas-width="396.30258333333313">Surveillance program and to evaluate the validation strategy by</div><div style="left: 90px; top: 385.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0381);" data-canvas-width="394.70458333333306">comparing data quality metrics before and after implementation of</div><div style="left: 90px; top: 402.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0146);" data-canvas-width="69.24666666666667">the strategy.</div><div style="left: 90px; top: 434.091px; font-size: 14.1667px; font-family: sans-serif; transform: scaleX(1.11768);" data-canvas-width="82.63416666666666">Introduction</div><div style="left: 105px; top: 449.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01627);" data-canvas-width="379.20483333333334">As a participant in the National Syndromic Surveillance Program</div><div style="left: 90px; top: 465.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.974523);" data-canvas-width="395.1366666666668">(NSSP), the Massachusetts Department of Public Health (MDPH)</div><div style="left: 90px; top: 482.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00447);" data-canvas-width="393.7341666666667">has worked closely with our statewide Health Information Exchange</div><div style="left: 90px; top: 499.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.05636);" data-canvas-width="397.9303333333333">(HIE) and National Syndromic Surveillance Program (NSSP)</div><div style="left: 90px; top: 515.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.965017);" data-canvas-width="394.9170833333331">technical staff to collect and transmit emergency department (ED)</div><div style="left: 90px; top: 532.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00486);" data-canvas-width="393.5698333333332">data from eligible hospitals (EHs) to the NSSP. Our goal is to ensure</div><div style="left: 90px; top: 549.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.99009);" data-canvas-width="393.1150833333333">complete and accurate data using a multi-step process beginning with</div><div style="left: 90px; top: 565.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.03655);" data-canvas-width="394.51191666666654">pre-production data and continuing after EHs are sending live data</div><div style="left: 90px; top: 582.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0007);" data-canvas-width="79.48916666666668">to production.</div><div style="left: 90px; top: 614.091px; font-size: 14.1667px; font-family: sans-serif; transform: scaleX(1.07287);" data-canvas-width="58.23916666666666">Methods</div><div style="left: 105px; top: 629.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.05399);" data-canvas-width="382.6855833333332">We used an iterative process to establish a framework for</div><div style="left: 90px; top: 645.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.98352);" data-canvas-width="393.0144999999997">monitoring data quality during onboarding of EHs into our syndromic</div><div style="left: 90px; top: 662.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0007);" data-canvas-width="282.49749999999995">surveillance system and kept notes of the process.</div><div style="left: 105px; top: 679.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.976754);" data-canvas-width="380.4458333333332">To evaluate the framework, we compared data received during</div><div style="left: 90px; top: 695.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.960236);" data-canvas-width="395.12108333333333">the month of January 2016 to the most recent full month of data</div><div style="left: 90px; top: 712.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.04146);" data-canvas-width="394.8334999999997">(June 2016) to describe the following primary data quality metrics</div><div style="left: 90px; top: 729.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.03375);" data-canvas-width="394.21866666666676">and their change over time: total and daily average of message and</div><div style="left: 90px; top: 745.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.954699);" data-canvas-width="395.05450000000025">visit volume; percent of visits with a chief complaint or diagnosis</div><div style="left: 90px; top: 762.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.04573);" data-canvas-width="394.730083333333">code received in the NSSP dataset; and percentage of visits with a</div><div style="left: 90px; top: 779.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.04273);" data-canvas-width="394.725833333333">chief complaint/diagnosis code received within a specified time of</div><div style="left: 90px; top: 795.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00065);" data-canvas-width="118.84416666666668">admission to the ED.</div><div style="left: 90px; top: 827.425px; font-size: 14.1667px; font-family: sans-serif; transform: scaleX(1.08488);" data-canvas-width="51.17">Results</div><div style="left: 105px; top: 842.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.08633);" data-canvas-width="383.36416666666656">The strategies for validation we found effective included</div><div style="left: 90px; top: 859.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01135);" data-canvas-width="394.0203333333334">examination of pre-production test HL7 messages and the execution</div><div style="left: 90px; top: 875.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.02398);" data-canvas-width="394.23283333333336">of R scripts for validation of live data in the staging and production</div><div style="left: 90px; top: 892.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.03344);" data-canvas-width="397.04775000000006">environments. Both the staging and production validations are</div><div style="left: 90px; top: 909.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.02241);" data-canvas-width="394.34475">performed at the individual message level as well as the aggregated</div><div style="left: 90px; top: 925.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00859);" data-canvas-width="396.24166666666656">visit level, and included measures of completeness for required</div><div style="left: 90px; top: 942.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0199);" data-canvas-width="394.24416666666644">fields (Chief Complaint, Diagnosis Codes, Discharge Dispositions),</div><div style="left: 90px; top: 959.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00368);" data-canvas-width="396.1679999999999">timeliness, examples of text fields (Chief Complaint and Triage</div><div style="left: 90px; top: 975.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00225);" data-canvas-width="396.1722500000003">Notes), and demographic information. We required EHs to pass</div><div style="left: 90px; top: 992.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01222);" data-canvas-width="394.0231666666667">validation in the staging environment before granting access to send</div><div style="left: 90px; top: 1009.04px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00061);" data-canvas-width="202.64000000000004">data to the production environment.</div><div style="left: 105px; top: 1025.71px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.939548);" data-canvas-width="379.6241666666667">From January to June 2016, the number of EHs sending data to</div><div style="left: 90px; top: 1042.38px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.998431);" data-canvas-width="393.465">the production environment increased from 44 to 48, and the number</div><div style="left: 90px; top: 1059.04px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01892);" data-canvas-width="396.7346666666666">of messages and visits captured in the production environment</div><div style="left: 90px; top: 1075.71px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.0385);" data-canvas-width="394.5969166666666">increased substantially (see Table 1). The percentage of visits with</div><div style="left: 90px; top: 1092.38px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01839);" data-canvas-width="396.595833333333">a chief complaint remained consistently high (>99%); however</div><div style="left: 90px; top: 1109.04px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.962023);" data-canvas-width="395.0389166666667">the percentage of visits with a chief complaint within three hours</div><div style="left: 90px; top: 1125.71px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.977495);" data-canvas-width="395.6084166666665">of admission decreased during the study period. Both the overall</div><div style="left: 90px; top: 1142.38px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.988067);" data-canvas-width="393.1519166666666">percentage of visits with a diagnosis code and the percentage of visits</div><div style="left: 90px; top: 1159.04px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00078);" data-canvas-width="351.7724999999998">with a diagnosis code within 24 hours of admission increased.</div><div style="left: 510px; top: 320.758px; font-size: 14.1667px; font-family: sans-serif; transform: scaleX(1.10336);" data-canvas-width="85.01416666666667">Conclusions</div><div style="left: 525px; top: 335.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.990405);" data-canvas-width="378.0870833333331">From January to June 2016, Massachusetts syndromic surveillance</div><div style="left: 510px; top: 352.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.982039);" data-canvas-width="393.09383333333324">data improved in the percentage of visits with diagnosis codes and the</div><div style="left: 510px; top: 369.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.996411);" data-canvas-width="393.33324999999996">time from admission to first diagnosis code. This was achieved while</div><div style="left: 510px; top: 385.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00525);" data-canvas-width="393.75825000000003">the volume of data coming into the system increased. The timeliness</div><div style="left: 510px; top: 402.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.995477);" data-canvas-width="393.34599999999995">of chief complaints decreased slightly during the study period, which</div><div style="left: 510px; top: 419.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.9838);" data-canvas-width="393.0938333333334">may be due to the inclusion of several new facilities that are unable to</div><div style="left: 510px; top: 435.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00089);" data-canvas-width="393.5684166666667">send real-time data. Even with the improvements in the timeliness of</div><div style="left: 510px; top: 452.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.982165);" data-canvas-width="393.0952500000001">the diagnosis code field, and the subsequent decrease in the timeliness</div><div style="left: 510px; top: 469.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.992026);" data-canvas-width="393.1731666666666">of the chief complaint field, chief complaints remained a more timely</div><div style="left: 510px; top: 485.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01613);" data-canvas-width="394.31075">option for syndromic surveillance. Pre-production and ongoing data</div><div style="left: 510px; top: 502.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.985655);" data-canvas-width="395.3463333333337">quality assurance activities are crucial to ensure meaningful data</div><div style="left: 510px; top: 519.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.959821);" data-canvas-width="394.8221666666665">are acquired for secondary analyses. We found that reviewing test</div><div style="left: 510px; top: 535.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.00024);" data-canvas-width="396.12833333333367">HL7 messages and staging data, daily monitoring of production</div><div style="left: 510px; top: 552.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.949206);" data-canvas-width="394.99641666666696">data for key factors such as message volume and percent of visits</div><div style="left: 510px; top: 569.043px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01662);" data-canvas-width="394.1251666666667">with a diagnosis code, and monthly full validation in the production</div><div style="left: 510px; top: 585.709px; font-size: 14.1667px; font-family: serif; transform: scaleX(0.991214);" data-canvas-width="393.23125000000005">environment were and will continue to be essential to ensure ongoing</div><div style="left: 510px; top: 602.376px; font-size: 14.1667px; font-family: serif; transform: scaleX(1.01282);" data-canvas-width="78.69583333333333">data integrity.</div><div style="left: 510px; top: 630.528px; font-size: 12.5px; font-family: serif; transform: scaleX(1.00442);" data-canvas-width="245.46249999999995">Table 1: ED Data in the Production Environment</div>}, number={1}, journal={Online Journal of Public Health Informatics}, author={Bova, Mark and Ergas, Roas}, year={2017}, month={May} }