Repository navigation
Expand file tree
/
Copy pathPre_Process.py
More file actions
226 lines (173 loc) · 5.87 KB
/
Copy pathPre_Process.py
File metadata and controls
226 lines (173 loc) · 5.87 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
# This code is going to do signal pre-process
# Author : Chong Liu (02542904)
#import libraries
import os
import re
import glob
import math
import numpy as np
import pandas as pd
from scipy.signal import butter, filtfilt, find_peaks
import os # Check folders
import re
from pathlib import Path
from typing import Dict, List, Optional, Tuple
import numpy as np
import matplotlib.pyplot as plt
import wfdb # specific required for MIMIC-III
from scipy.signal import resample_poly
# Function to filter signal
def band_pass_filter(signal, fs=125, low=0.5, high=40, order=3):
nyquist = fs / 2
low = low/nyquist
high = high/nyquist
b,a = butter(order, [low, high], btype='band')
return filtfilt(b, a, signal)
def flatline(segment,fs=125, win = 10, eps = 1e-6):
has_nan = np.isnan(segment).any()
if has_nan:
return False
if len(segment) < win * fs:
return False
ave = np.mean(segment)
buffer = 0
# Use Var to find flatline
for n in range(len(segment)):
buffer += (segment[n] - ave)**2
var = buffer/len(segment)
if var < eps:
return False
else:
return True
def qrs_complex(segment, fs=125, win = 10, N = 0.2, threshold = 0.01):
ecg_bp = band_pass_filter(segment,fs=fs, low = 0.5, high=40)
# differentiation
has_nan = np.isnan(segment).any()
if has_nan:
return False
if len(segment) < win * fs:
return False
diff = []
for n in range(len(ecg_bp) - 1):
diff.append((ecg_bp[n+1] - ecg_bp[n])**2)
# Moving Window Integration
emwi = []
N = N * fs
R = 0
for n in range(25, len(diff), 25):
emwi.append(np.mean(np.array(diff[n-25:n], dtype=float)))
for n in emwi:
if n > threshold:
R += 1
if R >= 5:
if R <= 25:
return True
return False
def pulsatile(segment, fs=125, win=10, d_threshold = 0.05, a_threshold = 10):
has_nan = np.isnan(segment).any()
if has_nan:
return False
if len(segment) < win * fs:
return False
peaks, property = find_peaks(segment, distance=0.5 * fs, prominence= 0.05)
troughs, property2 = find_peaks(-1 * segment, distance=0.5 * fs, prominence=0.05)
# check each pulse difference
peak_diff = []
for n in range(len(peaks) - 1):
peak_diff.append(peaks[n+1] - peaks[n])
A = np.sort(peaks)
B = np.sort(troughs)
# Find indices in peaks which is the next peak for the troughs
ind = np.searchsorted(A, B, side='right')
# Find the valid indices in B
valid = ind < len(A)
#set a numpy array to record diff
match = np.full_like(B, np.nan, dtype=float)
diff = np.full_like(B, np.nan, dtype=float)
match[valid] = A[ind[valid]]
n = min(match.shape[0], A.shape[0], B.shape[0])
match = match[:n]
A = A[:n]
diff = diff[:n]
valid = valid[:n]
#count
diff[valid] = match[valid] - A[valid]
if (len(diff) >= 5):
if len(diff) <= 25:
return True
return False
if __name__ == "__main__":
import argparse
Load_path = "C:/Users/Chong/PycharmProjects/pythonProject/MLDS_Final/converted_npy"
fs_out = 125 # Hz
win = 10
eps = 1e-6
np.save('index.npy', '0')
# Read all npy files
npy_ecg = [file for file in os.listdir(Load_path) if file.endswith("ECG.npy")]
npy_ecg.sort()
# Quality check ( if any of them not qualified, removed all of them) Good = 0 unqualified > 0
for file in npy_ecg:
ecg_qual = []
ppg_qual = []
abp_qual = []
filepath = os.path.join(Load_path, file)
data = np.load(filepath)
ecg = data[:,0]
for n in range(0, len(ecg)-1250, 1250):
ecg_q = 0
segment = ecg[n : n+1250]
# check flatline
if flatline(segment):
ecg_q += 0
else:
ecg_q += 1
if qrs_complex(segment):
ecg_q += 0
else:
ecg_q += 1
ecg_qual.append(ecg_q)
# Find address for PPG
file2 = file.replace('ECG', 'PPG')
filepath = os.path.join(Load_path, file2)
ppg = np.load(filepath)
for n in range(0, len(ppg), 1250):
segment = ppg[n:n + 1250]
ppg_q = 0
# check flatline
if flatline(segment):
ppg_q += 0
else:
ppg_q += 1
if pulsatile(segment):
ppg_q += 0
else:
ppg_q += 1
ppg_qual.append(ppg_q)
# Find the address for abp
file3 = file.replace('ECG', 'ABP')
filepath = os.path.join(Load_path, file3)
abp = np.load(filepath)
for n in range(0, len(abp), 1250):
abp_q = 0
segment = abp[n:n + 1250]
# check flatline
if flatline(segment):
abp_q += 0
else:
abp_q += 1
if pulsatile(segment * -1):
abp_q += 0
else:
abp_q += 1
abp_qual.append(abp_q)
for n in range(len(ecg_qual)):
# If segments are qualified
if (ecg_qual[n] == 0) & (ppg_qual[n] == 0) & (abp_qual[n] == 0):
ind = int(np.load('index.npy'))
np.save('D:/MLDS/ecg/' + str(ind) + 'ecg.npy', ecg[n * 1250: (n + 1) * 1250])
np.save('D:/MLDS/ppg/' + str(ind) + 'ppg.npy', ppg[n * 1250: (n + 1) * 1250])
np.save('D:/MLDS/abp/' + str(ind) + 'abp.npy', abp[n * 1250: (n + 1) * 1250])
ind = ind + 1
print(str(ind) + ' segments')
np.save('index.npy', ind)