Coverage for ibllib/io/raw_data_loaders.py: 88%

1""" 

2Raw Data Loader functions for PyBpod rig. 

3 

4Module contains one loader function per raw datafile. 

5""" 

6import re (empty)

7import json (empty)

8import logging (empty)

9import wave (empty)

10from collections import OrderedDict (empty)

11from datetime import datetime (empty)

12from pathlib import Path, PureWindowsPath (empty)

13from typing import Union (empty)

14 

15from dateutil import parser as dateparser (empty)

16from packaging import version (empty)

17import numpy as np (empty)

18import pandas as pd (empty)

19 

20from iblutil.io import jsonable (empty)

21from ibllib.io.video import assert_valid_label (empty)

22from ibllib.time import uncycle_pgts, convert_pgts, date2isostr (empty)

23 

24_logger = logging.getLogger(__name__) (empty)

25 

26 

27def trial_times_to_times(raw_trial): (empty)

28 """ 

29 Parse and convert all trial timestamps to "absolute" time. 

30 Float64 seconds from session start. 

31 

32 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1--- 

33 tsN...absTS = tsN + TrialStartN - BpodStart 

34 

35 Bpod timestamps are in microseconds (µs) 

36 PyBpod timestamps are is seconds (s) 

37 

38 :param raw_trial: raw trial data 

39 :type raw_trial: dict 

40 :return: trial data with modified timestamps 

41 :rtype: dict 

42 """ 

43 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

44 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

45 # ts_te = raw_trial['behavior_data']['Trial end timestamp'] 

46 

47 def convert(ts): 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

48 return ts + ts_ts - ts_bs 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

49 

50 converted_events = {} 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

51 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

52 converted_events.update({k: [convert(i) for i in v]}) 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

53 raw_trial['behavior_data']['Events timestamps'] = converted_events 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

54 

55 converted_states = {} 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

56 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

57 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

58 raw_trial['behavior_data']['States timestamps'] = converted_states 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

59 

60 shift = raw_trial['behavior_data']['Bpod start timestamp'] 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

61 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

62 raw_trial['behavior_data']['Trial start timestamp'] -= shift 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

63 raw_trial['behavior_data']['Trial end timestamp'] -= shift 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

64 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

65 return raw_trial 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

66 

67 

68def load_bpod(session_path, task_collection='raw_behavior_data'): (empty)

69 """ 

70 Load both settings and data from bpod (.json and .jsonable) 

71 

72 :param session_path: Absolute path of session folder 

73 :param task_collection: Collection within sesison path with behavior data 

74 :return: dict settings and list of dicts data 

75 """ 

76 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 7 ctx1a567cYX

77 

78 

79def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'): (empty)

80 """ 

81 Load PyBpod data files (.jsonable). 

82 

83 Bpod timestamps are in microseconds (µs) 

84 PyBpod timestamps are is seconds (s) 

85 

86 :param session_path: Absolute path of session folder 

87 :type session_path: str, Path 

88 :return: A list of len ntrials each trial being a dictionary 

89 :rtype: list of dicts 

90 """ 

91 if session_path is None: 51 ctx1aGHIJKyLMNOPQFRSTUez567AlDpghcEvnuqtVrwisjkfYBXbmod

92 _logger.warning('No data loaded: session_path is None') 

93 return 

94 path = Path(session_path).joinpath(task_collection) 51 ctx1aGHIJKyLMNOPQFRSTUez567AlDpghcEvnuqtVrwisjkfYBXbmod

95 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 51 ctx1aGHIJKyLMNOPQFRSTUez567AlDpghcEvnuqtVrwisjkfYBXbmod

96 if not path: 51 ctx1aGHIJKyLMNOPQFRSTUez567AlDpghcEvnuqtVrwisjkfYBXbmod

97 _logger.warning('No data loaded: could not find raw data file') 3 ctx1567

98 return None 3 ctx1567

99 data = jsonable.read(path) 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

100 if time == 'absolute': 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

101 data = [trial_times_to_times(t) for t in data] 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

102 return data 48 ctx1aGHIJKyLMNOPQFRSTUezAlDpghcEvnuqtVrwisjkfYBXbmod

103 

104 

105def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: (empty)

106 """Loads binary frame data from Bonsai camera recording workflow. 

107 

108 Args: 

109 session_path (StrPath): Path to session folder 

110 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'. 

111 raw (bool, optional): Whether to return raw or parsed data. Defaults to False. 

112 

113 Returns: 

114 parsed: (raw=False, Default) 

115 pandas.DataFrame: 4 int64 columns: { 

116 Timestamp, # float64 (seconds from session start) 

117 embeddedTimeStamp, # float64 (seconds from session start) 

118 embeddedFrameCounter, # int64 (Frame number from session start) 

119 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a 

120 # list of numpy boolean arrays 

121 # e.g. np.array([True, False, False, False]) 

122 } 

123 raw: 

124 pandas.DataFrame: 4 int64 columns: { 

125 Timestamp, # UTC ticks from BehaviorPC 

126 # (100's of ns since midnight 1/1/0001) 

127 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion) 

128 embeddedFrameCounter, # Frame counter (int) 

129 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins 

130 } 

131 """ 

132 camera = assert_valid_label(camera) 3 ctx1:1t

133 fpath = Path(session_path).joinpath("raw_video_data") 3 ctx1:1t

134 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 3 ctx1:1t

135 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 3 ctx1:1t

136 rdata = np.fromfile(fpath, dtype=np.float64) 3 ctx1:1t

137 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 3 ctx1:1t

138 rows = int(rdata.size / 4) 3 ctx1:1t

139 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 3 ctx1:1t

140 df_dict = dict.fromkeys( 3 ctx1:1t

141 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"] 

142 ) 

143 df = pd.DataFrame(data, columns=df_dict.keys()) 3 ctx1:1t

144 if raw: 3 ctx1:1t

145 return df 1 ctx1:

146 

147 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 3 ctx1:1t

148 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 3 ctx1:1t

149 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 3 ctx1:1t

150 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 3 ctx1:1t

151 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 3 ctx1:1t

152 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 3 ctx1:1t

153 

154 parsed_df = pd.DataFrame.from_dict(df_dict) 3 ctx1:1t

155 return parsed_df 3 ctx1:1t

156 

157 

158def load_camera_ssv_times(session_path, camera: str): (empty)

159 """ 

160 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv 

161 

162 NB: For some sessions the frame times are in the first column, in others the order is reversed. 

163 NB: If using the new bin file the bonsai_times is a float in seconds since first frame 

164 :param session_path: Absolute path of session folder 

165 :param camera: Name of the camera to load, e.g. 'left' 

166 :return: array of datetimes, array of frame times in seconds 

167 """ 

168 camera = assert_valid_label(camera) 12 ctx1=;cZnuq?0trw

169 video_path = Path(session_path).joinpath('raw_video_data') 12 ctx1=;cZnuq?0trw

170 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 12 ctx1=;cZnuq?0trw

171 df = load_camera_frameData(session_path, camera=camera) 1 ctx1t

172 return df['Timestamp'].values, df['embeddedTimeStamp'].values 1 ctx1t

173 

174 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 12 ctx1=;cZnuq?0trw

175 if not file: 12 ctx1=;cZnuq?0trw

176 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 1 ctx1;

177 raise FileNotFoundError(file + ' not found') 1 ctx1;

178 # NB: Numpy has deprecated support for non-naive timestamps. 

179 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s. 

180 # from datetime import timezone 

181 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)} 

182 

183 # Determine the order of the columns by reading one line and testing whether the first value 

184 # is an integer or not. 

185 with open(file, 'r') as f: 12 ctx1=;cZnuq?0trw

186 line = f.readline() 12 ctx1=;cZnuq?0trw

187 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 12 ctx1=;cZnuq?0trw

188 try: 12 ctx1=;cZnuq?0trw

189 int(line.split(' ')[1]) 12 ctx1=;cZnuq?0trw

190 except ValueError: 6 ctx1;cnqrw

191 type_map.move_to_end('bonsai') 6 ctx1;cnqrw

192 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 12 ctx1=;cZnuq?0trw

193 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 12 ctx1=;cZnuq?0trw

194 bonsai_times = ssv_times['bonsai'] 12 ctx1=;cZnuq?0trw

195 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 12 ctx1=;cZnuq?0trw

196 return bonsai_times, camera_times 12 ctx1=;cZnuq?0trw

197 

198 

199def load_embedded_frame_data(session_path, label: str, raw=False): (empty)

200 """ 

201 Load the embedded frame count and GPIO for a given session. If the file doesn't exist, 

202 or is empty, None values are returned. 

203 

204 :param session_path: Absolute path of session folder 

205 :param label: The specific video to load, one of ('left', 'right', 'body') 

206 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

207 returned starting from 0 and the GPIO is returned as a dict of indices 

208 :return: The frame count, GPIO 

209 """ 

210 count = load_camera_frame_count(session_path, label, raw=raw) 12 ctx19cv/Znuq0trw

211 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 12 ctx19cv/Znuq0trw

212 return count, gpio 12 ctx19cv/Znuq0trw

213 

214 

215def load_camera_frame_count(session_path, label: str, raw=True): (empty)

216 """ 

217 Load the embedded frame count for a given session. If the file doesn't exist, or is empty, 

218 a None value is returned. 

219 :param session_path: Absolute path of session folder 

220 :param label: The specific video to load, one of ('left', 'right', 'body') 

221 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

222 returned starting from 0 

223 :return: The frame count 

224 """ 

225 if session_path is None: 13 ctx1@9cv/Znuq0trw

226 return 1 ctx1@

227 

228 label = assert_valid_label(label) 13 ctx1@9cv/Znuq0trw

229 video_path = Path(session_path).joinpath('raw_video_data') 13 ctx1@9cv/Znuq0trw

230 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 13 ctx1@9cv/Znuq0trw

231 df = load_camera_frameData(session_path, camera=label) 1 ctx1t

232 return df['embeddedFrameCounter'].values 1 ctx1t

233 

234 # Load frame count 

235 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 13 ctx1@9cv/Znuq0trw

236 count_file = next(glob, None) 13 ctx1@9cv/Znuq0trw

237 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 13 ctx1@9cv/Znuq0trw

238 if len(count) == 0: 13 ctx1@9cv/Znuq0trw

239 return 5 ctx1@cv/n

240 if not raw: 9 ctx1@9Zuq0trw

241 count -= count[0] # start from zero 8 ctx1@9Zq0trw

242 return count 9 ctx1@9Zuq0trw

243 

244 

245def load_camera_gpio(session_path, label: str, as_dicts=False): (empty)

246 """ 

247 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is 

248 returned. 

249 

250 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits 

251 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by 

252 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state). 

253 

254 :param session_path: Absolute path of session folder 

255 :param label: The specific video to load, one of ('left', 'right', 'body') 

256 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins) 

257 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities'). 

258 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4. 

259 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'), 

260 or None if the dictionary is empty. 

261 """ 

262 if session_path is None: 13 ctx119cv/Znuq0trw

263 return 1 ctx11

264 raw_path = Path(session_path).joinpath('raw_video_data') 13 ctx119cv/Znuq0trw

265 label = assert_valid_label(label) 13 ctx119cv/Znuq0trw

266 

267 # Load pin state 

268 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 13 ctx119cv/Znuq0trw

269 df = load_camera_frameData(session_path, camera=label, raw=False) 2 ctx11t

270 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 2 ctx11t

271 if len(gpio) == 0: 2 ctx11t

272 return [None] * 4 if as_dicts else None 

273 else: 

274 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 13 ctx119cv/Znuq0trw

275 # This deals with missing and empty files the same 

276 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 13 ctx119cv/Znuq0trw

277 # Check values make sense (4 pins = 16 possible values) 

278 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 13 ctx119cv/Znuq0trw

279 _logger.warning('Unexpected GPIO values; decoding may fail') 1 ctx11

280 if len(gpio) == 0: 13 ctx119cv/Znuq0trw

281 return [None] * 4 if as_dicts else None 5 ctx11cv/n

282 # 4 pins represented as uint32 

283 # For each pin, shift its bit to the end and check the bit is set 

284 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 9 ctx119Zuq0trw

285 

286 if as_dicts: 9 ctx119Zuq0trw

287 if not gpio.any(): 8 ctx119Zq0trw

288 _logger.error('No GPIO changes') 1 ctx11

289 return [None] * 4 1 ctx11

290 # Find state changes for each pin and construct a dict of indices and polarities for each 

291 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 8 ctx119Zq0trw

292 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)] 

293 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8 

294 gpio = [{'indices': np.where(edges[:, i])[0], 8 ctx119Zq0trw

295 'polarities': edges[edges[:, i] != 0, i]} 

296 for i in range(4)] 

297 # Replace empty dicts with None 

298 gpio = [None if x['indices'].size == 0 else x for x in gpio] 8 ctx119Zq0trw

299 

300 return gpio 9 ctx119Zuq0trw

301 

302 

303def _read_settings_json_compatibility_enforced(settings): (empty)

304 """ 

305 Patch iblrig settings for compatibility across rig versions. 

306 

307 Parameters 

308 ---------- 

309 settings : pathlib.Path, dict 

310 Either a _iblrig_taskSettings.raw.json file path or the loaded settings. 

311 

312 Returns 

313 ------- 

314 dict 

315 The task settings patched for compatibility. 

316 """ 

317 if isinstance(settings, dict): 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

318 md = settings.copy() 

319 else: 

320 with open(settings) as js: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

321 md = json.load(js) 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

322 if 'IS_MOCK' not in md: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

323 md['IS_MOCK'] = False 54 ctx1aGHIJKyLMNOPQFRSTUWxez!%'#8567AlDghEvnqVrijkfCYBX(bmod

324 # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag. 

325 # Some sessions have neither key. From v8 onwards we will use IBLRIG_VERSION to test rig 

326 # version, however some places may still use the version tag. 

327 if 'IBLRIG_VERSION_TAG' not in md.keys(): 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

328 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 6 ctx1#567Ef

329 if 'IBLRIG_VERSION' not in md.keys(): 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

330 md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG'] 57 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'8AlD)pghEvnuqVr*+,isjkCYBX(bmod

331 elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]): 7 ctx1#567cfC

332 # This may not be an issue; not sure what the intended difference between these keys was 

333 assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch' 7 ctx1#567cfC

334 # Test version can be parsed. If not, log an error and set the version to nothing 

335 try: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

336 version.parse(md['IBLRIG_VERSION'] or '0') 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

337 except version.InvalidVersion as ex: 

338 _logger.error('%s in iblrig settings, this may affect extraction', ex) 

339 # try a more relaxed version parse 

340 laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION']) 

341 # Set the tag as the invalid version 

342 md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION'] 

343 # overwrite version with either successfully parsed one or an empty string 

344 md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else '' 

345 if 'device_sound' not in md: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

346 # sound device must be defined in version 8 and later # FIXME this assertion will cause tests to break 

347 assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0') 60 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghvnuqVr*+,isjkCYBX(bmod

348 # in v7 we must infer the device from the sampling frequency if SD is None 

349 if 'sounddevice' in md.get('SD', ''): 60 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghvnuqVr*+,isjkCYBX(bmod

350 device = 'xonar' 14 ctx1alDghvqVrCYXbd

351 else: 

352 freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'} 50 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567A)pnu*+,isjkBX(bmod

353 device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown') 50 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567A)pnu*+,isjkBX(bmod

354 md['device_sound'] = {'OUTPUT': device} 60 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghvnuqVr*+,isjkCYBX(bmod

355 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on) 

356 if md['IBLRIG_VERSION'] == '': 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

357 pass 5 ctx1F.Eid

358 elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'): 61 ctx1aGHIJKyLMNOPQFRSTUWxez!%'#8567AlD)pghcvnuqVr*+,isjkfCYBX(bmod

359 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 1 ctx1f

360 md['PYBPOD_BOARD'] = md['RIG_NAME'] 1 ctx1f

361 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 1 ctx1f

362 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 1 ctx1f

363 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 1 ctx1f

364 elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'): 60 ctx1aGHIJKyLMNOPQFRSTUWxez!%'#8567AlD)pghcvnuqVr*+,isjkCYBX(bmod

365 if 'LAST_TRIAL_DATA' in md.keys(): 1 ctx1b

366 md.pop('LAST_TRIAL_DATA') 1 ctx1b

367 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

368 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1 ctx1b

369 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

370 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1 ctx1b

371 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1 ctx1b

372 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1 ctx1b

373 # parse the date format to Django supported ISO 

374 dt = dateparser.parse(md['SESSION_DATETIME']) 1 ctx1b

375 md['SESSION_DATETIME'] = date2isostr(dt) 1 ctx1b

376 # add the weight key if it doesn't already exist 

377 if 'SUBJECT_WEIGHT' not in md: 1 ctx1b

378 md['SUBJECT_WEIGHT'] = None 1 ctx1b

379 return md 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

380 

381 

382def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'): (empty)

383 """ 

384 Load PyBpod Settings files (.json). 

385 

386 [description] 

387 

388 :param session_path: Absolute path of session folder 

389 :type session_path: str, Path 

390 :return: Settings dictionary 

391 :rtype: dict 

392 """ 

393 if session_path is None: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

394 _logger.warning("No data loaded: session_path is None") 1 ctx18

395 return 1 ctx18

396 path = Path(session_path).joinpath(task_collection) 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

397 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

398 if not path: 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

399 _logger.warning("No data loaded: could not find raw settings file") 4 ctx1!85C

400 return None 4 ctx1!85C

401 settings = _read_settings_json_compatibility_enforced(path) 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

402 return settings 63 ctx1aGHIJKyLMNOPQFRSTUWxez!%.'#8567AlD)pghcEvnuqVr*+,isjkfCYBX(bmod

403 

404 

405def load_stim_position_screen(session_path, task_collection='raw_behavior_data'): (empty)

406 path = Path(session_path).joinpath(task_collection) 

407 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None) 

408 

409 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip') 

410 data.columns = ['contrast', 'position', 'bns_ts'] 

411 data['bns_ts'] = pd.to_datetime(data['bns_ts']) 

412 return data 

413 

414 

415def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False): (empty)

416 """ 

417 Load Rotary Encoder (RE) events raw data file. 

418 

419 Assumes that a folder called "raw_behavior_data" exists in folder. 

420 

421 Events number correspond to following bpod states: 

422 1: correct / hide_stim 

423 2: stim_on 

424 3: closed_loop 

425 4: freeze_error / freeze_correct 

426 

427 >>> data.columns 

428 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

429 'sm_ev', # State Machine Event 'numpy.int64' 

430 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp' 

431 # pd.to_datetime(data.bns_ts) to work in datetimes 

432 

433 :param session_path: [description] 

434 :type session_path: [type] 

435 :return: dataframe w/ 3 cols and (ntrials * 3) lines 

436 :rtype: Pandas.DataFrame 

437 """ 

438 if session_path is None: 17 ctx1aWelpghcisjkfbmod

439 return 

440 path = Path(session_path).joinpath(task_collection) 17 ctx1aWelpghcisjkfbmod

441 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 17 ctx1aWelpghcisjkfbmod

442 if not settings: 17 ctx1aWelpghcisjkfbmod

443 settings = load_settings(session_path, task_collection=task_collection) 17 ctx1aWelpghcisjkfbmod

444 if settings is None or not settings.get('IBLRIG_VERSION'): 17 ctx1aWelpghcisjkfbmod

445 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1d

446 # auto-detect old files when version is not labeled 

447 with open(path) as fid: 1 ctx1d

448 line = fid.readline() 1 ctx1d

449 if line.startswith('Event') and 'StateMachine' in line: 1 ctx1d

450 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1d

451 if not path: 17 ctx1aWelpghcisjkfbmod

452 return None 

453 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 17 ctx1aWelpghcisjkfbmod

454 return _load_encoder_events_file_ge5(path) 15 ctx1Welpghisjkfbmod

455 else: 

456 return _load_encoder_events_file_lt5(path) 6 ctx1aWecbd

457 

458 

459def _load_encoder_ssv_file(file_path, **kwargs): (empty)

460 file_path = Path(file_path) 22 ctx1a4$Wxe32lpghcisjkfbmod

461 if file_path.stat().st_size == 0: 22 ctx1a4$Wxe32lpghcisjkfbmod

462 _logger.error(f"{file_path.name} is an empty file. ") 

463 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ") 

464 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 22 ctx1a4$Wxe32lpghcisjkfbmod

465 

466 

467def _load_encoder_positions_file_lt5(file_path): (empty)

468 """ 

469 File loader without the session overhead 

470 :param file_path: 

471 :return: dataframe of encoder events 

472 """ 

473 data = _load_encoder_ssv_file(file_path, 9 ctx1a4$xe2cbd

474 names=['_', 're_ts', 're_pos', 'bns_ts', '__'], 

475 usecols=['re_ts', 're_pos', 'bns_ts']) 

476 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 9 ctx1a4$xe2cbd

477 

478 

479def _load_encoder_positions_file_ge5(file_path): (empty)

480 """ 

481 File loader without the session overhead 

482 :param file_path: 

483 :return: dataframe of encoder events 

484 """ 

485 data = _load_encoder_ssv_file(file_path, 17 ctx14xe2lpghisjkfbmod

486 names=['re_ts', 're_pos', '_'], 

487 usecols=['re_ts', 're_pos']) 

488 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 17 ctx14xe2lpghisjkfbmod

489 

490 

491def _load_encoder_events_file_lt5(file_path): (empty)

492 """ 

493 File loader without the session overhead 

494 :param file_path: 

495 :return: dataframe of encoder events 

496 """ 

497 data = _load_encoder_ssv_file(file_path, 7 ctx1aWe3cbd

498 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'], 

499 usecols=['re_ts', 'sm_ev', 'bns_ts']) 

500 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 7 ctx1aWe3cbd

501 

502 

503def _load_encoder_events_file_ge5(file_path): (empty)

504 """ 

505 File loader without the session overhead 

506 :param file_path: 

507 :return: dataframe of encoder events 

508 """ 

509 data = _load_encoder_ssv_file(file_path, 16 ctx1We3lpghisjkfbmod

510 names=['re_ts', 'sm_ev', '_'], 

511 usecols=['re_ts', 'sm_ev']) 

512 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 16 ctx1We3lpghisjkfbmod

513 

514 

515def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False): (empty)

516 """ 

517 Load Rotary Encoder (RE) positions from raw data file within a session path. 

518 

519 Assumes that a folder called "raw_behavior_data" exists in folder. 

520 Positions are RE ticks [-512, 512] == [-180º, 180º] 

521 0 == trial stim init position 

522 Positive nums are rightwards movements (mouse) or RE CW (mouse) 

523 

524 Variable line number, depends on movements. 

525 

526 Raw datafile Columns: 

527 Position, RE timestamp, RE Position, Bonsai Timestamp 

528 

529 Position is always equal to 'Position' so this column was dropped. 

530 

531 >>> data.columns 

532 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

533 're_pos', # Rotary Encoder position (ticks) 'numpy.int64' 

534 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp' 

535 # pd.to_datetime(data.bns_ts) to work in datetimes 

536 

537 :param session_path: Absolute path of session folder 

538 :type session_path: str 

539 :return: dataframe w/ 3 cols and N positions 

540 :rtype: Pandas.DataFrame 

541 """ 

542 if session_path is None: 18 ctx1axelpghcnisjkfbmod

543 return 

544 path = Path(session_path).joinpath(task_collection) 18 ctx1axelpghcnisjkfbmod

545 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 18 ctx1axelpghcnisjkfbmod

546 if not settings: 18 ctx1axelpghcnisjkfbmod

547 settings = load_settings(session_path, task_collection=task_collection) 18 ctx1axelpghcnisjkfbmod

548 if settings is None or not settings.get('IBLRIG_VERSION'): 18 ctx1axelpghcnisjkfbmod

549 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1d

550 # auto-detect old files when version is not labeled 

551 with open(path) as fid: 1 ctx1d

552 line = fid.readline() 1 ctx1d

553 if line.startswith('Position'): 1 ctx1d

554 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1d

555 if not path: 18 ctx1axelpghcnisjkfbmod

556 _logger.warning("No data loaded: could not find raw encoderPositions file") 1 ctx1n

557 return None 1 ctx1n

558 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 17 ctx1axelpghcisjkfbmod

559 return _load_encoder_positions_file_ge5(path) 15 ctx1xelpghisjkfbmod

560 else: 

561 return _load_encoder_positions_file_lt5(path) 6 ctx1axecbd

562 

563 

564def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'): (empty)

565 """ 

566 Load Rotary Encoder trial info from raw data file. 

567 

568 Assumes that a folder calles "raw_behavior_data" exists in folder. 

569 

570 NOTE: Last trial probably inexistent data (Trial info is sent on trial start 

571 and data is only saved on trial exit...) max(trialnum) should be N+1 if N 

572 is the amount of trial data saved. 

573 

574 Raw datafile Columns: 

575 

576 >>> data.columns 

577 >>> ['trial_num', # Trial Number 'numpy.int64' 

578 'stim_pos_init', # Initial position of visual stim 'numpy.int64' 

579 'stim_contrast', # Contrast of visual stimulus 'numpy.float64' 

580 'stim_freq', # Frequency of gabor patch 'numpy.float64' 

581 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64' 

582 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64' 

583 'stim_sigma', # Size of patch 'numpy.float64' 

584 'stim_phase', # Phase of gabor 'numpy.float64' 

585 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp' 

586 # pd.to_datetime(data.bns_ts) to work in datetimes 

587 

588 :param session_path: Absoulte path of session folder 

589 :type session_path: str 

590 :return: dataframe w/ 9 cols and ntrials lines 

591 :rtype: Pandas.DataFrame 

592 """ 

593 if session_path is None: 1 ctx1[

594 return 1 ctx1[

595 path = Path(session_path).joinpath(task_collection) 1 ctx1[

596 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 1 ctx1[

597 if not path: 1 ctx1[

598 return None 1 ctx1[

599 data = pd.read_csv(path, sep=' ', header=None) 1 ctx1[

600 data = data.drop([9], axis=1) 1 ctx1[

601 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 1 ctx1[

602 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts'] 

603 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path) 

604 return data 1 ctx1[

605 

606 

607def load_ambient_sensor(session_path, task_collection='raw_behavior_data'): (empty)

608 """ 

609 Load Ambient Sensor data from session. 

610 

611 Probably could be extracted to DatasetTypes: 

612 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb, 

613 _ibl_trials.relativeHumidity 

614 Returns a list of dicts one dict per trial. 

615 dict keys are: 

616 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity']) 

617 

618 :param session_path: Absoulte path of session folder 

619 :type session_path: str 

620 :return: list of dicts 

621 :rtype: list 

622 """ 

623 if session_path is None: 

624 return 

625 path = Path(session_path).joinpath(task_collection) 

626 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None) 

627 if not path: 

628 return None 

629 data = [] 

630 with open(path, 'r') as f: 

631 for line in f: 

632 data.append(json.loads(line)) 

633 return data 

634 

635 

636def load_mic(session_path, task_collection='raw_behavior_data'): (empty)

637 """ 

638 Load Microphone wav file to np.array of len nSamples 

639 

640 :param session_path: Absolute path of session folder 

641 :type session_path: str 

642 :return: An array of values of the sound waveform 

643 :rtype: numpy.array 

644 """ 

645 if session_path is None: 

646 return 

647 path = Path(session_path).joinpath(task_collection) 

648 path = next(path.glob("_iblrig_micData.raw*.wav"), None) 

649 if not path: 

650 return None 

651 fp = wave.open(path) 

652 nchan = fp.getnchannels() 

653 N = fp.getnframes() 

654 dstr = fp.readframes(N * nchan) 

655 data = np.frombuffer(dstr, np.int16) 

656 data = np.reshape(data, (-1, nchan)) 

657 return data 

658 

659 

660def _clean_wheel_dataframe(data, label, path): (empty)

661 if np.any(data.isna()): 22 ctx1a4$Wxe32lpghcisjkfbmod

662 _logger.warning(label + ' has missing/incomplete records \n %s', path) 8 ctx1a$xe32bd

663 # first step is to re-interpret as numeric objects if not already done 

664 for col in data.columns: 22 ctx1a4$Wxe32lpghcisjkfbmod

665 if data[col].dtype == object and col not in ['bns_ts']: 22 ctx1a4$Wxe32lpghcisjkfbmod

666 data[col] = pd.to_numeric(data[col], errors='coerce') 2 ctx13d

667 # then drop Nans and duplicates 

668 data.dropna(inplace=True) 22 ctx1a4$Wxe32lpghcisjkfbmod

669 data.drop_duplicates(keep='first', inplace=True) 22 ctx1a4$Wxe32lpghcisjkfbmod

670 data.reset_index(inplace=True) 22 ctx1a4$Wxe32lpghcisjkfbmod

671 # handle the clock resets when microseconds exceed uint32 max value 

672 drop_first = False 22 ctx1a4$Wxe32lpghcisjkfbmod

673 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 22 ctx1a4$Wxe32lpghcisjkfbmod

674 if any(np.diff(data['re_ts']) < 0): 22 ctx1a4$Wxe32lpghcisjkfbmod

675 ind = np.where(np.diff(data['re_ts']) < 0)[0] 16 ctx14$xe32lghcijkfbd

676 for i in ind: 16 ctx14$xe32lghcijkfbd

677 # the first sample may be corrupt, in this case throw away 

678 if i <= 1: 16 ctx14$xe32lghcijkfbd

679 drop_first = i 6 ctx14xe2cb

680 _logger.warning(label + ' rotary encoder positions timestamps' 6 ctx14xe2cb

681 ' first sample corrupt ' + str(path)) 

682 # if it's an uint32 wraparound, the diff should be close to 2 ** 32 

683 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 14 ctx14$xe32lghijkfd

684 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 11 ctx1$xe2lghijkd

685 # there is also the case where 2 positions are swapped and need to be swapped back 

686 

687 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 9 ctx14xe32ghfd

688 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 8 ctx14xe2ghfd

689 str(i) + ' ' + str(path)) 

690 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 8 ctx14xe2ghfd

691 data.iloc[i], data.iloc[i + 1] = b, a 8 ctx14xe2ghfd

692 # if none of those 3 cases apply, raise an error 

693 else: 

694 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1 ctx13

695 data.sort_values('re_ts', inplace=True) 1 ctx13

696 data.reset_index(inplace=True) 1 ctx13

697 if drop_first is not False: 22 ctx1a4$Wxe32lpghcisjkfbmod

698 data.drop(data.loc[:drop_first].index, inplace=True) 6 ctx14xe2cb

699 data = data.reindex() 6 ctx14xe2cb

700 return data 22 ctx1a4$Wxe32lpghcisjkfbmod

701 

702 

703def _groom_wheel_data_lt5(data, label='file ', path=''): (empty)

704 """ 

705 The whole purpose of this function is to account for variability and corruption in 

706 the wheel position files. There are many possible errors described below, but 

707 nothing excludes getting new ones. 

708 """ 

709 data = _clean_wheel_dataframe(data, label, path) 11 ctx1a4$Wxe32cbd

710 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 11 ctx1a4$Wxe32cbd

711 # check if the time scale is in ms 

712 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 11 ctx1a4$Wxe32cbd

713 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds