Coverage for ibllib/io/raw_data_loaders.py: 88%

450 statements  

« prev     ^ index     » next       coverage.py v7.5.4, created at 2024-07-08 17:16 +0100

1""" 

2Raw Data Loader functions for PyBpod rig. 

3 

4Module contains one loader function per raw datafile. 

5""" 

6import re 

7import json 

8import logging 

9import wave 

10from collections import OrderedDict 

11from datetime import datetime 

12from pathlib import Path, PureWindowsPath 

13from typing import Union 

14 

15from dateutil import parser as dateparser 

16from packaging import version 

17import numpy as np 

18import pandas as pd 

19 

20from iblutil.io import jsonable 

21from ibllib.io.video import assert_valid_label 

22from ibllib.time import uncycle_pgts, convert_pgts, date2isostr 

23 

24_logger = logging.getLogger(__name__) 

25 

26 

27def trial_times_to_times(raw_trial): 

28 """ 

29 Parse and convert all trial timestamps to "absolute" time. 

30 Float64 seconds from session start. 

31 

32 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1--- 

33 tsN...absTS = tsN + TrialStartN - BpodStart 

34 

35 Bpod timestamps are in microseconds (µs) 

36 PyBpod timestamps are is seconds (s) 

37 

38 :param raw_trial: raw trial data 

39 :type raw_trial: dict 

40 :return: trial data with modified timestamps 

41 :rtype: dict 

42 """ 

43 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

44 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

45 # ts_te = raw_trial['behavior_data']['Trial end timestamp'] 

46 

47 def convert(ts): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

48 return ts + ts_ts - ts_bs 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

49 

50 converted_events = {} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

51 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

52 converted_events.update({k: [convert(i) for i in v]}) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

53 raw_trial['behavior_data']['Events timestamps'] = converted_events 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

54 

55 converted_states = {} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

56 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

57 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

58 raw_trial['behavior_data']['States timestamps'] = converted_states 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

59 

60 shift = raw_trial['behavior_data']['Bpod start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

61 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

62 raw_trial['behavior_data']['Trial start timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

63 raw_trial['behavior_data']['Trial end timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

64 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

65 return raw_trial 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

66 

67 

68def load_bpod(session_path, task_collection='raw_behavior_data'): 

69 """ 

70 Load both settings and data from bpod (.json and .jsonable) 

71 

72 :param session_path: Absolute path of session folder 

73 :param task_collection: Collection within sesison path with behavior data 

74 :return: dict settings and list of dicts data 

75 """ 

76 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 1b=?@c'pqa%

77 

78 

79def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'): 

80 """ 

81 Load PyBpod data files (.jsonable). 

82 

83 Bpod timestamps are in microseconds (µs) 

84 PyBpod timestamps are is seconds (s) 

85 

86 :param session_path: Absolute path of session folder 

87 :type session_path: str, Path 

88 :return: A list of len ntrials each trial being a dictionary 

89 :rtype: list of dicts 

90 """ 

91 if session_path is None: 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

92 _logger.warning('No data loaded: session_path is None') 

93 return 

94 path = Path(session_path).joinpath(task_collection) 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

95 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

96 if not path: 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

97 _logger.warning('No data loaded: could not find raw data file') 1=?@

98 return None 1=?@

99 data = jsonable.read(path) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

100 if time == 'absolute': 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

101 data = [trial_times_to_times(t) for t in data] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

102 return data 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd

103 

104 

105def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: 

106 """Loads binary frame data from Bonsai camera recording workflow. 

107 

108 Args: 

109 session_path (StrPath): Path to session folder 

110 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'. 

111 raw (bool, optional): Whether to return raw or parsed data. Defaults to False. 

112 

113 Returns: 

114 parsed: (raw=False, Default) 

115 pandas.DataFrame: 4 int64 columns: { 

116 Timestamp, # float64 (seconds from session start) 

117 embeddedTimeStamp, # float64 (seconds from session start) 

118 embeddedFrameCounter, # int64 (Frame number from session start) 

119 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a 

120 # list of numpy boolean arrays 

121 # e.g. np.array([True, False, False, False]) 

122 } 

123 raw: 

124 pandas.DataFrame: 4 int64 columns: { 

125 Timestamp, # UTC ticks from BehaviorPC 

126 # (100's of ns since midnight 1/1/0001) 

127 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion) 

128 embeddedFrameCounter, # Frame counter (int) 

129 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins 

130 } 

131 """ 

132 camera = assert_valid_label(camera) 2lb. E C

133 fpath = Path(session_path).joinpath("raw_video_data") 2lb. E C

134 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 2lb. E C

135 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 2lb. E C

136 rdata = np.fromfile(fpath, dtype=np.float64) 2lb. E C

137 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 2lb. E C

138 rows = int(rdata.size / 4) 2lb. E C

139 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 2lb. E C

140 df_dict = dict.fromkeys( 2lb. E C

141 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"] 

142 ) 

143 df = pd.DataFrame(data, columns=df_dict.keys()) 2lb. E C

144 if raw: 2lb. E C

145 return df 2lb

146 

147 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 2lb. E C

148 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 2lb. E C

149 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 2lb. E C

150 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 2lb. E C

151 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 2lb. E C

152 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 2lb. E C

153 

154 parsed_df = pd.DataFrame.from_dict(df_dict) 2lb. E C

155 return parsed_df 2lb. E C

156 

157 

158def load_camera_ssv_times(session_path, camera: str): 

159 """ 

160 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv 

161 

162 NB: For some sessions the frame times are in the first column, in others the order is reversed. 

163 NB: If using the new bin file the bonsai_times is a float in seconds since first frame 

164 :param session_path: Absolute path of session folder 

165 :param camera: Name of the camera to load, e.g. 'left' 

166 :return: array of datetimes, array of frame times in seconds 

167 """ 

168 camera = assert_valid_label(camera) 2nbmbc R s D y obS E - C z A H a

169 video_path = Path(session_path).joinpath('raw_video_data') 2nbmbc R s D y obS E - C z A H a

170 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 2nbmbc R s D y obS E - C z A H a

171 df = load_camera_frameData(session_path, camera=camera) 1EC

172 return df['Timestamp'].values, df['embeddedTimeStamp'].values 1EC

173 

174 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 2nbmbc R s D y obS E - C z A H a

175 if not file: 2nbmbc R s D y obS E - C z A H a

176 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 2mb

177 raise FileNotFoundError(file + ' not found') 2mb

178 # NB: Numpy has deprecated support for non-naive timestamps. 

179 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s. 

180 # from datetime import timezone 

181 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)} 

182 

183 # Determine the order of the columns by reading one line and testing whether the first value 

184 # is an integer or not. 

185 with open(file, 'r') as f: 2nbmbc R s D y obS E - C z A H a

186 line = f.readline() 2nbmbc R s D y obS E - C z A H a

187 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 2nbmbc R s D y obS E - C z A H a

188 try: 2nbmbc R s D y obS E - C z A H a

189 int(line.split(' ')[1]) 2nbmbc R s D y obS E - C z A H a

190 except ValueError: 2mbc s y z A H a

191 type_map.move_to_end('bonsai') 2mbc s y z A H a

192 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 2nbmbc R s D y obS E - C z A H a

193 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 2nbmbc R s D y obS E - C z A H a

194 bonsai_times = ssv_times['bonsai'] 2nbmbc R s D y obS E - C z A H a

195 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 2nbmbc R s D y obS E - C z A H a

196 return bonsai_times, camera_times 2nbmbc R s D y obS E - C z A H a

197 

198 

199def load_embedded_frame_data(session_path, label: str, raw=False): 

200 """ 

201 Load the embedded frame count and GPIO for a given session. If the file doesn't exist, 

202 or is empty, None values are returned. 

203 :param session_path: Absolute path of session folder 

204 :param label: The specific video to load, one of ('left', 'right', 'body') 

205 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

206 returned starting from 0 and the GPIO is returned as a dict of indices 

207 :return: The frame count, GPIO 

208 """ 

209 count = load_camera_frame_count(session_path, label, raw=raw) 2] c F kb, G R s D y S E - C z A H a

210 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 2] c F kb, G R s D y S E - C z A H a

211 return count, gpio 2] c F kb, G R s D y S E - C z A H a

212 

213 

214def load_camera_frame_count(session_path, label: str, raw=True): 

215 """ 

216 Load the embedded frame count for a given session. If the file doesn't exist, or is empty, 

217 a None value is returned. 

218 :param session_path: Absolute path of session folder 

219 :param label: The specific video to load, one of ('left', 'right', 'body') 

220 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

221 returned starting from 0 

222 :return: The frame count 

223 """ 

224 if session_path is None: 2pb] c F kb, G R s D y S E - C z A H a

225 return 2pb

226 

227 label = assert_valid_label(label) 2pb] c F kb, G R s D y S E - C z A H a

228 video_path = Path(session_path).joinpath('raw_video_data') 2pb] c F kb, G R s D y S E - C z A H a

229 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 2pb] c F kb, G R s D y S E - C z A H a

230 df = load_camera_frameData(session_path, camera=label) 1EC

231 return df['embeddedFrameCounter'].values 1EC

232 

233 # Load frame count 

234 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 2pb] c F kb, G R s D y S E - C z A H a

235 count_file = next(glob, None) 2pb] c F kb, G R s D y S E - C z A H a

236 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 2pb] c F kb, G R s D y S E - C z A H a

237 if len(count) == 0: 2pb] c F kb, G R s D y S E - C z A H a

238 return 2pbc F kb, G s a

239 if not raw: 2pb] R D y S E - C z A H

240 count -= count[0] # start from zero 2pb] R y S - C z A H

241 return count 2pb] R D y S E - C z A H

242 

243 

244def load_camera_gpio(session_path, label: str, as_dicts=False): 

245 """ 

246 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is 

247 returned. 

248 

249 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits 

250 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by 

251 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state). 

252 

253 :param session_path: Absolute path of session folder 

254 :param label: The specific video to load, one of ('left', 'right', 'body') 

255 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins) 

256 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities'). 

257 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4. 

258 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'), 

259 or None if the dictionary is empty. 

260 """ 

261 if session_path is None: 2. ] c F kb, G R s D y S E - C z A H a

262 return 1.

263 raw_path = Path(session_path).joinpath('raw_video_data') 2. ] c F kb, G R s D y S E - C z A H a

264 label = assert_valid_label(label) 2. ] c F kb, G R s D y S E - C z A H a

265 

266 # Load pin state 

267 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 2. ] c F kb, G R s D y S E - C z A H a

268 df = load_camera_frameData(session_path, camera=label, raw=False) 1.EC

269 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 1.EC

270 if len(gpio) == 0: 1.EC

271 return [None] * 4 if as_dicts else None 

272 else: 

273 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 2. ] c F kb, G R s D y S - C z A H a

274 # This deals with missing and empty files the same 

275 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 2. ] c F kb, G R s D y S - C z A H a

276 # Check values make sense (4 pins = 16 possible values) 

277 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 2. ] c F kb, G R s D y S - C z A H a

278 _logger.warning('Unexpected GPIO values; decoding may fail') 1.

279 if len(gpio) == 0: 2. ] c F kb, G R s D y S - C z A H a

280 return [None] * 4 if as_dicts else None 2. c F kb, G s a

281 # 4 pins represented as uint32 

282 # For each pin, shift its bit to the end and check the bit is set 

283 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 1.]RDyS-CzAH

284 

285 if as_dicts: 1.]RDySE-CzAH

286 if not gpio.any(): 1.]RySE-CzAH

287 _logger.error('No GPIO changes') 1.

288 return [None] * 4 1.

289 # Find state changes for each pin and construct a dict of indices and polarities for each 

290 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 1.]RySE-CzAH

291 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)] 

292 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8 

293 gpio = [{'indices': np.where(edges[:, i])[0], 1.]RySE-CzAH

294 'polarities': edges[edges[:, i] != 0, i]} 

295 for i in range(4)] 

296 # Replace empty dicts with None 

297 gpio = [None if x['indices'].size == 0 else x for x in gpio] 1.]RySE-CzAH

298 

299 return gpio 1.]RDySE-CzAH

300 

301 

302def _read_settings_json_compatibility_enforced(settings): 

303 """ 

304 Patch iblrig settings for compatibility across rig versions. 

305 

306 Parameters 

307 ---------- 

308 settings : pathlib.Path, dict 

309 Either a _iblrig_taskSettings.raw.json file path or the loaded settings. 

310 

311 Returns 

312 ------- 

313 dict 

314 The task settings patched for compatibility. 

315 """ 

316 if isinstance(settings, dict): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

317 md = settings.copy() 

318 else: 

319 with open(settings) as js: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

320 md = json.load(js) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

321 if 'IS_MOCK' not in md: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

322 md['IS_MOCK'] = False 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ ~ ab) * + bbcbdbw h M o P j k Q # F , G s y ! z A m n i O ' t l u ` p q N a % ( g r v d

323 # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag. 

324 # Some sessions have neither key. From v8 onwards we will use IBLRIG_VERSION to test rig 

325 # version, however some places may still use the version tag. 

326 if 'IBLRIG_VERSION_TAG' not in md.keys(): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

327 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 2_ = ? @ ~ ab) * + bbcbdbQ i

328 if 'IBLRIG_VERSION' not in md.keys(): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

329 md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG'] 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} [ ~ ab) * + bbcbdbw h M o P ebx j k Q # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d

330 elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]): 1_=?@ciO

331 # This may not be an issue; not sure what the intended difference between these keys was 

332 assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch' 1_=?@ciO

333 # Test version can be parsed. If not, log an error and set the version to nothing 

334 try: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

335 version.parse(md['IBLRIG_VERSION'] or '0') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

336 except version.InvalidVersion as ex: 

337 _logger.error('%s in iblrig settings, this may affect extraction', ex) 

338 # try a more relaxed version parse 

339 laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION']) 

340 # Set the tag as the invalid version 

341 md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION'] 

342 # overwrite version with either successfully parsed one or an empty string 

343 md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else '' 

344 if 'device_sound' not in md: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

345 # sound device must be defined in version 8 and later # FIXME this assertion will cause tests to break 

346 assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d

347 # in v7 we must infer the device from the sampling frequency if SD is None 

348 if 'sounddevice' in md.get('SD', ''): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d

349 device = 'xonar' 1bwoPjk#FGy!zAO'a%gd

350 else: 

351 freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'} 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbh M ebx , R s D S E fbgbhbm B n t l u ` p q N a % ( g r v d

352 device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbh M ebx , R s D S E fbgbhbm B n t l u ` p q N a % ( g r v d

353 md['device_sound'] = {'OUTPUT': device} 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d

354 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on) 

355 if md['IBLRIG_VERSION'] == '': 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

356 pass 2T jb~ ab) * + bbcbdbQ # m d

357 elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ w h M o P ebx j k c F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

358 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 1i

359 md['PYBPOD_BOARD'] = md['RIG_NAME'] 1i

360 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 1i

361 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 1i

362 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 1i

363 elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ w h M o P ebx j k c F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d

364 if 'LAST_TRIAL_DATA' in md.keys(): 1a

365 md.pop('LAST_TRIAL_DATA') 1a

366 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1a

367 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1a

368 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1a

369 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1a

370 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1a

371 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1a

372 # parse the date format to Django supported ISO 

373 dt = dateparser.parse(md['SESSION_DATETIME']) 1a

374 md['SESSION_DATETIME'] = date2isostr(dt) 1a

375 # add the weight key if it doesn't already exist 

376 if 'SUBJECT_WEIGHT' not in md: 1a

377 md['SUBJECT_WEIGHT'] = None 1a

378 return md 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

379 

380 

381def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'): 

382 """ 

383 Load PyBpod Settings files (.json). 

384 

385 [description] 

386 

387 :param session_path: Absolute path of session folder 

388 :type session_path: str, Path 

389 :return: Settings dictionary 

390 :rtype: dict 

391 """ 

392 if session_path is None: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

393 _logger.warning("No data loaded: session_path is None") 1[

394 return 1[

395 path = Path(session_path).joinpath(task_collection) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

396 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

397 if not path: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

398 _logger.warning("No data loaded: could not find raw settings file") 1^[=O`

399 return None 1^[=O`

400 settings = _read_settings_json_compatibility_enforced(path) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

401 return settings 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d

402 

403 

404def load_stim_position_screen(session_path, task_collection='raw_behavior_data'): 

405 path = Path(session_path).joinpath(task_collection) 

406 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None) 

407 

408 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip') 

409 data.columns = ['contrast', 'position', 'bns_ts'] 

410 data['bns_ts'] = pd.to_datetime(data['bns_ts']) 

411 return data 

412 

413 

414def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False): 

415 """ 

416 Load Rotary Encoder (RE) events raw data file. 

417 

418 Assumes that a folder called "raw_behavior_data" exists in folder. 

419 

420 Events number correspond to following bpod states: 

421 1: correct / hide_stim 

422 2: stim_on 

423 3: closed_loop 

424 4: freeze_error / freeze_correct 

425 

426 >>> data.columns 

427 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

428 'sm_ev', # State Machine Event 'numpy.int64' 

429 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp' 

430 # pd.to_datetime(data.bns_ts) to work in datetimes 

431 

432 :param session_path: [description] 

433 :type session_path: [type] 

434 :return: dataframe w/ 3 cols and (ntrials * 3) lines 

435 :rtype: Pandas.DataFrame 

436 """ 

437 if session_path is None: 1bf$ewhoxjkcmBnitlupqagrvd

438 return 

439 path = Path(session_path).joinpath(task_collection) 1bf$ewhoxjkcmBnitlupqagrvd

440 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 1bf$ewhoxjkcmBnitlupqagrvd

441 if not settings: 1bf$ewhoxjkcmBnitlupqagrvd

442 settings = load_settings(session_path, task_collection=task_collection) 1bf$ewhoxjkcmBnitlupqagrvd

443 if settings is None or not settings.get('IBLRIG_VERSION'): 1bf$ewhoxjkcmBnitlupqagrvd

444 settings = {'IBLRIG_VERSION': '100.0.0'} 1d

445 # auto-detect old files when version is not labeled 

446 with open(path) as fid: 1d

447 line = fid.readline() 1d

448 if line.startswith('Event') and 'StateMachine' in line: 1d

449 settings = {'IBLRIG_VERSION': '0.0.0'} 1d

450 if not path: 1bf$ewhoxjkcmBnitlupqagrvd

451 return None 

452 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 1bf$ewhoxjkcmBnitlupqagrvd

453 return _load_encoder_events_file_ge5(path) 1f$ewhoxjkmBnitluagrvd

454 else: 

455 return _load_encoder_events_file_lt5(path) 1bf$ecpqagd

456 

457 

458def _load_encoder_ssv_file(file_path, **kwargs): 

459 file_path = Path(file_path) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

460 if file_path.stat().st_size == 0: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

461 _logger.error(f"{file_path.name} is an empty file. ") 

462 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ") 

463 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

464 

465 

466def _load_encoder_positions_file_lt5(file_path): 

467 """ 

468 File loader without the session overhead 

469 :param file_path: 

470 :return: dataframe of encoder events 

471 """ 

472 data = _load_encoder_ssv_file(file_path, 1b;{fIe/cpqagd

473 names=['_', 're_ts', 're_pos', 'bns_ts', '__'], 

474 usecols=['re_ts', 're_pos', 'bns_ts']) 

475 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 1b;{fIe/cpqagd

476 

477 

478def _load_encoder_positions_file_ge5(file_path): 

479 """ 

480 File loader without the session overhead 

481 :param file_path: 

482 :return: dataframe of encoder events 

483 """ 

484 data = _load_encoder_ssv_file(file_path, 1;fIe/whoxjkmBnitluagrvd

485 names=['re_ts', 're_pos', '_'], 

486 usecols=['re_ts', 're_pos']) 

487 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 1;fIe/whoxjkmBnitluagrvd

488 

489 

490def _load_encoder_events_file_lt5(file_path): 

491 """ 

492 File loader without the session overhead 

493 :param file_path: 

494 :return: dataframe of encoder events 

495 """ 

496 data = _load_encoder_ssv_file(file_path, 1bf$e:cpqagd

497 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'], 

498 usecols=['re_ts', 'sm_ev', 'bns_ts']) 

499 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 1bf$e:cpqagd

500 

501 

502def _load_encoder_events_file_ge5(file_path): 

503 """ 

504 File loader without the session overhead 

505 :param file_path: 

506 :return: dataframe of encoder events 

507 """ 

508 data = _load_encoder_ssv_file(file_path, 1f$e:whoxjkmBnitluagrvd

509 names=['re_ts', 'sm_ev', '_'], 

510 usecols=['re_ts', 'sm_ev']) 

511 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 1f$e:whoxjkmBnitluagrvd

512 

513 

514def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False): 

515 """ 

516 Load Rotary Encoder (RE) positions from raw data file within a session path. 

517 

518 Assumes that a folder called "raw_behavior_data" exists in folder. 

519 Positions are RE ticks [-512, 512] == [-180º, 180º] 

520 0 == trial stim init position 

521 Positive nums are rightwards movements (mouse) or RE CW (mouse) 

522 

523 Variable line number, depends on movements. 

524 

525 Raw datafile Columns: 

526 Position, RE timestamp, RE Position, Bonsai Timestamp 

527 

528 Position is always equal to 'Position' so this column was dropped. 

529 

530 >>> data.columns 

531 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

532 're_pos', # Rotary Encoder position (ticks) 'numpy.int64' 

533 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp' 

534 # pd.to_datetime(data.bns_ts) to work in datetimes 

535 

536 :param session_path: Absolute path of session folder 

537 :type session_path: str 

538 :return: dataframe w/ 3 cols and N positions 

539 :rtype: Pandas.DataFrame 

540 """ 

541 if session_path is None: 1bfIewhoxjkcsmBnitlupqagrvd

542 return 

543 path = Path(session_path).joinpath(task_collection) 1bfIewhoxjkcsmBnitlupqagrvd

544 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 1bfIewhoxjkcsmBnitlupqagrvd

545 if not settings: 1bfIewhoxjkcsmBnitlupqagrvd

546 settings = load_settings(session_path, task_collection=task_collection) 1bfIewhoxjkcsmBnitlupqagrvd

547 if settings is None or not settings.get('IBLRIG_VERSION'): 1bfIewhoxjkcsmBnitlupqagrvd

548 settings = {'IBLRIG_VERSION': '100.0.0'} 1d

549 # auto-detect old files when version is not labeled 

550 with open(path) as fid: 1d

551 line = fid.readline() 1d

552 if line.startswith('Position'): 1d

553 settings = {'IBLRIG_VERSION': '0.0.0'} 1d

554 if not path: 1bfIewhoxjkcsmBnitlupqagrvd

555 _logger.warning("No data loaded: could not find raw encoderPositions file") 1sa

556 return None 1sa

557 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 1bfIewhoxjkcmBnitlupqagrvd

558 return _load_encoder_positions_file_ge5(path) 1fIewhoxjkmBnitluagrvd

559 else: 

560 return _load_encoder_positions_file_lt5(path) 1bfIecpqagd

561 

562 

563def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'): 

564 """ 

565 Load Rotary Encoder trial info from raw data file. 

566 

567 Assumes that a folder calles "raw_behavior_data" exists in folder. 

568 

569 NOTE: Last trial probably inexistent data (Trial info is sent on trial start 

570 and data is only saved on trial exit...) max(trialnum) should be N+1 if N 

571 is the amount of trial data saved. 

572 

573 Raw datafile Columns: 

574 

575 >>> data.columns 

576 >>> ['trial_num', # Trial Number 'numpy.int64' 

577 'stim_pos_init', # Initial position of visual stim 'numpy.int64' 

578 'stim_contrast', # Contrast of visual stimulus 'numpy.float64' 

579 'stim_freq', # Frequency of gabor patch 'numpy.float64' 

580 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64' 

581 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64' 

582 'stim_sigma', # Size of patch 'numpy.float64' 

583 'stim_phase', # Phase of gabor 'numpy.float64' 

584 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp' 

585 # pd.to_datetime(data.bns_ts) to work in datetimes 

586 

587 :param session_path: Absoulte path of session folder 

588 :type session_path: str 

589 :return: dataframe w/ 9 cols and ntrials lines 

590 :rtype: Pandas.DataFrame 

591 """ 

592 if session_path is None: 2qb

593 return 2qb

594 path = Path(session_path).joinpath(task_collection) 2qb

595 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 2qb

596 if not path: 2qb

597 return None 2qb

598 data = pd.read_csv(path, sep=' ', header=None) 2qb

599 data = data.drop([9], axis=1) 2qb

600 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 2qb

601 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts'] 

602 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path) 

603 return data 2qb

604 

605 

606def load_ambient_sensor(session_path, task_collection='raw_behavior_data'): 

607 """ 

608 Load Ambient Sensor data from session. 

609 

610 Probably could be extracted to DatasetTypes: 

611 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb, 

612 _ibl_trials.relativeHumidity 

613 Returns a list of dicts one dict per trial. 

614 dict keys are: 

615 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity']) 

616 

617 :param session_path: Absoulte path of session folder 

618 :type session_path: str 

619 :return: list of dicts 

620 :rtype: list 

621 """ 

622 if session_path is None: 

623 return 

624 path = Path(session_path).joinpath(task_collection) 

625 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None) 

626 if not path: 

627 return None 

628 data = [] 

629 with open(path, 'r') as f: 

630 for line in f: 

631 data.append(json.loads(line)) 

632 return data 

633 

634 

635def load_mic(session_path, task_collection='raw_behavior_data'): 

636 """ 

637 Load Microphone wav file to np.array of len nSamples 

638 

639 :param session_path: Absolute path of session folder 

640 :type session_path: str 

641 :return: An array of values of the sound waveform 

642 :rtype: numpy.array 

643 """ 

644 if session_path is None: 

645 return 

646 path = Path(session_path).joinpath(task_collection) 

647 path = next(path.glob("_iblrig_micData.raw*.wav"), None) 

648 if not path: 

649 return None 

650 fp = wave.open(path) 

651 nchan = fp.getnchannels() 

652 N = fp.getnframes() 

653 dstr = fp.readframes(N * nchan) 

654 data = np.frombuffer(dstr, np.int16) 

655 data = np.reshape(data, (-1, nchan)) 

656 return data 

657 

658 

659def _clean_wheel_dataframe(data, label, path): 

660 if np.any(data.isna()): 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

661 _logger.warning(label + ' has missing/incomplete records \n %s', path) 1b{fIe:/pqgd

662 # first step is to re-interpret as numeric objects if not already done 

663 for col in data.columns: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

664 if data[col].dtype == object and col not in ['bns_ts']: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

665 data[col] = pd.to_numeric(data[col], errors='coerce') 1:d

666 # then drop Nans and duplicates 

667 data.dropna(inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

668 data.drop_duplicates(keep='first', inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

669 data.reset_index(inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

670 # handle the clock resets when microseconds exceed uint32 max value 

671 drop_first = False 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

672 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

673 if any(np.diff(data['re_ts']) < 0): 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

674 ind = np.where(np.diff(data['re_ts']) < 0)[0] 1;{fIe:/hojkcmnilad

675 for i in ind: 1;{fIe:/hojkcmnilad

676 # the first sample may be corrupt, in this case throw away 

677 if i <= 1: 1;{fIe:/hojkcmnilad

678 drop_first = i 1;Ie/ca

679 _logger.warning(label + ' rotary encoder positions timestamps' 1;Ie/ca

680 ' first sample corrupt ' + str(path)) 

681 # if it's an uint32 wraparound, the diff should be close to 2 ** 32 

682 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 1;{fIe:/hojkmnild

683 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 1{fIe/hojkmnd

684 # there is also the case where 2 positions are swapped and need to be swapped back 

685 

686 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 1;Ie:/hjkild

687 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 1;Ie/hjkild

688 str(i) + ' ' + str(path)) 

689 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 1;Ie/hjkild

690 data.iloc[i], data.iloc[i + 1] = b, a 1;Ie/hjkild

691 # if none of those 3 cases apply, raise an error 

692 else: 

693 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1:

694 data.sort_values('re_ts', inplace=True) 1:

695 data.reset_index(inplace=True) 1:

696 if drop_first is not False: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

697 data.drop(data.loc[:drop_first].index, inplace=True) 1;Ie/ca

698 data = data.reindex() 1;Ie/ca

699 return data 1b;{f$Ie:/whoxjkcmBnitlupqagrvd

700 

701 

702def _groom_wheel_data_lt5(data, label='file ', path=''): 

703 """ 

704 The whole purpose of this function is to account for variability and corruption in 

705 the wheel position files. There are many possible errors described below, but 

706 nothing excludes getting new ones. 

707 """ 

708 data = _clean_wheel_dataframe(data, label, path) 1b;{f$Ie:/cpqagd

709 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 1b;{f$Ie:/cpqagd

710 # check if the time scale is in ms 

711 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 1b;{f$Ie:/cpqagd

712 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds 

713 if data['re_ts'].iloc[-1] / (sess_len_sec + 1e-6) < 1e5: # should be 1e6 normally 1b;{f$Ie:/cpqagd

714 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 1$e:ca

715 'RE firmware needs upgrading and wheel velocity is potentially inaccurate') 

716 data['re_ts'] = data['re_ts'] * 1000 1$e:ca

717 return data 1b;{f$Ie:/cpqagd

718 

719 

720def _groom_wheel_data_ge5(data, label='file ', path=''): 

721 """ 

722 The whole purpose of this function is to account for variability and corruption in 

723 the wheel position files. There are many possible errors described below, but 

724 nothing excludes getting new ones. 

725 """ 

726 data = _clean_wheel_dataframe(data, label, path) 1;f$Ie:/whoxjkmBnitluagrvd

727 # check if the time scale is in ms 

728 if (data['re_ts'].iloc[-1] - data['re_ts'].iloc[0]) / 1e6 < 20: 1;f$Ie:/whoxjkmBnitluagrvd

729 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 1:/grd

730 'RE firmware needs upgrading and wheel velocity is potentially inaccurate') 

731 data['re_ts'] = data['re_ts'] * 1000 1:/grd

732 return data 1;f$Ie:/whoxjkmBnitluagrvd

733 

734 

735def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5, 

736 return_index=False): 

737 """ 

738 Attempts to find matching timestamps in 2 time-series that have an offset, are drifting, 

739 and are most likely incomplete: sizes don't have to match, some pulses may be missing 

740 in any series. 

741 Only works with irregular time series as it relies on the derivative to match sync. 

742 :param t0: 

743 :param t1: 

744 :param diff_threshold: 

745 :param drift_threshold_ppm: (150) 

746 :param max_shift: (200) 

747 :param return_index (False) 

748 :return: 

749 """ 

750 nsync = min(t0.size, t1.size) 2iba d

751 dt0 = np.diff(t0) 2iba d

752 dt1 = np.diff(t1) 2iba d

753 ind = np.zeros_like(dt0) * np.nan 2iba d

754 i0 = 0 2iba d

755 i1 = 0 2iba d

756 cdt = np.nan # the current time difference between the two series to compute drift 2iba d

757 while i0 < (nsync - 1): 2iba d

758 # look in the next max_shift events the ones whose derivative match 

759 isearch = np.arange(i1, min(max_shift + i1, dt1.size)) 2iba d

760 dec = np.abs(dt0[i0] - dt1[isearch]) < diff_threshold 2iba d

761 # another constraint is to check the dt for the maximum drift 

762 if ~np.isnan(cdt): 2iba d

763 drift_ppm = np.abs((cdt - (t0[i0] - t1[isearch])) / dt1[isearch]) * 1e6 2iba d

764 dec = np.logical_and(dec, drift_ppm <= drift_threshold_ppm) 2iba d

765 # if one is found 

766 if np.any(dec): 2iba d

767 ii1 = np.where(dec)[0][0] 2iba d

768 ind[i0] = i1 + ii1 2iba d

769 i1 += ii1 + 1 2iba d

770 cdt = t0[i0 + 1] - t1[i1 + ii1] 2iba d

771 i0 += 1 2iba d

772 it0 = np.where(~np.isnan(ind))[0] 2iba d

773 it1 = ind[it0].astype(int) 2iba d

774 ind0 = np.unique(np.r_[it0, it0 + 1]) 2iba d

775 ind1 = np.unique(np.r_[it1, it1 + 1]) 2iba d

776 if return_index: 2iba d

777 return t0[ind0], t1[ind1], ind0, ind1 

778 else: 

779 return t0[ind0], t1[ind1] 2iba d

780 

781 

782def load_bpod_fronts(session_path: str, data: list = False, task_collection: str = 'raw_behavior_data') -> list: 

783 """load_bpod_fronts 

784 Loads BNC1 and BNC2 bpod channels times and polarities from session_path 

785 

786 :param session_path: a valid session_path 

787 :type session_path: str 

788 :param data: pre-loaded raw data dict, defaults to False 

789 :type data: list, optional 

790 :return: List of dicts BNC1 and BNC2 {"times": np.array, "polarities":np.array} 

791 :rtype: list 

792 """ 

793 if not data: 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

794 data = load_data(session_path, task_collection) 1ca

795 

796 BNC1_fronts = np.array([[np.nan, np.nan]]) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

797 BNC2_fronts = np.array([[np.nan, np.nan]]) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

798 for tr in data: 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

799 BNC1_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

800 BNC1_fronts, 

801 np.array( 

802 [ 

803 [x, 1] 

804 for x in tr["behavior_data"]["Events timestamps"].get("BNC1High", [np.nan]) 

805 ] 

806 ), 

807 axis=0, 

808 ) 

809 BNC1_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

810 BNC1_fronts, 

811 np.array( 

812 [ 

813 [x, -1] 

814 for x in tr["behavior_data"]["Events timestamps"].get("BNC1Low", [np.nan]) 

815 ] 

816 ), 

817 axis=0, 

818 ) 

819 BNC2_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

820 BNC2_fronts, 

821 np.array( 

822 [ 

823 [x, 1] 

824 for x in tr["behavior_data"]["Events timestamps"].get("BNC2High", [np.nan]) 

825 ] 

826 ), 

827 axis=0, 

828 ) 

829 BNC2_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

830 BNC2_fronts, 

831 np.array( 

832 [ 

833 [x, -1] 

834 for x in tr["behavior_data"]["Events timestamps"].get("BNC2Low", [np.nan]) 

835 ] 

836 ), 

837 axis=0, 

838 ) 

839 

840 BNC1_fronts = BNC1_fronts[1:, :] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

841 BNC1_fronts = BNC1_fronts[BNC1_fronts[:, 0].argsort()] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

842 BNC2_fronts = BNC2_fronts[1:, :] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

843 BNC2_fronts = BNC2_fronts[BNC2_fronts[:, 0].argsort()] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

844 

845 BNC1 = {"times": BNC1_fronts[:, 0], "polarities": BNC1_fronts[:, 1]} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

846 BNC2 = {"times": BNC2_fronts[:, 0], "polarities": BNC2_fronts[:, 1]} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

847 

848 return [BNC1, BNC2] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv

849 

850 

851def get_port_events(trial: dict, name: str = '') -> list: 

852 """get_port_events 

853 Return all event timestamps from bpod raw data trial that match 'name' 

854 --> looks in trial['behavior_data']['Events timestamps'] 

855 

856 :param trial: raw trial dict 

857 :type trial: dict 

858 :param name: name of event, defaults to '' 

859 :type name: str, optional 

860 :return: Sorted list of event timestamps 

861 :rtype: list 

862 TODO: add polarities? 

863 """ 

864 out: list = [] 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

865 events = trial['behavior_data']['Events timestamps'] 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

866 for k in events: 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

867 if name in k: 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

868 out.extend(events[k]) 1fJKeLwhMoPxjkcQmBnitluNagrv

869 out = sorted(out) 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

870 

871 return out 1bfJKeLwhMoPxjkcQmBnitlupqNagrv

872 

873 

874def load_widefield_mmap(session_path, dtype=np.uint16, shape=(540, 640), n_frames=None, mode='r'): 

875 """ 

876 TODO Document this function 

877 

878 Parameters 

879 ---------- 

880 session_path 

881 

882 Returns 

883 ------- 

884 

885 """ 

886 filepath = Path(session_path).joinpath('raw_widefield_data').glob('widefield.raw.*.dat') 

887 filepath = next(filepath, None) 

888 if not filepath: 

889 _logger.warning("No data loaded: could not find raw data file") 

890 return None 

891 

892 if type(dtype) is str: 

893 dtype = np.dtype(dtype) 

894 

895 if n_frames is None: 

896 # Get the number of samples from the file size 

897 n_frames = int(filepath.stat().st_size / (np.prod(shape) * dtype.itemsize)) 

898 

899 return np.memmap(str(filepath), mode=mode, dtype=dtype, shape=(int(n_frames), *shape)) 

900 

901 

902def patch_settings(session_path, collection='raw_behavior_data', 

903 new_collection=None, subject=None, number=None, date=None): 

904 """Modify various details in a settings file. 

905 

906 This function makes it easier to change things like subject name in a settings as it will 

907 modify the subject name in the myriad paths. NB: This saves the settings into the same location 

908 it was loaded from. 

909 

910 Parameters 

911 ---------- 

912 session_path : str, pathlib.Path 

913 The session path containing the settings file. 

914 collection : str 

915 The subfolder containing the settings file. 

916 new_collection : str 

917 An optional new subfolder to change in the settings paths. 

918 subject : str 

919 An optional new subject name to change in the settings. 

920 number : str, int 

921 An optional new number to change in the settings. 

922 date : str, datetime.date 

923 An optional date to change in the settings. 

924 

925 Returns 

926 ------- 

927 dict 

928 The modified settings. 

929 

930 Examples 

931 -------- 

932 File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location. 

933 >>> patch_settings('/data/subject/2020-01-01/002', number='001') 

934 >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/') 

935 

936 File is moved into new collection within the same session, then patched. 

937 >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/') 

938 >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01') 

939 

940 Update subject, date and number. 

941 >>> new_session_path = Path('/data/foobar/2024-02-24/002') 

942 >>> old_session_path = Path('/data/baz/2024-02-23/001') 

943 >>> patch_settings(old_session_path, collection='raw_task_data_00', 

944 ... subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1]) 

945 >>> shutil.move(old_session_path, new_session_path) 

946 """ 

947 settings = load_settings(session_path, collection) 1b)*+O

948 if not settings: 1b)*+O

949 raise IOError('Settings file not found') 1O

950 

951 filename = PureWindowsPath(settings.get('SETTINGS_FILE_PATH', '_iblrig_taskSettings.raw.json')).name 1b)*+O

952 file_path = Path(session_path).joinpath(collection, filename) 1b)*+O

953 

954 if subject: 1b)*+O

955 # Patch subject name 

956 old_subject = settings['SUBJECT_NAME'] 1bO

957 settings['SUBJECT_NAME'] = subject 1bO

958 for k in settings.keys(): 1bO

959 if isinstance(settings[k], str): 1bO

960 settings[k] = settings[k].replace(f'\\Subjects\\{old_subject}', f'\\Subjects\\{subject}') 1bO

961 if 'SESSION_NAME' in settings: 1bO

962 settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) 1bO

963 settings.pop('PYBPOD_SUBJECT_EXTRA', None) # Get rid of Alyx subject info 1bO

964 

965 if date: 1b)*+O

966 # Patch session datetime 

967 date = str(date) 1bO

968 old_date = settings['SESSION_DATE'] 1bO

969 settings['SESSION_DATE'] = date 1bO

970 for k in settings.keys(): 1bO

971 if isinstance(settings[k], str): 1bO

972 settings[k] = settings[k].replace( 1bO

973 f'\\{settings["SUBJECT_NAME"]}\\{old_date}', 

974 f'\\{settings["SUBJECT_NAME"]}\\{date}' 

975 ) 

976 settings['SESSION_DATETIME'] = date + settings['SESSION_DATETIME'][10:] 1bO

977 if 'SESSION_END_TIME' in settings: 1bO

978 settings['SESSION_END_TIME'] = date + settings['SESSION_END_TIME'][10:] 1O

979 if 'SESSION_START_TIME' in settings: 1bO

980 settings['SESSION_START_TIME'] = date + settings['SESSION_START_TIME'][10:] 1O

981 

982 if number: 1b)*+O

983 # Patch session number 

984 old_number = settings['SESSION_NUMBER'] 1b)*+O

985 if isinstance(number, int): 1b)*+O

986 number = f'{number:03}' 1O

987 settings['SESSION_NUMBER'] = number 1b)*+O

988 for k in settings.keys(): 1b)*+O

989 if isinstance(settings[k], str): 1b)*+O

990 settings[k] = settings[k].replace( 1b)*+O

991 f'\\{settings["SESSION_DATE"]}\\{old_number}', 

992 f'\\{settings["SESSION_DATE"]}\\{number}' 

993 ) 

994 

995 if new_collection: 1b)*+O

996 if 'SESSION_RAW_DATA_FOLDER' not in settings: 1)*+O

997 _logger.warning('SESSION_RAW_DATA_FOLDER key not in settings; collection not updated') 1O

998 else: 

999 old_path = settings['SESSION_RAW_DATA_FOLDER'] 1)*+O

1000 new_path = PureWindowsPath(settings['SESSION_RAW_DATA_FOLDER']).with_name(new_collection) 1)*+O

1001 for k in settings.keys(): 1)*+O

1002 if isinstance(settings[k], str): 1)*+O

1003 settings[k] = settings[k].replace(old_path, str(new_path)) 1)*+O

1004 with open(file_path, 'w') as fp: 1b)*+O

1005 json.dump(settings, fp, indent=' ') 1b)*+O

1006 return settings 1b)*+O