Coverage for ibllib/io/raw_data

1"""

2Raw Data Loader functions for PyBpod rig.

4Module contains one loader function per raw datafile.

5"""

6import re (empty)

7import json (empty)

8import logging (empty)

9import wave (empty)

10from collections import OrderedDict (empty)

11from datetime import datetime (empty)

12from pathlib import Path, PureWindowsPath (empty)

13from typing import Union (empty)

15from dateutil import parser as dateparser (empty)

16from packaging import version (empty)

17import numpy as np (empty)

18import pandas as pd (empty)

20from iblutil.io import jsonable (empty)

21from ibllib.io.video import assert_valid_label (empty)

22from ibllib.time import uncycle_pgts, convert_pgts, date2isostr (empty)

24_logger = logging.getLogger(__name__) (empty)

27def trial_times_to_times(raw_trial): (empty)

28 """

29 Parse and convert all trial timestamps to "absolute" time.

30 Float64 seconds from session start.

32 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1---

33 tsN...absTS = tsN + TrialStartN - BpodStart

35 Bpod timestamps are in microseconds (µs)

36 PyBpod timestamps are is seconds (s)

38 :param raw_trial: raw trial data

39 :type raw_trial: dict

40 :return: trial data with modified timestamps

41 :rtype: dict

42 """

43 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

44 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

45 # ts_te = raw_trial['behavior_data']['Trial end timestamp']

47 def convert(ts): 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

48 return ts + ts_ts - ts_bs 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

50 converted_events = {} 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

51 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

52 converted_events.update({k: [convert(i) for i in v]}) 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

53 raw_trial['behavior_data']['Events timestamps'] = converted_events 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

55 converted_states = {} 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

56 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

57 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

58 raw_trial['behavior_data']['States timestamps'] = converted_states 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

60 shift = raw_trial['behavior_data']['Bpod start timestamp'] 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

61 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

62 raw_trial['behavior_data']['Trial start timestamp'] -= shift 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

63 raw_trial['behavior_data']['Trial end timestamp'] -= shift 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

64 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

65 return raw_trial 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

68def load_bpod(session_path, task_collection='raw_behavior_data'): (empty)

69 """

70 Load both settings and data from bpod (.json and .jsonable)

72 :param session_path: Absolute path of session folder

73 :param task_collection: Collection within sesison path with behavior data

74 :return: dict settings and list of dicts data

75 """

76 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 7 ctx1a9!#h21

79def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'): (empty)

80 """

81 Load PyBpod data files (.jsonable).

83 Bpod timestamps are in microseconds (µs)

84 PyBpod timestamps are is seconds (s)

86 :param session_path: Absolute path of session folder

87 :type session_path: str, Path

88 :return: A list of len ntrials each trial being a dictionary

89 :rtype: list of dicts

90 """

91 if session_path is None: 55 ctx1aKLMNOBPQRSTUIVWXYdC9!#DmGrfghHAiywsvZtxJojuklep2E1bnqc

92 _logger.warning('No data loaded: session_path is None')

93 return

94 path = Path(session_path).joinpath(task_collection) 55 ctx1aKLMNOBPQRSTUIVWXYdC9!#DmGrfghHAiywsvZtxJojuklep2E1bnqc

95 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 55 ctx1aKLMNOBPQRSTUIVWXYdC9!#DmGrfghHAiywsvZtxJojuklep2E1bnqc

96 if not path: 55 ctx1aKLMNOBPQRSTUIVWXYdC9!#DmGrfghHAiywsvZtxJojuklep2E1bnqc

97 _logger.warning('No data loaded: could not find raw data file') 3 ctx19!#

98 return None 3 ctx19!#

99 data = jsonable.read(path) 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

100 if time == 'absolute': 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

101 data = [trial_times_to_times(t) for t in data] 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

102 return data 52 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklep2E1bnqc

103

104

105def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: (empty)

106 """Loads binary frame data from Bonsai camera recording workflow.

107

108 Args:

109 session_path (StrPath): Path to session folder

110 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'.

111 raw (bool, optional): Whether to return raw or parsed data. Defaults to False.

112

113 Returns:

114 parsed: (raw=False, Default)

115 pandas.DataFrame: 4 int64 columns: {

116 Timestamp, # float64 (seconds from session start)

117 embeddedTimeStamp, # float64 (seconds from session start)

118 embeddedFrameCounter, # int64 (Frame number from session start)

119 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a

120 # list of numpy boolean arrays

121 # e.g. np.array([True, False, False, False])

122 }

123 raw:

124 pandas.DataFrame: 4 int64 columns: {

125 Timestamp, # UTC ticks from BehaviorPC

126 # (100's of ns since midnight 1/1/0001)

127 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion)

128 embeddedFrameCounter, # Frame counter (int)

129 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins

130 }

131 """

132 camera = assert_valid_label(camera) 5 ctx1a@5vo

133 fpath = Path(session_path).joinpath("raw_video_data") 5 ctx1a@5vo

134 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 5 ctx1a@5vo

135 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 5 ctx1a@5vo

136 rdata = np.fromfile(fpath, dtype=np.float64) 5 ctx1a@5vo

137 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 5 ctx1a@5vo

138 rows = int(rdata.size / 4) 5 ctx1a@5vo

139 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 5 ctx1a@5vo

140 df_dict = dict.fromkeys( 5 ctx1a@5vo

141 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"]

142 )

143 df = pd.DataFrame(data, columns=df_dict.keys()) 5 ctx1a@5vo

144 if raw: 5 ctx1a@5vo

145 return df 1 ctx1@

146

147 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 5 ctx1a@5vo

148 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 5 ctx1a@5vo

149 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 5 ctx1a@5vo

150 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 5 ctx1a@5vo

151 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 5 ctx1a@5vo

152 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 5 ctx1a@5vo

153

154 parsed_df = pd.DataFrame.from_dict(df_dict) 5 ctx1a@5vo

155 return parsed_df 5 ctx1a@5vo

156

157

158def load_camera_ssv_times(session_path, camera: str): (empty)

159 """

160 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv

161

162 NB: For some sessions the frame times are in the first column, in others the order is reversed.

163 NB: If using the new bin file the bonsai_times is a float in seconds since first frame

164 :param session_path: Absolute path of session folder

165 :param camera: Name of the camera to load, e.g. 'left'

166 :return: array of datetimes, array of frame times in seconds

167 """

168 camera = assert_valid_label(camera) 13 ctx1][3iyws^4vtxo

169 video_path = Path(session_path).joinpath('raw_video_data') 13 ctx1][3iyws^4vtxo

170 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 13 ctx1][3iyws^4vtxo

171 df = load_camera_frameData(session_path, camera=camera) 2 ctx1vo

172 return df['Timestamp'].values, df['embeddedTimeStamp'].values 2 ctx1vo

173

174 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 12 ctx1][3iyws^4vtx

175 if not file: 12 ctx1][3iyws^4vtx

176 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 1 ctx1[

177 raise FileNotFoundError(file + ' not found') 1 ctx1[

178 # NB: Numpy has deprecated support for non-naive timestamps.

179 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s.

180 # from datetime import timezone

181 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)}

182

183 # Determine the order of the columns by reading one line and testing whether the first value

184 # is an integer or not.

185 with open(file, 'r') as f: 12 ctx1][3iyws^4vtx

186 line = f.readline() 12 ctx1][3iyws^4vtx

187 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 12 ctx1][3iyws^4vtx

188 try: 12 ctx1][3iyws^4vtx

189 int(line.split(' ')[1]) 12 ctx1][3iyws^4vtx

190 except ValueError: 6 ctx1[iystx

191 type_map.move_to_end('bonsai') 6 ctx1[iystx

192 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 12 ctx1][3iyws^4vtx

193 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 12 ctx1][3iyws^4vtx

194 bonsai_times = ssv_times['bonsai'] 12 ctx1][3iyws^4vtx

195 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 12 ctx1][3iyws^4vtx

196 return bonsai_times, camera_times 12 ctx1][3iyws^4vtx

197

198

199def load_embedded_frame_data(session_path, label: str, raw=False): (empty)

200 """

201 Load the embedded frame count and GPIO for a given session. If the file doesn't exist,

202 or is empty, None values are returned.

203

204 :param session_path: Absolute path of session folder

205 :param label: The specific video to load, one of ('left', 'right', 'body')

206 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is

207 returned starting from 0 and the GPIO is returned as a dict of indices

208 :return: The frame count, GPIO

209 """

210 count = load_camera_frame_count(session_path, label, raw=raw) 12 ctx1%?3iyws4vtxo

211 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 12 ctx1%?3iyws4vtxo

212 return count, gpio 12 ctx1%?3iyws4vtxo

213

214

215def load_camera_frame_count(session_path, label: str, raw=True): (empty)

216 """

217 Load the embedded frame count for a given session. If the file doesn't exist, or is empty,

218 a None value is returned.

219 :param session_path: Absolute path of session folder

220 :param label: The specific video to load, one of ('left', 'right', 'body')

221 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is

222 returned starting from 0

223 :return: The frame count

224 """

225 if session_path is None: 13 ctx1_%?3iyws4vtxo

226 return 1 ctx1_

227

228 label = assert_valid_label(label) 13 ctx1_%?3iyws4vtxo

229 video_path = Path(session_path).joinpath('raw_video_data') 13 ctx1_%?3iyws4vtxo

230 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 13 ctx1_%?3iyws4vtxo

231 df = load_camera_frameData(session_path, camera=label) 2 ctx1vo

232 return df['embeddedFrameCounter'].values 2 ctx1vo

233

234 # Load frame count

235 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 12 ctx1_%?3iyws4vtx

236 count_file = next(glob, None) 12 ctx1_%?3iyws4vtx

237 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 12 ctx1_%?3iyws4vtx

238 if len(count) == 0: 12 ctx1_%?3iyws4vtx

239 return 3 ctx1_?y

240 if not raw: 10 ctx1_%3iws4vtx

241 count -= count[0] # start from zero 9 ctx1_%3is4vtx

242 return count 10 ctx1_%3iws4vtx

243

244

245def load_camera_gpio(session_path, label: str, as_dicts=False): (empty)

246 """

247 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is

248 returned.

249

250 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits

251 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by

252 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state).

253

254 :param session_path: Absolute path of session folder

255 :param label: The specific video to load, one of ('left', 'right', 'body')

256 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins)

257 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities').

258 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4.

259 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'),

260 or None if the dictionary is empty.

261 """

262 if session_path is None: 13 ctx15%?3iyws4vtxo

263 return 1 ctx15

264 raw_path = Path(session_path).joinpath('raw_video_data') 13 ctx15%?3iyws4vtxo

265 label = assert_valid_label(label) 13 ctx15%?3iyws4vtxo

266

267 # Load pin state

268 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 13 ctx15%?3iyws4vtxo

269 df = load_camera_frameData(session_path, camera=label, raw=False) 3 ctx15vo

270 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 3 ctx15vo

271 if len(gpio) == 0: 3 ctx15vo

272 return [None] * 4 if as_dicts else None

273 else:

274 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 12 ctx15%?3iyws4vtx

275 # This deals with missing and empty files the same

276 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 12 ctx15%?3iyws4vtx

277 # Check values make sense (4 pins = 16 possible values)

278 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 12 ctx15%?3iyws4vtx

279 _logger.warning('Unexpected GPIO values; decoding may fail') 1 ctx15

280 if len(gpio) == 0: 12 ctx15%?3iyws4vtx

281 return [None] * 4 if as_dicts else None 3 ctx15?y

282 # 4 pins represented as uint32

283 # For each pin, shift its bit to the end and check the bit is set

284 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 10 ctx15%3iws4vtx

285

286 if as_dicts: 11 ctx15%3iws4vtxo

287 if not gpio.any(): 10 ctx15%3is4vtxo

288 _logger.error('No GPIO changes') 1 ctx15

289 return [None] * 4 1 ctx15

290 # Find state changes for each pin and construct a dict of indices and polarities for each

291 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 10 ctx15%3is4vtxo

292 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)]

293 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8

294 gpio = [{'indices': np.where(edges[:, i])[0], 10 ctx15%3is4vtxo

295 'polarities': edges[edges[:, i] != 0, i]}

296 for i in range(4)]

297 # Replace empty dicts with None

298 gpio = [None if x['indices'].size == 0 else x for x in gpio] 10 ctx15%3is4vtxo

299

300 return gpio 11 ctx15%3iws4vtxo

301

302

303def _read_settings_json_compatibility_enforced(settings): (empty)

304 """

305 Patch iblrig settings for compatibility across rig versions.

306

307 Parameters

308 ----------

309 settings : pathlib.Path, dict

310 Either a _iblrig_taskSettings.raw.json file path or the loaded settings.

311

312 Returns

313 -------

314 dict

315 The task settings patched for compatibility.

316 """

317 if isinstance(settings, dict): 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

318 md = settings.copy()

319 else:

320 with open(settings) as js: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

321 md = json.load(js) 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

322 if 'IS_MOCK' not in md: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

323 md['IS_MOCK'] = False 57 ctx1aKLMNOBPQRSTUIVWXY0zdC'*+($9!#DmGfgHAisZtJojklepF2E1,bnqc

324 # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag.

325 # Some sessions have neither key. From v8 onwards we will use IBLRIG_VERSION to test rig

326 # version, however some places may still use the version tag.

327 if 'IBLRIG_VERSION_TAG' not in md.keys(): 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

328 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 10 ctx1(9!#HAJoep

329 if 'IBLRIG_VERSION' not in md.keys(): 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

330 md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG'] 56 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+$DmG-rfgHiwsZt./:juklF2E1,bnqc

331 elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]): 11 ctx1(9!#hAJoepF

332 # This may not be an issue; not sure what the intended difference between these keys was

333 assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch' 11 ctx1(9!#hAJoepF

334 # Test version can be parsed. If not, log an error and set the version to nothing

335 try: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

336 version.parse(md['IBLRIG_VERSION'] or '0') 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

337 except version.InvalidVersion as ex:

338 _logger.error('%s in iblrig settings, this may affect extraction', ex)

339 # try a more relaxed version parse

340 laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION'])

341 # Set the tag as the invalid version

342 md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION']

343 # overwrite version with either successfully parsed one or an empty string

344 md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else ''

345 if 'device_sound' not in md: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

346 # sound device must be defined in version 8 and later # FIXME this assertion will cause tests to break

347 assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0') 59 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfgiwsZt./:juklF2E1,bnqc

348 # in v7 we must infer the device from the sampling frequency if SD is None

349 if 'sounddevice' in md.get('SD', ''): 59 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfgiwsZt./:juklF2E1,bnqc

350 device = 'xonar' 14 ctx1amGfgisZtF21bc

351 else:

352 freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'} 49 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#D-rw./:juklE1,bnqc

353 device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown') 49 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#D-rw./:juklE1,bnqc

354 md['device_sound'] = {'OUTPUT': device} 59 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfgiwsZt./:juklF2E1,bnqc

355 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on)

356 if md['IBLRIG_VERSION'] == '': 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

357 pass 5 ctx1I=Hjc

358 elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'): 64 ctx1aKLMNOBPQRSTUIVWXY0zdC'*+($9!#DmG-rfghAiwsZtJo./:juklepF2E1,bnqc

359 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 5 ctx1AJoep

360 md['PYBPOD_BOARD'] = md['RIG_NAME'] 5 ctx1AJoep

361 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 5 ctx1AJoep

362 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 5 ctx1AJoep

363 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 5 ctx1AJoep

364 elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'): 59 ctx1aKLMNOBPQRSTUIVWXY0zdC'*+($9!#DmG-rfghiwsZt./:juklF2E1,bnqc

365 if 'LAST_TRIAL_DATA' in md.keys(): 1 ctx1b

366 md.pop('LAST_TRIAL_DATA') 1 ctx1b

367 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

368 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1 ctx1b

369 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

370 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1 ctx1b

371 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1 ctx1b

372 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1 ctx1b

373 # parse the date format to Django supported ISO

374 dt = dateparser.parse(md['SESSION_DATETIME']) 1 ctx1b

375 md['SESSION_DATETIME'] = date2isostr(dt) 1 ctx1b

376 # add the weight key if it doesn't already exist

377 if 'SUBJECT_WEIGHT' not in md: 1 ctx1b

378 md['SUBJECT_WEIGHT'] = None 1 ctx1b

379 return md 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

380

381

382def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'): (empty)

383 """

384 Load PyBpod Settings files (.json).

385

386 [description]

387

388 :param session_path: Absolute path of session folder

389 :type session_path: str, Path

390 :return: Settings dictionary

391 :rtype: dict

392 """

393 if session_path is None: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

394 _logger.warning("No data loaded: session_path is None") 1 ctx1$

395 return 1 ctx1$

396 path = Path(session_path).joinpath(task_collection) 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

397 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

398 if not path: 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

399 _logger.warning("No data loaded: could not find raw settings file") 4 ctx1'$9F

400 return None 4 ctx1'$9F

401 settings = _read_settings_json_compatibility_enforced(path) 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

402 return settings 66 ctx1aKLMNOBPQRSTUIVWXY0zdC'*=+($9!#DmG-rfghHAiwsZtJo./:juklepF2E1,bnqc

403

404

405def load_stim_position_screen(session_path, task_collection='raw_behavior_data'): (empty)

406 path = Path(session_path).joinpath(task_collection)

407 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None)

408

409 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip')

410 data.columns = ['contrast', 'position', 'bns_ts']

411 data['bns_ts'] = pd.to_datetime(data['bns_ts'])

412 return data

413

414

415def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False): (empty)

416 """

417 Load Rotary Encoder (RE) events raw data file.

418

419 Assumes that a folder called "raw_behavior_data" exists in folder.

420

421 Events number correspond to following bpod states:

422 1: correct / hide_stim

423 2: stim_on

424 3: closed_loop

425 4: freeze_error / freeze_correct

426

427 >>> data.columns

428 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'

429 'sm_ev', # State Machine Event 'numpy.int64'

430 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp'

431 # pd.to_datetime(data.bns_ts) to work in datetimes

432

433 :param session_path: [description]

434 :type session_path: [type]

435 :return: dataframe w/ 3 cols and (ntrials * 3) lines

436 :rtype: Pandas.DataFrame

437 """

438 if session_path is None: 18 ctx1a0dmrfghjuklepbnqc

439 return

440 path = Path(session_path).joinpath(task_collection) 18 ctx1a0dmrfghjuklepbnqc

441 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 18 ctx1a0dmrfghjuklepbnqc

442 if not settings: 18 ctx1a0dmrfghjuklepbnqc

443 settings = load_settings(session_path, task_collection=task_collection) 18 ctx1a0dmrfghjuklepbnqc

444 if settings is None or not settings.get('IBLRIG_VERSION'): 18 ctx1a0dmrfghjuklepbnqc

445 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1c

446 # auto-detect old files when version is not labeled

447 with open(path) as fid: 1 ctx1c

448 line = fid.readline() 1 ctx1c

449 if line.startswith('Event') and 'StateMachine' in line: 1 ctx1c

450 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1c

451 if not path: 18 ctx1a0dmrfghjuklepbnqc

452 return None

453 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 18 ctx1a0dmrfghjuklepbnqc

454 return _load_encoder_events_file_ge5(path) 16 ctx10dmrfgjuklepbnqc

455 else:

456 return _load_encoder_events_file_lt5(path) 6 ctx1a0dhbc

457

458

459def _load_encoder_ssv_file(file_path, **kwargs): (empty)

460 file_path = Path(file_path) 23 ctx1a8)0zd76mrfghjuklepbnqc

461 if file_path.stat().st_size == 0: 23 ctx1a8)0zd76mrfghjuklepbnqc

462 _logger.error(f"{file_path.name} is an empty file. ")

463 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ")

464 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 23 ctx1a8)0zd76mrfghjuklepbnqc

465

466

467def _load_encoder_positions_file_lt5(file_path): (empty)

468 """

469 File loader without the session overhead

470 :param file_path:

471 :return: dataframe of encoder events

472 """

473 data = _load_encoder_ssv_file(file_path, 9 ctx1a8)zd6hbc

474 names=['_', 're_ts', 're_pos', 'bns_ts', '__'],

475 usecols=['re_ts', 're_pos', 'bns_ts'])

476 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 9 ctx1a8)zd6hbc

477

478

479def _load_encoder_positions_file_ge5(file_path): (empty)

480 """

481 File loader without the session overhead

482 :param file_path:

483 :return: dataframe of encoder events

484 """

485 data = _load_encoder_ssv_file(file_path, 18 ctx18zd6mrfgjuklepbnqc

486 names=['re_ts', 're_pos', '_'],

487 usecols=['re_ts', 're_pos'])

488 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 18 ctx18zd6mrfgjuklepbnqc

489

490

491def _load_encoder_events_file_lt5(file_path): (empty)

492 """

493 File loader without the session overhead

494 :param file_path:

495 :return: dataframe of encoder events

496 """

497 data = _load_encoder_ssv_file(file_path, 7 ctx1a0d7hbc

498 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'],

499 usecols=['re_ts', 'sm_ev', 'bns_ts'])

500 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 7 ctx1a0d7hbc

501

502

503def _load_encoder_events_file_ge5(file_path): (empty)

504 """

505 File loader without the session overhead

506 :param file_path:

507 :return: dataframe of encoder events

508 """

509 data = _load_encoder_ssv_file(file_path, 17 ctx10d7mrfgjuklepbnqc

510 names=['re_ts', 'sm_ev', '_'],

511 usecols=['re_ts', 'sm_ev'])

512 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 17 ctx10d7mrfgjuklepbnqc

513

514

515def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False): (empty)

516 """

517 Load Rotary Encoder (RE) positions from raw data file within a session path.

518

519 Assumes that a folder called "raw_behavior_data" exists in folder.

520 Positions are RE ticks [-512, 512] == [-180º, 180º]

521 0 == trial stim init position

522 Positive nums are rightwards movements (mouse) or RE CW (mouse)

523

524 Variable line number, depends on movements.

525

526 Raw datafile Columns:

527 Position, RE timestamp, RE Position, Bonsai Timestamp

528

529 Position is always equal to 'Position' so this column was dropped.

530

531 >>> data.columns

532 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'

533 're_pos', # Rotary Encoder position (ticks) 'numpy.int64'

534 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp'

535 # pd.to_datetime(data.bns_ts) to work in datetimes

536

537 :param session_path: Absolute path of session folder

538 :type session_path: str

539 :return: dataframe w/ 3 cols and N positions

540 :rtype: Pandas.DataFrame

541 """

542 if session_path is None: 19 ctx1azdmrfghijuklepbnqc

543 return

544 path = Path(session_path).joinpath(task_collection) 19 ctx1azdmrfghijuklepbnqc

545 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 19 ctx1azdmrfghijuklepbnqc

546 if not settings: 19 ctx1azdmrfghijuklepbnqc

547 settings = load_settings(session_path, task_collection=task_collection) 19 ctx1azdmrfghijuklepbnqc

548 if settings is None or not settings.get('IBLRIG_VERSION'): 19 ctx1azdmrfghijuklepbnqc

549 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1c

550 # auto-detect old files when version is not labeled

551 with open(path) as fid: 1 ctx1c

552 line = fid.readline() 1 ctx1c

553 if line.startswith('Position'): 1 ctx1c

554 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1c

555 if not path: 19 ctx1azdmrfghijuklepbnqc

556 _logger.warning("No data loaded: could not find raw encoderPositions file") 1 ctx1i

557 return None 1 ctx1i

558 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 18 ctx1azdmrfghjuklepbnqc

559 return _load_encoder_positions_file_ge5(path) 16 ctx1zdmrfgjuklepbnqc

560 else:

561 return _load_encoder_positions_file_lt5(path) 6 ctx1azdhbc

562

563

564def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'): (empty)

565 """

566 Load Rotary Encoder trial info from raw data file.

567

568 Assumes that a folder calles "raw_behavior_data" exists in folder.

569

570 NOTE: Last trial probably inexistent data (Trial info is sent on trial start

571 and data is only saved on trial exit...) max(trialnum) should be N+1 if N

572 is the amount of trial data saved.

573

574 Raw datafile Columns:

575

576 >>> data.columns

577 >>> ['trial_num', # Trial Number 'numpy.int64'

578 'stim_pos_init', # Initial position of visual stim 'numpy.int64'

579 'stim_contrast', # Contrast of visual stimulus 'numpy.float64'

580 'stim_freq', # Frequency of gabor patch 'numpy.float64'

581 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64'

582 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64'

583 'stim_sigma', # Size of patch 'numpy.float64'

584 'stim_phase', # Phase of gabor 'numpy.float64'

585 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp'

586 # pd.to_datetime(data.bns_ts) to work in datetimes

587

588 :param session_path: Absoulte path of session folder

589 :type session_path: str

590 :return: dataframe w/ 9 cols and ntrials lines

591 :rtype: Pandas.DataFrame

592 """

593 if session_path is None: 1 ctx1`

594 return 1 ctx1`

595 path = Path(session_path).joinpath(task_collection) 1 ctx1`

596 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 1 ctx1`

597 if not path: 1 ctx1`

598 return None 1 ctx1`

599 data = pd.read_csv(path, sep=' ', header=None) 1 ctx1`

600 data = data.drop([9], axis=1) 1 ctx1`

601 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 1 ctx1`

602 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts']

603 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path)

604 return data 1 ctx1`

605

606

607def load_ambient_sensor(session_path, task_collection='raw_behavior_data'): (empty)

608 """

609 Load Ambient Sensor data from session.

610

611 Probably could be extracted to DatasetTypes:

612 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb,

613 _ibl_trials.relativeHumidity

614 Returns a list of dicts one dict per trial.

615 dict keys are:

616 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity'])

617

618 :param session_path: Absoulte path of session folder

619 :type session_path: str

620 :return: list of dicts

621 :rtype: list

622 """

623 if session_path is None:

624 return

625 path = Path(session_path).joinpath(task_collection)

626 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None)

627 if not path:

628 return None

629 data = []

630 with open(path, 'r') as f:

631 for line in f:

632 data.append(json.loads(line))

633 return data

634

635

636def load_mic(session_path, task_collection='raw_behavior_data'): (empty)

637 """

638 Load Microphone wav file to np.array of len nSamples

639

640 :param session_path: Absolute path of session folder

641 :type session_path: str

642 :return: An array of values of the sound waveform

643 :rtype: numpy.array

644 """

645 if session_path is None:

646 return

647 path = Path(session_path).joinpath(task_collection)

648 path = next(path.glob("_iblrig_micData.raw*.wav"), None)

649 if not path:

650 return None

651 fp = wave.open(path)

652 nchan = fp.getnchannels()

653 N = fp.getnframes()

654 dstr = fp.readframes(N * nchan)

655 data = np.frombuffer(dstr, np.int16)

656 data = np.reshape(data, (-1, nchan))

657 return data

658

659

660def _clean_wheel_dataframe(data, label, path): (empty)

661 if np.any(data.isna()): 23 ctx1a8)0zd76mrfghjuklepbnqc

662 _logger.warning(label + ' has missing/incomplete records \n %s', path) 8 ctx1a)zd76bc

663 # first step is to re-interpret as numeric objects if not already done

664 for col in data.columns: 23 ctx1a8)0zd76mrfghjuklepbnqc

665 if data[col].dtype == object and col not in ['bns_ts']: 23 ctx1a8)0zd76mrfghjuklepbnqc

666 data[col] = pd.to_numeric(data[col], errors='coerce') 2 ctx17c

667 # then drop Nans and duplicates

668 data.dropna(inplace=True) 23 ctx1a8)0zd76mrfghjuklepbnqc

669 data.drop_duplicates(keep='first', inplace=True) 23 ctx1a8)0zd76mrfghjuklepbnqc

670 data.reset_index(inplace=True) 23 ctx1a8)0zd76mrfghjuklepbnqc

671 # handle the clock resets when microseconds exceed uint32 max value

672 drop_first = False 23 ctx1a8)0zd76mrfghjuklepbnqc

673 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 23 ctx1a8)0zd76mrfghjuklepbnqc

674 if any(np.diff(data['re_ts']) < 0): 23 ctx1a8)0zd76mrfghjuklepbnqc

675 ind = np.where(np.diff(data['re_ts']) < 0)[0] 16 ctx18)zd76mfghjklebc

676 for i in ind: 16 ctx18)zd76mfghjklebc

677 # the first sample may be corrupt, in this case throw away

678 if i <= 1: 16 ctx18)zd76mfghjklebc

679 drop_first = i 6 ctx18zd6hb

680 _logger.warning(label + ' rotary encoder positions timestamps' 6 ctx18zd6hb

681 ' first sample corrupt ' + str(path))

682 # if it's an uint32 wraparound, the diff should be close to 2 ** 32

683 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 14 ctx18)zd76mfgjklec

684 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 11 ctx1)zd6mfgjklc

685 # there is also the case where 2 positions are swapped and need to be swapped back

686

687 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 9 ctx18zd76fgec

688 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 8 ctx18zd6fgec

689 str(i) + ' ' + str(path))

690 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 8 ctx18zd6fgec

691 data.iloc[i], data.iloc[i + 1] = b, a 8 ctx18zd6fgec

692 # if none of those 3 cases apply, raise an error

693 else:

694 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1 ctx17

695 data.sort_values('re_ts', inplace=True) 1 ctx17

696 data.reset_index(inplace=True) 1 ctx17

697 if drop_first is not False: 23 ctx1a8)0zd76mrfghjuklepbnqc

698 data.drop(data.loc[:drop_first].index, inplace=True) 6 ctx18zd6hb

699 data = data.reindex() 6 ctx18zd6hb

700 return data 23 ctx1a8)0zd76mrfghjuklepbnqc

701

702

703def _groom_wheel_data_lt5(data, label='file ', path=''): (empty)

704 """

705 The whole purpose of this function is to account for variability and corruption in

706 the wheel position files. There are many possible errors described below, but

707 nothing excludes getting new ones.

708 """

709 data = _clean_wheel_dataframe(data, label, path) 11 ctx1a8)0zd76hbc

710 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 11 ctx1a8)0zd76hbc

711 # check if the time scale is in ms

712 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 11 ctx1a8)0zd76hbc

713 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds

714 if data['re_ts'].iloc[-1] / (sess_len_sec + 1e-6) < 1e5: # should be 1e6 normally 11 ctx1a8)0zd76hbc

715 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx10d7hb

716 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')

717 data['re_ts'] = data['re_ts'] * 1000 5 ctx10d7hb

718 return data 11 ctx1a8)0zd76hbc

719

720

721def _groom_wheel_data_ge5(data, label='file ', path=''): (empty)

722 """

723 The whole purpose of this function is to account for variability and corruption in

724 the wheel position files. There are many possible errors described below, but

725 nothing excludes getting new ones.

726 """

727 data = _clean_wheel_dataframe(data, label, path) 20 ctx180zd76mrfgjuklepbnqc

728 # check if the time scale is in ms

729 if (data['re_ts'].iloc[-1] - data['re_ts'].iloc[0]) / 1e6 < 20: 20 ctx180zd76mrfgjuklepbnqc

730 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx176bnc

731 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')

732 data['re_ts'] = data['re_ts'] * 1000 5 ctx176bnc

733 return data 20 ctx180zd76mrfgjuklepbnqc

734

735

736def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5, (empty)

737 return_index=False):

738 """

739 Attempts to find matching timestamps in 2 time-series that have an offset, are drifting,

740 and are most likely incomplete: sizes don't have to match, some pulses may be missing

741 in any series.

742 Only works with irregular time series as it relies on the derivative to match sync.

743 :param t0:

744 :param t1:

745 :param diff_threshold:

746 :param drift_threshold_ppm: (150)

747 :param max_shift: (200)

748 :param return_index (False)

749 :return:

750 """

751 nsync = min(t0.size, t1.size) 3 ctx1;bc

752 dt0 = np.diff(t0) 3 ctx1;bc

753 dt1 = np.diff(t1) 3 ctx1;bc

754 ind = np.zeros_like(dt0) * np.nan 3 ctx1;bc

755 i0 = 0 3 ctx1;bc

756 i1 = 0 3 ctx1;bc

757 cdt = np.nan # the current time difference between the two series to compute drift 3 ctx1;bc

758 while i0 < (nsync - 1): 3 ctx1;bc

759 # look in the next max_shift events the ones whose derivative match

760 isearch = np.arange(i1, min(max_shift + i1, dt1.size)) 3 ctx1;bc

761 dec = np.abs(dt0[i0] - dt1[isearch]) < diff_threshold 3 ctx1;bc

762 # another constraint is to check the dt for the maximum drift

763 if ~np.isnan(cdt): 3 ctx1;bc

764 drift_ppm = np.abs((cdt - (t0[i0] - t1[isearch])) / dt1[isearch]) * 1e6 3 ctx1;bc

765 dec = np.logical_and(dec, drift_ppm <= drift_threshold_ppm) 3 ctx1;bc

766 # if one is found

767 if np.any(dec): 3 ctx1;bc

768 ii1 = np.where(dec)[0][0] 3 ctx1;bc

769 ind[i0] = i1 + ii1 3 ctx1;bc

770 i1 += ii1 + 1 3 ctx1;bc

771 cdt = t0[i0 + 1] - t1[i1 + ii1] 3 ctx1;bc

772 i0 += 1 3 ctx1;bc

773 it0 = np.where(~np.isnan(ind))[0] 3 ctx1;bc

774 it1 = ind[it0].astype(int) 3 ctx1;bc

775 ind0 = np.unique(np.r_[it0, it0 + 1]) 3 ctx1;bc

776 ind1 = np.unique(np.r_[it1, it1 + 1]) 3 ctx1;bc

777 if return_index: 3 ctx1;bc

778 return t0[ind0], t1[ind1], ind0, ind1

779 else:

780 return t0[ind0], t1[ind1] 3 ctx1;bc

781

782

783def load_bpod_fronts(session_path: str, data: list = False, task_collection: str = 'raw_behavior_data') -> list: (empty)

784 """load_bpod_fronts

785 Loads BNC1 and BNC2 bpod channels times and polarities from session_path

786

787 :param session_path: a valid session_path

788 :type session_path: str

789 :param data: pre-loaded raw data dict, defaults to False

790 :type data: list, optional

791 :return: List of dicts BNC1 and BNC2 {"times": np.array, "polarities":np.array}

792 :rtype: list

793 """

794 if not data: 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

795 data = load_data(session_path, task_collection) 2 ctx1ah

796

797 BNC1_fronts = np.array([[np.nan, np.nan]]) 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

798 BNC2_fronts = np.array([[np.nan, np.nan]]) 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

799 for tr in data: 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

800 BNC1_fronts = np.append( 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

801 BNC1_fronts,

802 np.array(

803 [

804 [x, 1]

805 for x in tr["behavior_data"]["Events timestamps"].get("BNC1High", [np.nan])

806 ]

807 ),

808 axis=0,

809 )

810 BNC1_fronts = np.append( 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

811 BNC1_fronts,

812 np.array(

813 [

814 [x, -1]

815 for x in tr["behavior_data"]["Events timestamps"].get("BNC1Low", [np.nan])

816 ]

817 ),

818 axis=0,

819 )

820 BNC2_fronts = np.append( 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

821 BNC2_fronts,

822 np.array(

823 [

824 [x, 1]

825 for x in tr["behavior_data"]["Events timestamps"].get("BNC2High", [np.nan])

826 ]

827 ),

828 axis=0,

829 )

830 BNC2_fronts = np.append( 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

831 BNC2_fronts,

832 np.array(

833 [

834 [x, -1]

835 for x in tr["behavior_data"]["Events timestamps"].get("BNC2Low", [np.nan])

836 ]

837 ),

838 axis=0,

839 )

840

841 BNC1_fronts = BNC1_fronts[1:, :] 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

842 BNC1_fronts = BNC1_fronts[BNC1_fronts[:, 0].argsort()] 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

843 BNC2_fronts = BNC2_fronts[1:, :] 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

844 BNC2_fronts = BNC2_fronts[BNC2_fronts[:, 0].argsort()] 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

845

846 BNC1 = {"times": BNC1_fronts[:, 0], "polarities": BNC1_fronts[:, 1]} 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

847 BNC2 = {"times": BNC2_fronts[:, 0], "polarities": BNC2_fronts[:, 1]} 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

848

849 return [BNC1, BNC2] 49 ctx1aKLMNOBPQRSTUIVWXYdCDmGrfghHAiywsvZtxJojuklepEbnq

850

851

852def get_port_events(trial: dict, name: str = '') -> list: (empty)

853 """get_port_events

854 Return all event timestamps from bpod raw data trial that match 'name'

855 --> looks in trial['behavior_data']['Events timestamps']

856

857 :param trial: raw trial dict

858 :type trial: dict

859 :param name: name of event, defaults to ''

860 :type name: str, optional

861 :return: Sorted list of event timestamps

862 :rtype: list

863 TODO: add polarities?

864 """

865 out: list = [] 23 ctx1aBdCDmGrfghHAjuklepEbnq

866 events = trial['behavior_data']['Events timestamps'] 23 ctx1aBdCDmGrfghHAjuklepEbnq

867 for k in events: 23 ctx1aBdCDmGrfghHAjuklepEbnq

868 if name in k: 23 ctx1aBdCDmGrfghHAjuklepEbnq

869 out.extend(events[k]) 22 ctx1BdCDmGrfghHAjuklepEbnq

870 out = sorted(out) 23 ctx1aBdCDmGrfghHAjuklepEbnq

871

872 return out 23 ctx1aBdCDmGrfghHAjuklepEbnq

873

874

875def load_widefield_mmap(session_path, dtype=np.uint16, shape=(540, 640), n_frames=None, mode='r'): (empty)

876 """

877 TODO Document this function

878

879 Parameters

880 ----------

881 session_path

882

883 Returns

884 -------

885

886 """

887 filepath = Path(session_path).joinpath('raw_widefield_data').glob('widefield.raw.*.dat')

888 filepath = next(filepath, None)

889 if not filepath:

890 _logger.warning("No data loaded: could not find raw data file")

891 return None

892

893 if type(dtype) is str:

894 dtype = np.dtype(dtype)

895

896 if n_frames is None:

897 # Get the number of samples from the file size

898 n_frames = int(filepath.stat().st_size / (np.prod(shape) * dtype.itemsize))

899

900 return np.memmap(str(filepath), mode=mode, dtype=dtype, shape=(int(n_frames), *shape))

901

902

903def patch_settings(session_path, collection='raw_behavior_data', (empty)

904 new_collection=None, subject=None, number=None, date=None):

905 """Modify various details in a settings file.

906

907 This function makes it easier to change things like subject name in a settings as it will

908 modify the subject name in the myriad paths. NB: This saves the settings into the same location

909 it was loaded from.

910

911 Parameters

912 ----------

913 session_path : str, pathlib.Path

914 The session path containing the settings file.

915 collection : str

916 The subfolder containing the settings file.

917 new_collection : str

918 An optional new subfolder to change in the settings paths.

919 subject : str

920 An optional new subject name to change in the settings.

921 number : str, int

922 An optional new number to change in the settings.

923 date : str, datetime.date

924 An optional date to change in the settings.

925

926 Returns

927 -------

928 dict

929 The modified settings.

930

931 Examples

932 --------

933 File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location.

934 >>> patch_settings('/data/subject/2020-01-01/002', number='001')

935 >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/')

936

937 File is moved into new collection within the same session, then patched.

938 >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/')

939 >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01')

940

941 Update subject, date and number.

942 >>> new_session_path = Path('/data/foobar/2024-02-24/002')

943 >>> old_session_path = Path('/data/baz/2024-02-23/001')

944 >>> patch_settings(old_session_path, collection='raw_task_data_00',

945 ... subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1])

946 >>> shutil.move(old_session_path, new_session_path)

947 """

948 settings = load_settings(session_path, collection) 2 ctx1aF

949 if not settings: 2 ctx1aF

950 raise IOError('Settings file not found') 1 ctx1F

951

952 filename = PureWindowsPath(settings.get('SETTINGS_FILE_PATH', '_iblrig_taskSettings.raw.json')).name 2 ctx1aF

953 file_path = Path(session_path).joinpath(collection, filename) 2 ctx1aF

954

955 if subject: 2 ctx1aF

956 # Patch subject name

957 old_subject = settings['SUBJECT_NAME'] 2 ctx1aF

958 settings['SUBJECT_NAME'] = subject 2 ctx1aF

959 for k in settings.keys(): 2 ctx1aF

960 if isinstance(settings[k], str): 2 ctx1aF

961 settings[k] = settings[k].replace(f'\\Subjects\\{old_subject}', f'\\Subjects\\{subject}') 2 ctx1aF

962 if 'SESSION_NAME' in settings: 2 ctx1aF

963 settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) 2 ctx1aF

964 settings.pop('PYBPOD_SUBJECT_EXTRA', None) # Get rid of Alyx subject info 2 ctx1aF

965

966 if date: 2 ctx1aF

967 # Patch session datetime

968 date = str(date) 2 ctx1aF

969 old_date = settings['SESSION_DATE'] 2 ctx1aF

970 settings['SESSION_DATE'] = date 2 ctx1aF

971 for k in settings.keys(): 2 ctx1aF

972 if isinstance(settings[k], str): 2 ctx1aF

973 settings[k] = settings[k].replace( 2 ctx1aF

974 f'\\{settings["SUBJECT_NAME"]}\\{old_date}',

975 f'\\{settings["SUBJECT_NAME"]}\\{date}'

976 )

977 settings['SESSION_DATETIME'] = date + settings['SESSION_DATETIME'][10:] 2 ctx1aF

978 if 'SESSION_END_TIME' in settings: 2 ctx1aF

979 settings['SESSION_END_TIME'] = date + settings['SESSION_END_TIME'][10:] 1 ctx1F

980 if 'SESSION_START_TIME' in settings: 2 ctx1aF

981 settings['SESSION_START_TIME'] = date + settings['SESSION_START_TIME'][10:] 1 ctx1F

982

983 if number: 2 ctx1aF

984 # Patch session number

985 old_number = settings['SESSION_NUMBER'] 2 ctx1aF

986 if isinstance(number, int): 2 ctx1aF

987 number = f'{number:03}' 1 ctx1F

988 settings['SESSION_NUMBER'] = number 2 ctx1aF

989 for k in settings.keys(): 2 ctx1aF

990 if isinstance(settings[k], str): 2 ctx1aF

991 settings[k] = settings[k].replace( 2 ctx1aF

992 f'\\{settings["SESSION_DATE"]}\\{old_number}',

993 f'\\{settings["SESSION_DATE"]}\\{number}'

994 )

995

996 if new_collection: 2 ctx1aF

997 if 'SESSION_RAW_DATA_FOLDER' not in settings: 1 ctx1F

998 _logger.warning('SESSION_RAW_DATA_FOLDER key not in settings; collection not updated') 1 ctx1F

999 else:

1000 old_path = settings['SESSION_RAW_DATA_FOLDER'] 1 ctx1F

1001 new_path = PureWindowsPath(settings['SESSION_RAW_DATA_FOLDER']).with_name(new_collection) 1 ctx1F

1002 for k in settings.keys(): 1 ctx1F

1003 if isinstance(settings[k], str): 1 ctx1F

1004 settings[k] = settings[k].replace(old_path, str(new_path)) 1 ctx1F

1005 with open(file_path, 'w') as fp: 2 ctx1aF

1006 json.dump(settings, fp, indent=' ') 2 ctx1aF

1007 return settings 2 ctx1aF

Coverage for ibllib/io/raw_data_loaders.py: 88%

450 statements