Coverage for ibllib/io/raw_data

1"""

2Raw Data Loader functions for PyBpod rig.

4Module contains one loader function per raw datafile.

5"""

6import re (empty)

7import json (empty)

8import logging (empty)

9import wave (empty)

10from collections import OrderedDict (empty)

11from datetime import datetime (empty)

12from pathlib import Path, PureWindowsPath (empty)

13from typing import Union (empty)

15from dateutil import parser as dateparser (empty)

16from packaging import version (empty)

17import numpy as np (empty)

18import pandas as pd (empty)

20from iblutil.io import jsonable (empty)

21from ibllib.io.video import assert_valid_label (empty)

22from ibllib.time import uncycle_pgts, convert_pgts, date2isostr (empty)

24_logger = logging.getLogger(__name__) (empty)

27def trial_times_to_times(raw_trial): (empty)

28 """

29 Parse and convert all trial timestamps to "absolute" time.

30 Float64 seconds from session start.

32 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1---

33 tsN...absTS = tsN + TrialStartN - BpodStart

35 Bpod timestamps are in microseconds (µs)

36 PyBpod timestamps are is seconds (s)

38 :param raw_trial: raw trial data

39 :type raw_trial: dict

40 :return: trial data with modified timestamps

41 :rtype: dict

42 """

43 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

44 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

45 # ts_te = raw_trial['behavior_data']['Trial end timestamp']

47 def convert(ts): 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

48 return ts + ts_ts - ts_bs 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

50 converted_events = {} 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

51 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

52 converted_events.update({k: [convert(i) for i in v]}) 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

53 raw_trial['behavior_data']['Events timestamps'] = converted_events 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

55 converted_states = {} 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

56 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

57 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

58 raw_trial['behavior_data']['States timestamps'] = converted_states 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

60 shift = raw_trial['behavior_data']['Bpod start timestamp'] 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

61 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

62 raw_trial['behavior_data']['Trial start timestamp'] -= shift 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

63 raw_trial['behavior_data']['Trial end timestamp'] -= shift 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

64 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

65 return raw_trial 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

68def load_bpod(session_path, task_collection='raw_behavior_data'): (empty)

69 """

70 Load both settings and data from bpod (.json and .jsonable)

72 :param session_path: Absolute path of session folder

73 :param task_collection: Collection within sesison path with behavior data

74 :return: dict settings and list of dicts data

75 """

76 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 7 ctx1a!#$c32

79def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'): (empty)

80 """

81 Load PyBpod data files (.jsonable).

83 Bpod timestamps are in microseconds (µs)

84 PyBpod timestamps are is seconds (s)

86 :param session_path: Absolute path of session folder

87 :type session_path: str, Path

88 :return: A list of len ntrials each trial being a dictionary

89 :rtype: list of dicts

90 """

91 if session_path is None: 56 ctx1aLMNOPCQRSTUVJWXYZeD!#$EmHrghcIBxizwsv0tyKojuklfp3F2bnqd

92 _logger.warning('No data loaded: session_path is None')

93 return

94 path = Path(session_path).joinpath(task_collection) 56 ctx1aLMNOPCQRSTUVJWXYZeD!#$EmHrghcIBxizwsv0tyKojuklfp3F2bnqd

95 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 56 ctx1aLMNOPCQRSTUVJWXYZeD!#$EmHrghcIBxizwsv0tyKojuklfp3F2bnqd

96 if not path: 56 ctx1aLMNOPCQRSTUVJWXYZeD!#$EmHrghcIBxizwsv0tyKojuklfp3F2bnqd

97 _logger.warning('No data loaded: could not find raw data file') 3 ctx1!#$

98 return None 3 ctx1!#$

99 data = jsonable.read(path) 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

100 if time == 'absolute': 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

101 data = [trial_times_to_times(t) for t in data] 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

102 return data 53 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfp3F2bnqd

103

104

105def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: (empty)

106 """Loads binary frame data from Bonsai camera recording workflow.

107

108 Args:

109 session_path (StrPath): Path to session folder

110 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'.

111 raw (bool, optional): Whether to return raw or parsed data. Defaults to False.

112

113 Returns:

114 parsed: (raw=False, Default)

115 pandas.DataFrame: 4 int64 columns: {

116 Timestamp, # float64 (seconds from session start)

117 embeddedTimeStamp, # float64 (seconds from session start)

118 embeddedFrameCounter, # int64 (Frame number from session start)

119 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a

120 # list of numpy boolean arrays

121 # e.g. np.array([True, False, False, False])

122 }

123 raw:

124 pandas.DataFrame: 4 int64 columns: {

125 Timestamp, # UTC ticks from BehaviorPC

126 # (100's of ns since midnight 1/1/0001)

127 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion)

128 embeddedFrameCounter, # Frame counter (int)

129 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins

130 }

131 """

132 camera = assert_valid_label(camera) 5 ctx1a[6vo

133 fpath = Path(session_path).joinpath("raw_video_data") 5 ctx1a[6vo

134 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 5 ctx1a[6vo

135 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 5 ctx1a[6vo

136 rdata = np.fromfile(fpath, dtype=np.float64) 5 ctx1a[6vo

137 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 5 ctx1a[6vo

138 rows = int(rdata.size / 4) 5 ctx1a[6vo

139 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 5 ctx1a[6vo

140 df_dict = dict.fromkeys( 5 ctx1a[6vo

141 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"]

142 )

143 df = pd.DataFrame(data, columns=df_dict.keys()) 5 ctx1a[6vo

144 if raw: 5 ctx1a[6vo

145 return df 1 ctx1[

146

147 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 5 ctx1a[6vo

148 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 5 ctx1a[6vo

149 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 5 ctx1a[6vo

150 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 5 ctx1a[6vo

151 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 5 ctx1a[6vo

152 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 5 ctx1a[6vo

153

154 parsed_df = pd.DataFrame.from_dict(df_dict) 5 ctx1a[6vo

155 return parsed_df 5 ctx1a[6vo

156

157

158def load_camera_ssv_times(session_path, camera: str): (empty)

159 """

160 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv

161

162 NB: For some sessions the frame times are in the first column, in others the order is reversed.

163 NB: If using the new bin file the bonsai_times is a float in seconds since first frame

164 :param session_path: Absolute path of session folder

165 :param camera: Name of the camera to load, e.g. 'left'

166 :return: array of datetimes, array of frame times in seconds

167 """

168 camera = assert_valid_label(camera) 14 ctx1^]c4izws_5vtyo

169 video_path = Path(session_path).joinpath('raw_video_data') 14 ctx1^]c4izws_5vtyo

170 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 14 ctx1^]c4izws_5vtyo

171 df = load_camera_frameData(session_path, camera=camera) 2 ctx1vo

172 return df['Timestamp'].values, df['embeddedTimeStamp'].values 2 ctx1vo

173

174 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 13 ctx1^]c4izws_5vty

175 if not file: 13 ctx1^]c4izws_5vty

176 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 1 ctx1]

177 raise FileNotFoundError(file + ' not found') 1 ctx1]

178 # NB: Numpy has deprecated support for non-naive timestamps.

179 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s.

180 # from datetime import timezone

181 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)}

182

183 # Determine the order of the columns by reading one line and testing whether the first value

184 # is an integer or not.

185 with open(file, 'r') as f: 13 ctx1^]c4izws_5vty

186 line = f.readline() 13 ctx1^]c4izws_5vty

187 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 13 ctx1^]c4izws_5vty

188 try: 13 ctx1^]c4izws_5vty

189 int(line.split(' ')[1]) 13 ctx1^]c4izws_5vty

190 except ValueError: 7 ctx1]cizsty

191 type_map.move_to_end('bonsai') 7 ctx1]cizsty

192 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 13 ctx1^]c4izws_5vty

193 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 13 ctx1^]c4izws_5vty

194 bonsai_times = ssv_times['bonsai'] 13 ctx1^]c4izws_5vty

195 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 13 ctx1^]c4izws_5vty

196 return bonsai_times, camera_times 13 ctx1^]c4izws_5vty

197

198

199def load_embedded_frame_data(session_path, label: str, raw=False): (empty)

200 """

201 Load the embedded frame count and GPIO for a given session. If the file doesn't exist,

202 or is empty, None values are returned.

203

204 :param session_path: Absolute path of session folder

205 :param label: The specific video to load, one of ('left', 'right', 'body')

206 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is

207 returned starting from 0 and the GPIO is returned as a dict of indices

208 :return: The frame count, GPIO

209 """

210 count = load_camera_frame_count(session_path, label, raw=raw) 14 ctx1'cx@4izws5vtyo

211 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 14 ctx1'cx@4izws5vtyo

212 return count, gpio 14 ctx1'cx@4izws5vtyo

213

214

215def load_camera_frame_count(session_path, label: str, raw=True): (empty)

216 """

217 Load the embedded frame count for a given session. If the file doesn't exist, or is empty,

218 a None value is returned.

219 :param session_path: Absolute path of session folder

220 :param label: The specific video to load, one of ('left', 'right', 'body')

221 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is

222 returned starting from 0

223 :return: The frame count

224 """

225 if session_path is None: 15 ctx1`'cx@4izws5vtyo

226 return 1 ctx1`

227

228 label = assert_valid_label(label) 15 ctx1`'cx@4izws5vtyo

229 video_path = Path(session_path).joinpath('raw_video_data') 15 ctx1`'cx@4izws5vtyo

230 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 15 ctx1`'cx@4izws5vtyo

231 df = load_camera_frameData(session_path, camera=label) 2 ctx1vo

232 return df['embeddedFrameCounter'].values 2 ctx1vo

233

234 # Load frame count

235 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 14 ctx1`'cx@4izws5vty

236 count_file = next(glob, None) 14 ctx1`'cx@4izws5vty

237 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 14 ctx1`'cx@4izws5vty

238 if len(count) == 0: 14 ctx1`'cx@4izws5vty

239 return 5 ctx1`cx@z

240 if not raw: 10 ctx1`'4iws5vty

241 count -= count[0] # start from zero 9 ctx1`'4is5vty

242 return count 10 ctx1`'4iws5vty

243

244

245def load_camera_gpio(session_path, label: str, as_dicts=False): (empty)

246 """

247 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is

248 returned.

249

250 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits

251 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by

252 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state).

253

254 :param session_path: Absolute path of session folder

255 :param label: The specific video to load, one of ('left', 'right', 'body')

256 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins)

257 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities').

258 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4.

259 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'),

260 or None if the dictionary is empty.

261 """

262 if session_path is None: 15 ctx16'cx@4izws5vtyo

263 return 1 ctx16

264 raw_path = Path(session_path).joinpath('raw_video_data') 15 ctx16'cx@4izws5vtyo

265 label = assert_valid_label(label) 15 ctx16'cx@4izws5vtyo

266

267 # Load pin state

268 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 15 ctx16'cx@4izws5vtyo

269 df = load_camera_frameData(session_path, camera=label, raw=False) 3 ctx16vo

270 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 3 ctx16vo

271 if len(gpio) == 0: 3 ctx16vo

272 return [None] * 4 if as_dicts else None

273 else:

274 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 14 ctx16'cx@4izws5vty

275 # This deals with missing and empty files the same

276 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 14 ctx16'cx@4izws5vty

277 # Check values make sense (4 pins = 16 possible values)

278 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 14 ctx16'cx@4izws5vty

279 _logger.warning('Unexpected GPIO values; decoding may fail') 1 ctx16

280 if len(gpio) == 0: 14 ctx16'cx@4izws5vty

281 return [None] * 4 if as_dicts else None 5 ctx16cx@z

282 # 4 pins represented as uint32

283 # For each pin, shift its bit to the end and check the bit is set

284 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 10 ctx16'4iws5vty

285

286 if as_dicts: 11 ctx16'4iws5vtyo

287 if not gpio.any(): 10 ctx16'4is5vtyo

288 _logger.error('No GPIO changes') 1 ctx16

289 return [None] * 4 1 ctx16

290 # Find state changes for each pin and construct a dict of indices and polarities for each

291 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 10 ctx16'4is5vtyo

292 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)]

293 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8

294 gpio = [{'indices': np.where(edges[:, i])[0], 10 ctx16'4is5vtyo

295 'polarities': edges[edges[:, i] != 0, i]}

296 for i in range(4)]

297 # Replace empty dicts with None

298 gpio = [None if x['indices'].size == 0 else x for x in gpio] 10 ctx16'4is5vtyo

299

300 return gpio 11 ctx16'4iws5vtyo

301

302

303def _read_settings_json_compatibility_enforced(settings): (empty)

304 """

305 Patch iblrig settings for compatibility across rig versions.

306

307 Parameters

308 ----------

309 settings : pathlib.Path, dict

310 Either a _iblrig_taskSettings.raw.json file path or the loaded settings.

311

312 Returns

313 -------

314 dict

315 The task settings patched for compatibility.

316 """

317 if isinstance(settings, dict): 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

318 md = settings.copy()

319 else:

320 with open(settings) as js: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

321 md = json.load(js) 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

322 if 'IS_MOCK' not in md: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

323 md['IS_MOCK'] = False 58 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+,)%!#$EmHghIBxis0tKojklfpG3F2-bnqd

324 # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag.

325 # Some sessions have neither key. From v8 onwards we will use IBLRIG_VERSION to test rig

326 # version, however some places may still use the version tag.

327 if 'IBLRIG_VERSION_TAG' not in md.keys(): 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

328 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 10 ctx1)!#$IBKofp

329 if 'IBLRIG_VERSION' not in md.keys(): 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

330 md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG'] 57 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,%EmH.rghIxiws0t/:;juklG3F2-bnqd

331 elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]): 11 ctx1)!#$cBKofpG

332 # This may not be an issue; not sure what the intended difference between these keys was

333 assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch' 11 ctx1)!#$cBKofpG

334 # Test version can be parsed. If not, log an error and set the version to nothing

335 try: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

336 version.parse(md['IBLRIG_VERSION'] or '0') 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

337 except version.InvalidVersion as ex:

338 _logger.error('%s in iblrig settings, this may affect extraction', ex)

339 # try a more relaxed version parse

340 laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION'])

341 # Set the tag as the invalid version

342 md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION']

343 # overwrite version with either successfully parsed one or an empty string

344 md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else ''

345 if 'device_sound' not in md: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

346 # sound device must be defined in version 8 and later # FIXME this assertion will cause tests to break

347 assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0') 60 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghxiws0t/:;juklG3F2-bnqd

348 # in v7 we must infer the device from the sampling frequency if SD is None

349 if 'sounddevice' in md.get('SD', ''): 60 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghxiws0t/:;juklG3F2-bnqd

350 device = 'xonar' 15 ctx1amHghxis0tG32bd

351 else:

352 freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'} 49 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$E.rw/:;juklF2-bnqd

353 device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown') 49 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$E.rw/:;juklF2-bnqd

354 md['device_sound'] = {'OUTPUT': device} 60 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghxiws0t/:;juklG3F2-bnqd

355 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on)

356 if md['IBLRIG_VERSION'] == '': 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

357 pass 5 ctx1J?Ijd

358 elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'): 65 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+,)%!#$EmH.rghcBxiws0tKo/:;juklfpG3F2-bnqd

359 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 5 ctx1BKofp

360 md['PYBPOD_BOARD'] = md['RIG_NAME'] 5 ctx1BKofp

361 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 5 ctx1BKofp

362 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 5 ctx1BKofp

363 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 5 ctx1BKofp

364 elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'): 60 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+,)%!#$EmH.rghcxiws0t/:;juklG3F2-bnqd

365 if 'LAST_TRIAL_DATA' in md.keys(): 1 ctx1b

366 md.pop('LAST_TRIAL_DATA') 1 ctx1b

367 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

368 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1 ctx1b

369 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1b

370 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1 ctx1b

371 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1 ctx1b

372 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1 ctx1b

373 # parse the date format to Django supported ISO

374 dt = dateparser.parse(md['SESSION_DATETIME']) 1 ctx1b

375 md['SESSION_DATETIME'] = date2isostr(dt) 1 ctx1b

376 # add the weight key if it doesn't already exist

377 if 'SUBJECT_WEIGHT' not in md: 1 ctx1b

378 md['SUBJECT_WEIGHT'] = None 1 ctx1b

379 return md 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

380

381

382def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'): (empty)

383 """

384 Load PyBpod Settings files (.json).

385

386 [description]

387

388 :param session_path: Absolute path of session folder

389 :type session_path: str, Path

390 :return: Settings dictionary

391 :rtype: dict

392 """

393 if session_path is None: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

394 _logger.warning("No data loaded: session_path is None") 1 ctx1%

395 return 1 ctx1%

396 path = Path(session_path).joinpath(task_collection) 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

397 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

398 if not path: 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

399 _logger.warning("No data loaded: could not find raw settings file") 4 ctx1(%!G

400 return None 4 ctx1(%!G

401 settings = _read_settings_json_compatibility_enforced(path) 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

402 return settings 67 ctx1aLMNOPCQRSTUVJWXYZ1AeD(+?,)%!#$EmH.rghcIBxiws0tKo/:;juklfpG3F2-bnqd

403

404

405def load_stim_position_screen(session_path, task_collection='raw_behavior_data'): (empty)

406 path = Path(session_path).joinpath(task_collection)

407 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None)

408

409 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip')

410 data.columns = ['contrast', 'position', 'bns_ts']

411 data['bns_ts'] = pd.to_datetime(data['bns_ts'])

412 return data

413

414

415def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False): (empty)

416 """

417 Load Rotary Encoder (RE) events raw data file.

418

419 Assumes that a folder called "raw_behavior_data" exists in folder.

420

421 Events number correspond to following bpod states:

422 1: correct / hide_stim

423 2: stim_on

424 3: closed_loop

425 4: freeze_error / freeze_correct

426

427 >>> data.columns

428 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'

429 'sm_ev', # State Machine Event 'numpy.int64'

430 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp'

431 # pd.to_datetime(data.bns_ts) to work in datetimes

432

433 :param session_path: [description]

434 :type session_path: [type]

435 :return: dataframe w/ 3 cols and (ntrials * 3) lines

436 :rtype: Pandas.DataFrame

437 """

438 if session_path is None: 18 ctx1a1emrghcjuklfpbnqd

439 return

440 path = Path(session_path).joinpath(task_collection) 18 ctx1a1emrghcjuklfpbnqd

441 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 18 ctx1a1emrghcjuklfpbnqd

442 if not settings: 18 ctx1a1emrghcjuklfpbnqd

443 settings = load_settings(session_path, task_collection=task_collection) 18 ctx1a1emrghcjuklfpbnqd

444 if settings is None or not settings.get('IBLRIG_VERSION'): 18 ctx1a1emrghcjuklfpbnqd

445 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1d

446 # auto-detect old files when version is not labeled

447 with open(path) as fid: 1 ctx1d

448 line = fid.readline() 1 ctx1d

449 if line.startswith('Event') and 'StateMachine' in line: 1 ctx1d

450 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1d

451 if not path: 18 ctx1a1emrghcjuklfpbnqd

452 return None

453 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 18 ctx1a1emrghcjuklfpbnqd

454 return _load_encoder_events_file_ge5(path) 16 ctx11emrghjuklfpbnqd

455 else:

456 return _load_encoder_events_file_lt5(path) 6 ctx1a1ecbd

457

458

459def _load_encoder_ssv_file(file_path, **kwargs): (empty)

460 file_path = Path(file_path) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

461 if file_path.stat().st_size == 0: 23 ctx1a9*1Ae87mrghcjuklfpbnqd

462 _logger.error(f"{file_path.name} is an empty file. ")

463 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ")

464 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

465

466

467def _load_encoder_positions_file_lt5(file_path): (empty)

468 """

469 File loader without the session overhead

470 :param file_path:

471 :return: dataframe of encoder events

472 """

473 data = _load_encoder_ssv_file(file_path, 9 ctx1a9*Ae7cbd

474 names=['_', 're_ts', 're_pos', 'bns_ts', '__'],

475 usecols=['re_ts', 're_pos', 'bns_ts'])

476 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 9 ctx1a9*Ae7cbd

477

478

479def _load_encoder_positions_file_ge5(file_path): (empty)

480 """

481 File loader without the session overhead

482 :param file_path:

483 :return: dataframe of encoder events

484 """

485 data = _load_encoder_ssv_file(file_path, 18 ctx19Ae7mrghjuklfpbnqd

486 names=['re_ts', 're_pos', '_'],

487 usecols=['re_ts', 're_pos'])

488 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 18 ctx19Ae7mrghjuklfpbnqd

489

490

491def _load_encoder_events_file_lt5(file_path): (empty)

492 """

493 File loader without the session overhead

494 :param file_path:

495 :return: dataframe of encoder events

496 """

497 data = _load_encoder_ssv_file(file_path, 7 ctx1a1e8cbd

498 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'],

499 usecols=['re_ts', 'sm_ev', 'bns_ts'])

500 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 7 ctx1a1e8cbd

501

502

503def _load_encoder_events_file_ge5(file_path): (empty)

504 """

505 File loader without the session overhead

506 :param file_path:

507 :return: dataframe of encoder events

508 """

509 data = _load_encoder_ssv_file(file_path, 17 ctx11e8mrghjuklfpbnqd

510 names=['re_ts', 'sm_ev', '_'],

511 usecols=['re_ts', 'sm_ev'])

512 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 17 ctx11e8mrghjuklfpbnqd

513

514

515def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False): (empty)

516 """

517 Load Rotary Encoder (RE) positions from raw data file within a session path.

518

519 Assumes that a folder called "raw_behavior_data" exists in folder.

520 Positions are RE ticks [-512, 512] == [-180º, 180º]

521 0 == trial stim init position

522 Positive nums are rightwards movements (mouse) or RE CW (mouse)

523

524 Variable line number, depends on movements.

525

526 Raw datafile Columns:

527 Position, RE timestamp, RE Position, Bonsai Timestamp

528

529 Position is always equal to 'Position' so this column was dropped.

530

531 >>> data.columns

532 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'

533 're_pos', # Rotary Encoder position (ticks) 'numpy.int64'

534 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp'

535 # pd.to_datetime(data.bns_ts) to work in datetimes

536

537 :param session_path: Absolute path of session folder

538 :type session_path: str

539 :return: dataframe w/ 3 cols and N positions

540 :rtype: Pandas.DataFrame

541 """

542 if session_path is None: 19 ctx1aAemrghcijuklfpbnqd

543 return

544 path = Path(session_path).joinpath(task_collection) 19 ctx1aAemrghcijuklfpbnqd

545 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 19 ctx1aAemrghcijuklfpbnqd

546 if not settings: 19 ctx1aAemrghcijuklfpbnqd

547 settings = load_settings(session_path, task_collection=task_collection) 19 ctx1aAemrghcijuklfpbnqd

548 if settings is None or not settings.get('IBLRIG_VERSION'): 19 ctx1aAemrghcijuklfpbnqd

549 settings = {'IBLRIG_VERSION': '100.0.0'} 1 ctx1d

550 # auto-detect old files when version is not labeled

551 with open(path) as fid: 1 ctx1d

552 line = fid.readline() 1 ctx1d

553 if line.startswith('Position'): 1 ctx1d

554 settings = {'IBLRIG_VERSION': '0.0.0'} 1 ctx1d

555 if not path: 19 ctx1aAemrghcijuklfpbnqd

556 _logger.warning("No data loaded: could not find raw encoderPositions file") 1 ctx1i

557 return None 1 ctx1i

558 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 18 ctx1aAemrghcjuklfpbnqd

559 return _load_encoder_positions_file_ge5(path) 16 ctx1Aemrghjuklfpbnqd

560 else:

561 return _load_encoder_positions_file_lt5(path) 6 ctx1aAecbd

562

563

564def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'): (empty)

565 """

566 Load Rotary Encoder trial info from raw data file.

567

568 Assumes that a folder calles "raw_behavior_data" exists in folder.

569

570 NOTE: Last trial probably inexistent data (Trial info is sent on trial start

571 and data is only saved on trial exit...) max(trialnum) should be N+1 if N

572 is the amount of trial data saved.

573

574 Raw datafile Columns:

575

576 >>> data.columns

577 >>> ['trial_num', # Trial Number 'numpy.int64'

578 'stim_pos_init', # Initial position of visual stim 'numpy.int64'

579 'stim_contrast', # Contrast of visual stimulus 'numpy.float64'

580 'stim_freq', # Frequency of gabor patch 'numpy.float64'

581 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64'

582 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64'

583 'stim_sigma', # Size of patch 'numpy.float64'

584 'stim_phase', # Phase of gabor 'numpy.float64'

585 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp'

586 # pd.to_datetime(data.bns_ts) to work in datetimes

587

588 :param session_path: Absoulte path of session folder

589 :type session_path: str

590 :return: dataframe w/ 9 cols and ntrials lines

591 :rtype: Pandas.DataFrame

592 """

593 if session_path is None: 1 ctx1{

594 return 1 ctx1{

595 path = Path(session_path).joinpath(task_collection) 1 ctx1{

596 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 1 ctx1{

597 if not path: 1 ctx1{

598 return None 1 ctx1{

599 data = pd.read_csv(path, sep=' ', header=None) 1 ctx1{

600 data = data.drop([9], axis=1) 1 ctx1{

601 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 1 ctx1{

602 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts']

603 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path)

604 return data 1 ctx1{

605

606

607def load_ambient_sensor(session_path, task_collection='raw_behavior_data'): (empty)

608 """

609 Load Ambient Sensor data from session.

610

611 Probably could be extracted to DatasetTypes:

612 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb,

613 _ibl_trials.relativeHumidity

614 Returns a list of dicts one dict per trial.

615 dict keys are:

616 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity'])

617

618 :param session_path: Absoulte path of session folder

619 :type session_path: str

620 :return: list of dicts

621 :rtype: list

622 """

623 if session_path is None:

624 return

625 path = Path(session_path).joinpath(task_collection)

626 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None)

627 if not path:

628 return None

629 data = []

630 with open(path, 'r') as f:

631 for line in f:

632 data.append(json.loads(line))

633 return data

634

635

636def load_mic(session_path, task_collection='raw_behavior_data'): (empty)

637 """

638 Load Microphone wav file to np.array of len nSamples

639

640 :param session_path: Absolute path of session folder

641 :type session_path: str

642 :return: An array of values of the sound waveform

643 :rtype: numpy.array

644 """

645 if session_path is None:

646 return

647 path = Path(session_path).joinpath(task_collection)

648 path = next(path.glob("_iblrig_micData.raw*.wav"), None)

649 if not path:

650 return None

651 fp = wave.open(path)

652 nchan = fp.getnchannels()

653 N = fp.getnframes()

654 dstr = fp.readframes(N * nchan)

655 data = np.frombuffer(dstr, np.int16)

656 data = np.reshape(data, (-1, nchan))

657 return data

658

659

660def _clean_wheel_dataframe(data, label, path): (empty)

661 if np.any(data.isna()): 23 ctx1a9*1Ae87mrghcjuklfpbnqd

662 _logger.warning(label + ' has missing/incomplete records \n %s', path) 8 ctx1a*Ae87bd

663 # first step is to re-interpret as numeric objects if not already done

664 for col in data.columns: 23 ctx1a9*1Ae87mrghcjuklfpbnqd

665 if data[col].dtype == object and col not in ['bns_ts']: 23 ctx1a9*1Ae87mrghcjuklfpbnqd

666 data[col] = pd.to_numeric(data[col], errors='coerce') 2 ctx18d

667 # then drop Nans and duplicates

668 data.dropna(inplace=True) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

669 data.drop_duplicates(keep='first', inplace=True) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

670 data.reset_index(inplace=True) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

671 # handle the clock resets when microseconds exceed uint32 max value

672 drop_first = False 23 ctx1a9*1Ae87mrghcjuklfpbnqd

673 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 23 ctx1a9*1Ae87mrghcjuklfpbnqd

674 if any(np.diff(data['re_ts']) < 0): 23 ctx1a9*1Ae87mrghcjuklfpbnqd

675 ind = np.where(np.diff(data['re_ts']) < 0)[0] 16 ctx19*Ae87mghcjklfbd

676 for i in ind: 16 ctx19*Ae87mghcjklfbd

677 # the first sample may be corrupt, in this case throw away

678 if i <= 1: 16 ctx19*Ae87mghcjklfbd

679 drop_first = i 6 ctx19Ae7cb

680 _logger.warning(label + ' rotary encoder positions timestamps' 6 ctx19Ae7cb

681 ' first sample corrupt ' + str(path))

682 # if it's an uint32 wraparound, the diff should be close to 2 ** 32

683 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 14 ctx19*Ae87mghjklfd

684 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 11 ctx1*Ae7mghjkld

685 # there is also the case where 2 positions are swapped and need to be swapped back

686

687 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 9 ctx19Ae87ghfd

688 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 8 ctx19Ae7ghfd

689 str(i) + ' ' + str(path))

690 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 8 ctx19Ae7ghfd

691 data.iloc[i], data.iloc[i + 1] = b, a 8 ctx19Ae7ghfd

692 # if none of those 3 cases apply, raise an error

693 else:

694 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1 ctx18

695 data.sort_values('re_ts', inplace=True) 1 ctx18

696 data.reset_index(inplace=True) 1 ctx18

697 if drop_first is not False: 23 ctx1a9*1Ae87mrghcjuklfpbnqd

698 data.drop(data.loc[:drop_first].index, inplace=True) 6 ctx19Ae7cb

699 data = data.reindex() 6 ctx19Ae7cb

700 return data 23 ctx1a9*1Ae87mrghcjuklfpbnqd

701

702

703def _groom_wheel_data_lt5(data, label='file ', path=''): (empty)

704 """

705 The whole purpose of this function is to account for variability and corruption in

706 the wheel position files. There are many possible errors described below, but

707 nothing excludes getting new ones.

708 """

709 data = _clean_wheel_dataframe(data, label, path) 11 ctx1a9*1Ae87cbd

710 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 11 ctx1a9*1Ae87cbd

711 # check if the time scale is in ms

712 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 11 ctx1a9*1Ae87cbd

713 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds

714 if data['re_ts'].iloc[-1] / (sess_len_sec + 1e-6) < 1e5: # should be 1e6 normally 11 ctx1a9*1Ae87cbd

715 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx11e8cb

716 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')

717 data['re_ts'] = data['re_ts'] * 1000 5 ctx11e8cb

718 return data 11 ctx1a9*1Ae87cbd

719

720

721def _groom_wheel_data_ge5(data, label='file ', path=''): (empty)

722 """

723 The whole purpose of this function is to account for variability and corruption in

724 the wheel position files. There are many possible errors described below, but

725 nothing excludes getting new ones.

726 """

727 data = _clean_wheel_dataframe(data, label, path) 20 ctx191Ae87mrghjuklfpbnqd

728 # check if the time scale is in ms

729 if (data['re_ts'].iloc[-1] - data['re_ts'].iloc[0]) / 1e6 < 20: 20 ctx191Ae87mrghjuklfpbnqd

730 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx187bnd

731 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')

732 data['re_ts'] = data['re_ts'] * 1000 5 ctx187bnd

733 return data 20 ctx191Ae87mrghjuklfpbnqd

734

735

736def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5, (empty)

737 return_index=False):

738 """

739 Attempts to find matching timestamps in 2 time-series that have an offset, are drifting,

740 and are most likely incomplete: sizes don't have to match, some pulses may be missing

741 in any series.

742 Only works with irregular time series as it relies on the derivative to match sync.

743 :param t0:

744 :param t1:

745 :param diff_threshold:

746 :param drift_threshold_ppm: (150)

747 :param max_shift: (200)

748 :param return_index (False)

749 :return:

750 """

751 nsync = min(t0.size, t1.size) 3 ctx1=bd

752 dt0 = np.diff(t0) 3 ctx1=bd

753 dt1 = np.diff(t1) 3 ctx1=bd

754 ind = np.zeros_like(dt0) * np.nan 3 ctx1=bd

755 i0 = 0 3 ctx1=bd

756 i1 = 0 3 ctx1=bd

757 cdt = np.nan # the current time difference between the two series to compute drift 3 ctx1=bd

758 while i0 < (nsync - 1): 3 ctx1=bd

759 # look in the next max_shift events the ones whose derivative match

760 isearch = np.arange(i1, min(max_shift + i1, dt1.size)) 3 ctx1=bd

761 dec = np.abs(dt0[i0] - dt1[isearch]) < diff_threshold 3 ctx1=bd

762 # another constraint is to check the dt for the maximum drift

763 if ~np.isnan(cdt): 3 ctx1=bd

764 drift_ppm = np.abs((cdt - (t0[i0] - t1[isearch])) / dt1[isearch]) * 1e6 3 ctx1=bd

765 dec = np.logical_and(dec, drift_ppm <= drift_threshold_ppm) 3 ctx1=bd

766 # if one is found

767 if np.any(dec): 3 ctx1=bd

768 ii1 = np.where(dec)[0][0] 3 ctx1=bd

769 ind[i0] = i1 + ii1 3 ctx1=bd

770 i1 += ii1 + 1 3 ctx1=bd

771 cdt = t0[i0 + 1] - t1[i1 + ii1] 3 ctx1=bd

772 i0 += 1 3 ctx1=bd

773 it0 = np.where(~np.isnan(ind))[0] 3 ctx1=bd

774 it1 = ind[it0].astype(int) 3 ctx1=bd

775 ind0 = np.unique(np.r_[it0, it0 + 1]) 3 ctx1=bd

776 ind1 = np.unique(np.r_[it1, it1 + 1]) 3 ctx1=bd

777 if return_index: 3 ctx1=bd

778 return t0[ind0], t1[ind1], ind0, ind1

779 else:

780 return t0[ind0], t1[ind1] 3 ctx1=bd

781

782

783def load_bpod_fronts(session_path: str, data: list = False, task_collection: str = 'raw_behavior_data') -> list: (empty)

784 """load_bpod_fronts

785 Loads BNC1 and BNC2 bpod channels times and polarities from session_path

786

787 :param session_path: a valid session_path

788 :type session_path: str

789 :param data: pre-loaded raw data dict, defaults to False

790 :type data: list, optional

791 :return: List of dicts BNC1 and BNC2 {"times": np.array, "polarities":np.array}

792 :rtype: list

793 """

794 if not data: 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

795 data = load_data(session_path, task_collection) 2 ctx1ac

796

797 BNC1_fronts = np.array([[np.nan, np.nan]]) 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

798 BNC2_fronts = np.array([[np.nan, np.nan]]) 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

799 for tr in data: 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

800 BNC1_fronts = np.append( 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

801 BNC1_fronts,

802 np.array(

803 [

804 [x, 1]

805 for x in tr["behavior_data"]["Events timestamps"].get("BNC1High", [np.nan])

806 ]

807 ),

808 axis=0,

809 )

810 BNC1_fronts = np.append( 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

811 BNC1_fronts,

812 np.array(

813 [

814 [x, -1]

815 for x in tr["behavior_data"]["Events timestamps"].get("BNC1Low", [np.nan])

816 ]

817 ),

818 axis=0,

819 )

820 BNC2_fronts = np.append( 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

821 BNC2_fronts,

822 np.array(

823 [

824 [x, 1]

825 for x in tr["behavior_data"]["Events timestamps"].get("BNC2High", [np.nan])

826 ]

827 ),

828 axis=0,

829 )

830 BNC2_fronts = np.append( 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

831 BNC2_fronts,

832 np.array(

833 [

834 [x, -1]

835 for x in tr["behavior_data"]["Events timestamps"].get("BNC2Low", [np.nan])

836 ]

837 ),

838 axis=0,

839 )

840

841 BNC1_fronts = BNC1_fronts[1:, :] 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

842 BNC1_fronts = BNC1_fronts[BNC1_fronts[:, 0].argsort()] 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

843 BNC2_fronts = BNC2_fronts[1:, :] 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

844 BNC2_fronts = BNC2_fronts[BNC2_fronts[:, 0].argsort()] 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

845

846 BNC1 = {"times": BNC1_fronts[:, 0], "polarities": BNC1_fronts[:, 1]} 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

847 BNC2 = {"times": BNC2_fronts[:, 0], "polarities": BNC2_fronts[:, 1]} 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

848

849 return [BNC1, BNC2] 50 ctx1aLMNOPCQRSTUVJWXYZeDEmHrghcIBxizwsv0tyKojuklfpFbnq

850

851

852def get_port_events(trial: dict, name: str = '') -> list: (empty)

853 """get_port_events

854 Return all event timestamps from bpod raw data trial that match 'name'

855 --> looks in trial['behavior_data']['Events timestamps']

856

857 :param trial: raw trial dict

858 :type trial: dict

859 :param name: name of event, defaults to ''

860 :type name: str, optional

861 :return: Sorted list of event timestamps

862 :rtype: list

863 TODO: add polarities?

864 """

865 out: list = [] 23 ctx1aCeDEmHrghcIBjuklfpFbnq

866 events = trial['behavior_data']['Events timestamps'] 23 ctx1aCeDEmHrghcIBjuklfpFbnq

867 for k in events: 23 ctx1aCeDEmHrghcIBjuklfpFbnq

868 if name in k: 23 ctx1aCeDEmHrghcIBjuklfpFbnq

869 out.extend(events[k]) 22 ctx1CeDEmHrghcIBjuklfpFbnq

870 out = sorted(out) 23 ctx1aCeDEmHrghcIBjuklfpFbnq

871

872 return out 23 ctx1aCeDEmHrghcIBjuklfpFbnq

873

874

875def load_widefield_mmap(session_path, dtype=np.uint16, shape=(540, 640), n_frames=None, mode='r'): (empty)

876 """

877 TODO Document this function

878

879 Parameters

880 ----------

881 session_path

882

883 Returns

884 -------

885

886 """

887 filepath = Path(session_path).joinpath('raw_widefield_data').glob('widefield.raw.*.dat')

888 filepath = next(filepath, None)

889 if not filepath:

890 _logger.warning("No data loaded: could not find raw data file")

891 return None

892

893 if type(dtype) is str:

894 dtype = np.dtype(dtype)

895

896 if n_frames is None:

897 # Get the number of samples from the file size

898 n_frames = int(filepath.stat().st_size / (np.prod(shape) * dtype.itemsize))

899

900 return np.memmap(str(filepath), mode=mode, dtype=dtype, shape=(int(n_frames), *shape))

901

902

903def patch_settings(session_path, collection='raw_behavior_data', (empty)

904 new_collection=None, subject=None, number=None, date=None):

905 """Modify various details in a settings file.

906

907 This function makes it easier to change things like subject name in a settings as it will

908 modify the subject name in the myriad paths. NB: This saves the settings into the same location

909 it was loaded from.

910

911 Parameters

912 ----------

913 session_path : str, pathlib.Path

914 The session path containing the settings file.

915 collection : str

916 The subfolder containing the settings file.

917 new_collection : str

918 An optional new subfolder to change in the settings paths.

919 subject : str

920 An optional new subject name to change in the settings.

921 number : str, int

922 An optional new number to change in the settings.

923 date : str, datetime.date

924 An optional date to change in the settings.

925

926 Returns

927 -------

928 dict

929 The modified settings.

930

931 Examples

932 --------

933 File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location.

934 >>> patch_settings('/data/subject/2020-01-01/002', number='001')

935 >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/')

936

937 File is moved into new collection within the same session, then patched.

938 >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/')

939 >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01')

940

941 Update subject, date and number.

942 >>> new_session_path = Path('/data/foobar/2024-02-24/002')

943 >>> old_session_path = Path('/data/baz/2024-02-23/001')

944 >>> patch_settings(old_session_path, collection='raw_task_data_00',

945 ... subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1])

946 >>> shutil.move(old_session_path, new_session_path)

947 """

948 settings = load_settings(session_path, collection) 2 ctx1aG

949 if not settings: 2 ctx1aG

950 raise IOError('Settings file not found') 1 ctx1G

951

952 filename = PureWindowsPath(settings.get('SETTINGS_FILE_PATH', '_iblrig_taskSettings.raw.json')).name 2 ctx1aG

953 file_path = Path(session_path).joinpath(collection, filename) 2 ctx1aG

954

955 if subject: 2 ctx1aG

956 # Patch subject name

957 old_subject = settings['SUBJECT_NAME'] 2 ctx1aG

958 settings['SUBJECT_NAME'] = subject 2 ctx1aG

959 for k in settings.keys(): 2 ctx1aG

960 if isinstance(settings[k], str): 2 ctx1aG

961 settings[k] = settings[k].replace(f'\\Subjects\\{old_subject}', f'\\Subjects\\{subject}') 2 ctx1aG

962 if 'SESSION_NAME' in settings: 2 ctx1aG

963 settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) 2 ctx1aG

964 settings.pop('PYBPOD_SUBJECT_EXTRA', None) # Get rid of Alyx subject info 2 ctx1aG

965

966 if date: 2 ctx1aG

967 # Patch session datetime

968 date = str(date) 2 ctx1aG

969 old_date = settings['SESSION_DATE'] 2 ctx1aG

970 settings['SESSION_DATE'] = date 2 ctx1aG

971 for k in settings.keys(): 2 ctx1aG

972 if isinstance(settings[k], str): 2 ctx1aG

973 settings[k] = settings[k].replace( 2 ctx1aG

974 f'\\{settings["SUBJECT_NAME"]}\\{old_date}',

975 f'\\{settings["SUBJECT_NAME"]}\\{date}'

976 )

977 settings['SESSION_DATETIME'] = date + settings['SESSION_DATETIME'][10:] 2 ctx1aG

978 if 'SESSION_END_TIME' in settings: 2 ctx1aG

979 settings['SESSION_END_TIME'] = date + settings['SESSION_END_TIME'][10:] 1 ctx1G

980 if 'SESSION_START_TIME' in settings: 2 ctx1aG

981 settings['SESSION_START_TIME'] = date + settings['SESSION_START_TIME'][10:] 1 ctx1G

982

983 if number: 2 ctx1aG

984 # Patch session number

985 old_number = settings['SESSION_NUMBER'] 2 ctx1aG

986 if isinstance(number, int): 2 ctx1aG

987 number = f'{number:03}' 1 ctx1G

988 settings['SESSION_NUMBER'] = number 2 ctx1aG

989 for k in settings.keys(): 2 ctx1aG

990 if isinstance(settings[k], str): 2 ctx1aG

991 settings[k] = settings[k].replace( 2 ctx1aG

992 f'\\{settings["SESSION_DATE"]}\\{old_number}',

993 f'\\{settings["SESSION_DATE"]}\\{number}'

994 )

995

996 if new_collection: 2 ctx1aG

997 if 'SESSION_RAW_DATA_FOLDER' not in settings: 1 ctx1G

998 _logger.warning('SESSION_RAW_DATA_FOLDER key not in settings; collection not updated') 1 ctx1G

999 else:

1000 old_path = settings['SESSION_RAW_DATA_FOLDER'] 1 ctx1G

1001 new_path = PureWindowsPath(settings['SESSION_RAW_DATA_FOLDER']).with_name(new_collection) 1 ctx1G

1002 for k in settings.keys(): 1 ctx1G

1003 if isinstance(settings[k], str): 1 ctx1G

1004 settings[k] = settings[k].replace(old_path, str(new_path)) 1 ctx1G

1005 with open(file_path, 'w') as fp: 2 ctx1aG

1006 json.dump(settings, fp, indent=' ') 2 ctx1aG

1007 return settings 2 ctx1aG

Coverage for ibllib/io/raw_data_loaders.py: 88%

450 statements