Coverage for ibllib/io/raw_data_loaders.py: 84%

1#!/usr/bin/env python 

2# -*- coding:utf-8 -*- 

3# @Author: Niccolò Bonacchi, Miles Wells 

4# @Date: Monday, July 16th 2018, 1:28:46 pm 

5""" (empty)

6Raw Data Loader functions for PyBpod rig 

7 

8Module contains one loader function per raw datafile 

9""" 

10import json (empty)

11import logging (empty)

12import wave (empty)

13from collections import OrderedDict (empty)

14from datetime import datetime (empty)

15from pathlib import Path, PureWindowsPath (empty)

16from typing import Union (empty)

17 

18from dateutil import parser as dateparser (empty)

19from pkg_resources import parse_version (empty)

20import numpy as np (empty)

21import pandas as pd (empty)

22 

23from iblutil.io import jsonable (empty)

24from ibllib.io.video import assert_valid_label (empty)

25from ibllib.time import uncycle_pgts, convert_pgts, date2isostr (empty)

26 

27_logger = logging.getLogger(__name__) (empty)

28 

29 

30def trial_times_to_times(raw_trial): (empty)

31 """ 

32 Parse and convert all trial timestamps to "absolute" time. 

33 Float64 seconds from session start. 

34 

35 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1--- 

36 tsN...absTS = tsN + TrialStartN - BpodStart 

37 

38 Bpod timestamps are in microseconds (µs) 

39 PyBpod timestamps are is seconds (s) 

40 

41 :param raw_trial: raw trial data 

42 :type raw_trial: dict 

43 :return: trial data with modified timestamps 

44 :rtype: dict 

45 """ 

46 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

47 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

48 # ts_te = raw_trial['behavior_data']['Trial end timestamp'] 

49 

50 def convert(ts): 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

51 return ts + ts_ts - ts_bs 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

52 

53 converted_events = {} 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

54 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

55 converted_events.update({k: [convert(i) for i in v]}) 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

56 raw_trial['behavior_data']['Events timestamps'] = converted_events 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

57 

58 converted_states = {} 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

59 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

60 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

61 raw_trial['behavior_data']['States timestamps'] = converted_states 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

62 

63 shift = raw_trial['behavior_data']['Bpod start timestamp'] 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

64 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

65 raw_trial['behavior_data']['Trial start timestamp'] -= shift 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

66 raw_trial['behavior_data']['Trial end timestamp'] -= shift 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

67 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

68 return raw_trial 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

69 

70 

71def load_bpod(session_path, task_collection='raw_behavior_data'): (empty)

72 """ 

73 Load both settings and data from bpod (.json and .jsonable) 

74 

75 :param session_path: Absolute path of session folder 

76 :param task_collection: Collection within sesison path with behavior data 

77 :return: dict settings and list of dicts data 

78 """ 

79 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 15 ctx1b@]^HcfmhzVijaW

80 

81 

82def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'): (empty)

83 """ 

84 Load PyBpod data files (.jsonable). 

85 

86 Bpod timestamps are in microseconds (µs) 

87 PyBpod timestamps are is seconds (s) 

88 

89 :param session_path: Absolute path of session folder 

90 :type session_path: str, Path 

91 :return: A list of len ntrials each trial being a dictionary 

92 :rtype: list of dicts 

93 """ 

94 if session_path is None: 65 ctx1b+kXYZ01O23456789!#P$eQ@]^yrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

95 _logger.warning('No data loaded: session_path is None') 

96 return 

97 path = Path(session_path).joinpath(task_collection) 65 ctx1b+kXYZ01O23456789!#P$eQ@]^yrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

98 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 65 ctx1b+kXYZ01O23456789!#P$eQ@]^yrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

99 if not path: 65 ctx1b+kXYZ01O23456789!#P$eQ@]^yrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

100 _logger.warning('No data loaded: could not find raw data file') 4 ctx1@]^f

101 return None 4 ctx1@]^f

102 data = jsonable.read(path) 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

103 if time == 'absolute': 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

104 data = [trial_times_to_times(t) for t in data] 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

105 return data 62 ctx1b+kXYZ01O23456789!#P$eQyrRwHDstcS*IJlvng%opEfmhzVAuBijTaW'qxCd

106 

107 

108def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame: (empty)

109 """ Loads binary frame data from Bonsai camera recording workflow. 

110 

111 Args: 

112 session_path (StrPath): Path to session folder 

113 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'. 

114 raw (bool, optional): Whether to return raw or parsed data. Defaults to False. 

115 

116 Returns: 

117 parsed: (raw=False, Default) 

118 pandas.DataFrame: 4 int64 columns: { 

119 Timestamp, # float64 (seconds from session start) 

120 embeddedTimeStamp, # float64 (seconds from session start) 

121 embeddedFrameCounter, # int64 (Frame number from session start) 

122 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a 

123 # list of numpy boolean arrays 

124 # e.g. np.array([True, False, False, False]) 

125 } 

126 raw: 

127 pandas.DataFrame: 4 int64 columns: { 

128 Timestamp, # UTC ticks from BehaviorPC 

129 # (100's of ns since midnight 1/1/0001) 

130 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion) 

131 embeddedFrameCounter, # Frame counter (int) 

132 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins 

133 } 

134 """ 

135 camera = assert_valid_label(camera) 4 ctx1`(Fg

136 fpath = Path(session_path).joinpath("raw_video_data") 4 ctx1`(Fg

137 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 4 ctx1`(Fg

138 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 4 ctx1`(Fg

139 rdata = np.fromfile(fpath, dtype=np.float64) 4 ctx1`(Fg

140 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 4 ctx1`(Fg

141 rows = int(rdata.size / 4) 4 ctx1`(Fg

142 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 4 ctx1`(Fg

143 df_dict = dict.fromkeys( 4 ctx1`(Fg

144 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"] 

145 ) 

146 df = pd.DataFrame(data, columns=df_dict.keys()) 4 ctx1`(Fg

147 if raw: 4 ctx1`(Fg

148 return df 1 ctx1`

149 

150 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 4 ctx1`(Fg

151 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 4 ctx1`(Fg

152 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 4 ctx1`(Fg

153 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 4 ctx1`(Fg

154 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 4 ctx1`(Fg

155 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 4 ctx1`(Fg

156 

157 parsed_df = pd.DataFrame.from_dict(df_dict) 4 ctx1`(Fg

158 return parsed_df 4 ctx1`(Fg

159 

160 

161def load_camera_ssv_times(session_path, camera: str): (empty)

162 """ 

163 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv 

164 

165 NB: For some sessions the frame times are in the first column, in others the order is reversed. 

166 NB: If using the new bin file the bonsai_times is a float in seconds since first frame 

167 :param session_path: Absolute path of session folder 

168 :param camera: Name of the camera to load, e.g. 'left' 

169 :return: array of datetimes, array of frame times in seconds 

170 """ 

171 camera = assert_valid_label(camera) 16 ctx2eb| c K l v n fbL F U g o p E a

172 video_path = Path(session_path).joinpath('raw_video_data') 16 ctx2eb| c K l v n fbL F U g o p E a

173 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 16 ctx2eb| c K l v n fbL F U g o p E a

174 df = load_camera_frameData(session_path, camera=camera) 2 ctx1Fg

175 return df['Timestamp'].values, df['embeddedTimeStamp'].values 2 ctx1Fg

176 

177 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 16 ctx2eb| c K l v n fbL F U g o p E a

178 if not file: 16 ctx2eb| c K l v n fbL F U g o p E a

179 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 

180 raise FileNotFoundError(file + ' not found') 

181 # NB: Numpy has deprecated support for non-naive timestamps. 

182 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s. 

183 # from datetime import timezone 

184 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)} 

185 

186 # Determine the order of the columns by reading one line and testing whether the first value 

187 # is an integer or not. 

188 with open(file, 'r') as f: 16 ctx2eb| c K l v n fbL F U g o p E a

189 line = f.readline() 16 ctx2eb| c K l v n fbL F U g o p E a

190 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 16 ctx2eb| c K l v n fbL F U g o p E a

191 try: 16 ctx2eb| c K l v n fbL F U g o p E a

192 int(line.split(' ')[1]) 16 ctx2eb| c K l v n fbL F U g o p E a

193 except ValueError: 8 ctx1|clnopEa

194 type_map.move_to_end('bonsai') 8 ctx1|clnopEa

195 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 16 ctx2eb| c K l v n fbL F U g o p E a

196 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 16 ctx2eb| c K l v n fbL F U g o p E a

197 bonsai_times = ssv_times['bonsai'] 16 ctx2eb| c K l v n fbL F U g o p E a

198 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 16 ctx2eb| c K l v n fbL F U g o p E a

199 return bonsai_times, camera_times 16 ctx2eb| c K l v n fbL F U g o p E a

200 

201 

202def load_embedded_frame_data(session_path, label: str, raw=False): (empty)

203 """ 

204 Load the embedded frame count and GPIO for a given session. If the file doesn't exist, 

205 or is empty, None values are returned. 

206 :param session_path: Absolute path of session folder 

207 :param label: The specific video to load, one of ('left', 'right', 'body') 

208 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

209 returned starting from 0 and the GPIO is returned as a dict of indices 

210 :return: The frame count, GPIO 

211 """ 

212 count = load_camera_frame_count(session_path, label, raw=raw) 19 ctx1=cI_;JKlvnLFUgopEfa

213 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 19 ctx1=cI_;JKlvnLFUgopEfa

214 return count, gpio 19 ctx1=cI_;JKlvnLFUgopEfa

215 

216 

217def load_camera_frame_count(session_path, label: str, raw=True): (empty)

218 """ 

219 Load the embedded frame count for a given session. If the file doesn't exist, or is empty, 

220 a None value is returned. 

221 :param session_path: Absolute path of session folder 

222 :param label: The specific video to load, one of ('left', 'right', 'body') 

223 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is 

224 returned starting from 0 

225 :return: The frame count 

226 """ 

227 if session_path is None: 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

228 return 1 ctx2lb

229 

230 label = assert_valid_label(label) 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

231 video_path = Path(session_path).joinpath('raw_video_data') 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

232 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

233 df = load_camera_frameData(session_path, camera=label) 2 ctx1Fg

234 return df['embeddedFrameCounter'].values 2 ctx1Fg

235 

236 # Load frame count 

237 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

238 count_file = next(glob, None) 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

239 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

240 if len(count) == 0: 20 ctx2lb= c I _ ; J K l v n L F U g o p E f a

241 return 9 ctx2lbc I _ ; J l f a

242 if not raw: 12 ctx2lb= K v n L F U g o p E

243 count -= count[0] # start from zero 10 ctx2lb= K n L U g o p E

244 return count 12 ctx2lb= K v n L F U g o p E

245 

246 

247def load_camera_gpio(session_path, label: str, as_dicts=False): (empty)

248 """ 

249 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is 

250 returned. 

251 

252 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits 

253 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by 

254 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state). 

255 

256 :param session_path: Absolute path of session folder 

257 :param label: The specific video to load, one of ('left', 'right', 'body') 

258 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins) 

259 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities'). 

260 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4. 

261 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'), 

262 or None if the dictionary is empty. 

263 """ 

264 if session_path is None: 20 ctx1(=cI_;JKlvnLFUgopEfa

265 return 1 ctx1(

266 raw_path = Path(session_path).joinpath('raw_video_data') 20 ctx1(=cI_;JKlvnLFUgopEfa

267 label = assert_valid_label(label) 20 ctx1(=cI_;JKlvnLFUgopEfa

268 

269 # Load pin state 

270 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 20 ctx1(=cI_;JKlvnLFUgopEfa

271 df = load_camera_frameData(session_path, camera=label, raw=False) 3 ctx1(Fg

272 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 3 ctx1(Fg

273 if len(gpio) == 0: 3 ctx1(Fg

274 return [None] * 4 if as_dicts else None 

275 else: 

276 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 19 ctx1(=cI_;JKlvnLUgopEfa

277 # This deals with missing and empty files the same 

278 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 19 ctx1(=cI_;JKlvnLUgopEfa

279 # Check values make sense (4 pins = 16 possible values) 

280 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 19 ctx1(=cI_;JKlvnLUgopEfa

281 _logger.warning('Unexpected GPIO values; decoding may fail') 1 ctx1(

282 if len(gpio) == 0: 19 ctx1(=cI_;JKlvnLUgopEfa

283 return [None] * 4 if as_dicts else None 9 ctx1(cI_;Jlfa

284 # 4 pins represented as uint32 

285 # For each pin, shift its bit to the end and check the bit is set 

286 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 11 ctx1(=KvnLUgopE

287 

288 if as_dicts: 12 ctx1(=KvnLFUgopE

289 if not gpio.any(): 11 ctx1(=KnLFUgopE

290 _logger.error('No GPIO changes') 1 ctx1(

291 return [None] * 4 1 ctx1(

292 # Find state changes for each pin and construct a dict of indices and polarities for each 

293 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 11 ctx1(=KnLFUgopE

294 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)] 

295 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8 

296 gpio = [{'indices': np.where(edges[:, i])[0], 11 ctx1(=KnLFUgopE

297 'polarities': edges[edges[:, i] != 0, i]} 

298 for i in range(4)] 

299 # Replace empty dicts with None 

300 gpio = [None if x['indices'].size == 0 else x for x in gpio] 11 ctx1(=KnLFUgopE

301 

302 return gpio 12 ctx1(=KvnLFUgopE

303 

304 

305def _read_settings_json_compatibility_enforced(settings): (empty)

306 """ 

307 Patch iblrig settings for compatibility across rig versions. 

308 

309 Parameters 

310 ---------- 

311 settings : pathlib.Path, dict 

312 Either a _iblrig_taskSettings.raw.json file path or the loaded settings. 

313 

314 Returns 

315 ------- 

316 dict 

317 The task settings patched for compatibility. 

318 """ 

319 if isinstance(settings, dict): 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

320 md = settings.copy() 

321 else: 

322 with open(settings) as js: 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

323 md = json.load(js) 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

324 if 'IS_MOCK' not in md: 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

325 md['IS_MOCK'] = False 72 ctx2b k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } ~ @ ] ^ ab- . / bbcbdby r R w H s t S * I ; J l n % o p f m h M V A u B { i j T a W ' q x C d

326 if 'IBLRIG_VERSION_TAG' not in md.keys(): 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

327 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 8 ctx2ab- . / bbcbdbS

328 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on) 

329 if md['IBLRIG_VERSION_TAG'] == '': 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

330 pass 12 ctx2+ kbab- . / bbcbdbS * d

331 elif parse_version(md.get('IBLRIG_VERSION_TAG')) >= parse_version('8.0.0'): 74 ctx2b k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } ~ @ ] ^ y r R w H gbD s t c I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

332 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 

333 md['PYBPOD_BOARD'] = md['RIG_NAME'] 

334 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 

335 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 

336 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 

337 elif parse_version(md.get('IBLRIG_VERSION_TAG')) <= parse_version('3.2.3'): 74 ctx2b k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } ~ @ ] ^ y r R w H gbD s t c I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

338 if 'LAST_TRIAL_DATA' in md.keys(): 1 ctx1a

339 md.pop('LAST_TRIAL_DATA') 1 ctx1a

340 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1a

341 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1 ctx1a

342 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1 ctx1a

343 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1 ctx1a

344 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1 ctx1a

345 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1 ctx1a

346 # parse the date format to Django supported ISO 

347 dt = dateparser.parse(md['SESSION_DATETIME']) 1 ctx1a

348 md['SESSION_DATETIME'] = date2isostr(dt) 1 ctx1a

349 # add the weight key if it doesn't already exist 

350 if 'SUBJECT_WEIGHT' not in md: 1 ctx1a

351 md['SUBJECT_WEIGHT'] = None 1 ctx1a

352 return md 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

353 

354 

355def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'): (empty)

356 """ 

357 Load PyBpod Settings files (.json). 

358 

359 [description] 

360 

361 :param session_path: Absolute path of session folder 

362 :type session_path: str, Path 

363 :return: Settings dictionary 

364 :rtype: dict 

365 """ 

366 if session_path is None: 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

367 _logger.warning("No data loaded: session_path is None") 

368 return 

369 path = Path(session_path).joinpath(task_collection) 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

370 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

371 if not path: 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

372 _logger.warning("No data loaded: could not find raw settings file") 4 ctx1@fM{

373 return None 4 ctx1@fM{

374 settings = _read_settings_json_compatibility_enforced(path) 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

375 return settings 85 ctx2b + k X Y Z 0 1 O 2 3 4 5 6 7 8 9 ! # P $ N G e Q } kb~ @ ] ^ ab- . / bbcbdby r R w H gbD s t c S * I ; J K l v n L F % o p hbibjbf m h z M V A u B { i j T a W ' q x C d

376 

377 

378def load_stim_position_screen(session_path, task_collection='raw_behavior_data'): (empty)

379 path = Path(session_path).joinpath(task_collection) 

380 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None) 

381 

382 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip') 

383 data.columns = ['contrast', 'position', 'bns_ts'] 

384 data['bns_ts'] = pd.to_datetime(data['bns_ts']) 

385 return data 

386 

387 

388def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False): (empty)

389 """ 

390 Load Rotary Encoder (RE) events raw data file. 

391 

392 Assumes that a folder called "raw_behavior_data" exists in folder. 

393 

394 Events number correspond to following bpod states: 

395 1: correct / hide_stim 

396 2: stim_on 

397 3: closed_loop 

398 4: freeze_error / freeze_correct 

399 

400 >>> data.columns 

401 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

402 'sm_ev', # State Machine Event 'numpy.int64' 

403 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp' 

404 # pd.to_datetime(data.bns_ts) to work in datetimes 

405 

406 :param session_path: [description] 

407 :type session_path: [type] 

408 :return: dataframe w/ 3 cols and (ntrials * 3) lines 

409 :rtype: Pandas.DataFrame 

410 """ 

411 if session_path is None: 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

412 return 

413 path = Path(session_path).joinpath(task_collection) 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

414 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

415 if not settings: 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

416 settings = load_settings(session_path, task_collection=task_collection) 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

417 if settings is None or not settings.get('IBLRIG_VERSION_TAG'): 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

418 settings = {'IBLRIG_VERSION_TAG': '100.0.0'} 1 ctx1d

419 # auto-detect old files when version is not labeled 

420 with open(path) as fid: 1 ctx1d

421 line = fid.readline() 1 ctx1d

422 if line.startswith('Event') and 'StateMachine' in line: 1 ctx1d

423 settings = {'IBLRIG_VERSION_TAG': '0.0.0'} 1 ctx1d

424 if not path: 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

425 return None 

426 if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'): 25 ctx1bkNeyrwDstcfmhzAuBijaqxCd

427 return _load_encoder_events_file_ge5(path) 21 ctx1kNeyrwDstfmhzAuBaqxCd

428 else: 

429 return _load_encoder_events_file_lt5(path) 10 ctx1bkNecijaqd

430 

431 

432def _load_encoder_ssv_file(file_path, **kwargs): (empty)

433 file_path = Path(file_path) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

434 if file_path.stat().st_size == 0: 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

435 _logger.error(f"{file_path.name} is an empty file. ") 

436 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ") 

437 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

438 

439 

440def _load_encoder_positions_file_lt5(file_path): (empty)

441 """ 

442 File loader without the session overhead 

443 :param file_path: 

444 :return: dataframe of encoder events 

445 """ 

446 data = _load_encoder_ssv_file(file_path, 13 ctx1b:?kGe)cijaqd

447 names=['_', 're_ts', 're_pos', 'bns_ts', '__'], 

448 usecols=['re_ts', 're_pos', 'bns_ts']) 

449 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 13 ctx1b:?kGe)cijaqd

450 

451 

452def _load_encoder_positions_file_ge5(file_path): (empty)

453 """ 

454 File loader without the session overhead 

455 :param file_path: 

456 :return: dataframe of encoder events 

457 """ 

458 data = _load_encoder_ssv_file(file_path, 23 ctx1:kGe)yrwDstfmhzAuBaqxCd

459 names=['re_ts', 're_pos', '_'], 

460 usecols=['re_ts', 're_pos']) 

461 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 23 ctx1:kGe)yrwDstfmhzAuBaqxCd

462 

463 

464def _load_encoder_events_file_lt5(file_path): (empty)

465 """ 

466 File loader without the session overhead 

467 :param file_path: 

468 :return: dataframe of encoder events 

469 """ 

470 data = _load_encoder_ssv_file(file_path, 11 ctx1bkNe,cijaqd

471 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'], 

472 usecols=['re_ts', 'sm_ev', 'bns_ts']) 

473 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 11 ctx1bkNe,cijaqd

474 

475 

476def _load_encoder_events_file_ge5(file_path): (empty)

477 """ 

478 File loader without the session overhead 

479 :param file_path: 

480 :return: dataframe of encoder events 

481 """ 

482 data = _load_encoder_ssv_file(file_path, 22 ctx1kNe,yrwDstfmhzAuBaqxCd

483 names=['re_ts', 'sm_ev', '_'], 

484 usecols=['re_ts', 'sm_ev']) 

485 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 22 ctx1kNe,yrwDstfmhzAuBaqxCd

486 

487 

488def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False): (empty)

489 """ 

490 Load Rotary Encoder (RE) positions from raw data file within a session path. 

491 

492 Assumes that a folder called "raw_behavior_data" exists in folder. 

493 Positions are RE ticks [-512, 512] == [-180º, 180º] 

494 0 == trial stim init position 

495 Positive nums are rightwards movements (mouse) or RE CW (mouse) 

496 

497 Variable line number, depends on movements. 

498 

499 Raw datafile Columns: 

500 Position, RE timestamp, RE Position, Bonsai Timestamp 

501 

502 Position is always equal to 'Position' so this column was dropped. 

503 

504 >>> data.columns 

505 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64' 

506 're_pos', # Rotary Encoder position (ticks) 'numpy.int64' 

507 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp' 

508 # pd.to_datetime(data.bns_ts) to work in datetimes 

509 

510 :param session_path: Absolute path of session folder 

511 :type session_path: str 

512 :return: dataframe w/ 3 cols and N positions 

513 :rtype: Pandas.DataFrame 

514 """ 

515 if session_path is None: 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

516 return 

517 path = Path(session_path).joinpath(task_collection) 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

518 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

519 if not settings: 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

520 settings = load_settings(session_path, task_collection=task_collection) 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

521 if settings is None or not settings.get('IBLRIG_VERSION_TAG'): 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

522 settings = {'IBLRIG_VERSION_TAG': '100.0.0'} 1 ctx1d

523 # auto-detect old files when version is not labeled 

524 with open(path) as fid: 1 ctx1d

525 line = fid.readline() 1 ctx1d

526 if line.startswith('Position'): 1 ctx1d

527 settings = {'IBLRIG_VERSION_TAG': '0.0.0'} 1 ctx1d

528 if not path: 26 ctx1bkGeyrwDstclfmhzAuBijaqxCd

529 _logger.warning("No data loaded: could not find raw encoderPositions file") 2 ctx1la

530 return None 2 ctx1la

531 if parse_version(settings['IBLRIG_VERSION_TAG']) >= parse_version('5.0.0'): 25 ctx1bkGeyrwDstcfmhzAuBijaqxCd

532 return _load_encoder_positions_file_ge5(path) 21 ctx1kGeyrwDstfmhzAuBaqxCd

533 else: 

534 return _load_encoder_positions_file_lt5(path) 10 ctx1bkGecijaqd

535 

536 

537def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'): (empty)

538 """ 

539 Load Rotary Encoder trial info from raw data file. 

540 

541 Assumes that a folder calles "raw_behavior_data" exists in folder. 

542 

543 NOTE: Last trial probably inexistent data (Trial info is sent on trial start 

544 and data is only saved on trial exit...) max(trialnum) should be N+1 if N 

545 is the amount of trial data saved. 

546 

547 Raw datafile Columns: 

548 

549 >>> data.columns 

550 >>> ['trial_num', # Trial Number 'numpy.int64' 

551 'stim_pos_init', # Initial position of visual stim 'numpy.int64' 

552 'stim_contrast', # Contrast of visual stimulus 'numpy.float64' 

553 'stim_freq', # Frequency of gabor patch 'numpy.float64' 

554 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64' 

555 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64' 

556 'stim_sigma', # Size of patch 'numpy.float64' 

557 'stim_phase', # Phase of gabor 'numpy.float64' 

558 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp' 

559 # pd.to_datetime(data.bns_ts) to work in datetimes 

560 

561 :param session_path: Absoulte path of session folder 

562 :type session_path: str 

563 :return: dataframe w/ 9 cols and ntrials lines 

564 :rtype: Pandas.DataFrame 

565 """ 

566 if session_path is None: 1 ctx2mb

567 return 

568 path = Path(session_path).joinpath(task_collection) 1 ctx2mb

569 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 1 ctx2mb

570 if not path: 1 ctx2mb

571 return None 

572 data = pd.read_csv(path, sep=' ', header=None) 1 ctx2mb

573 data = data.drop([9], axis=1) 1 ctx2mb

574 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 1 ctx2mb

575 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts'] 

576 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path) 

577 return data 1 ctx2mb

578 

579 

580def load_ambient_sensor(session_path, task_collection='raw_behavior_data'): (empty)

581 """ 

582 Load Ambient Sensor data from session. 

583 

584 Probably could be extracted to DatasetTypes: 

585 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb, 

586 _ibl_trials.relativeHumidity 

587 Returns a list of dicts one dict per trial. 

588 dict keys are: 

589 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity']) 

590 

591 :param session_path: Absoulte path of session folder 

592 :type session_path: str 

593 :return: list of dicts 

594 :rtype: list 

595 """ 

596 if session_path is None: 

597 return 

598 path = Path(session_path).joinpath(task_collection) 

599 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None) 

600 if not path: 

601 return None 

602 data = [] 

603 with open(path, 'r') as f: 

604 for line in f: 

605 data.append(json.loads(line)) 

606 return data 

607 

608 

609def load_mic(session_path, task_collection='raw_behavior_data'): (empty)

610 """ 

611 Load Microphone wav file to np.array of len nSamples 

612 

613 :param session_path: Absolute path of session folder 

614 :type session_path: str 

615 :return: An array of values of the sound waveform 

616 :rtype: numpy.array 

617 """ 

618 if session_path is None: 

619 return 

620 path = Path(session_path).joinpath(task_collection) 

621 path = next(path.glob("_iblrig_micData.raw*.wav"), None) 

622 if not path: 

623 return None 

624 fp = wave.open(path) 

625 nchan = fp.getnchannels() 

626 N = fp.getnframes() 

627 dstr = fp.readframes(N * nchan) 

628 data = np.frombuffer(dstr, np.int16) 

629 data = np.reshape(data, (-1, nchan)) 

630 return data 

631 

632 

633def _clean_wheel_dataframe(data, label, path): (empty)

634 if np.any(data.isna()): 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

635 _logger.warning(label + ' has missing/incomplete records \n %s', path) 11 ctx1b?kGe,)ijqd

636 # first step is to re-interpret as numeric objects if not already done 

637 for col in data.columns: 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

638 if data[col].dtype == object and col not in ['bns_ts']: 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

639 data[col] = pd.to_numeric(data[col], errors='coerce') 2 ctx1,d

640 # then drop Nans and duplicates 

641 data.dropna(inplace=True) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

642 data.drop_duplicates(keep='first', inplace=True) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

643 data.reset_index(inplace=True) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

644 # handle the clock resets when microseconds exceed uint32 max value 

645 drop_first = False 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

646 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

647 if any(np.diff(data['re_ts']) < 0): 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

648 ind = np.where(np.diff(data['re_ts']) < 0)[0] 16 ctx1:?kGe,)rwstchuad

649 for i in ind: 16 ctx1:?kGe,)rwstchuad

650 # the first sample may be corrupt, in this case throw away 

651 if i <= 1: 16 ctx1:?kGe,)rwstchuad

652 drop_first = i 6 ctx1:Ge)ca

653 _logger.warning(label + ' rotary encoder positions timestamps' 6 ctx1:Ge)ca

654 ' first sample corrupt ' + str(path)) 

655 # if it's an uint32 wraparound, the diff should be close to 2 ** 32 

656 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 14 ctx1:?kGe,)rwsthud

657 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 11 ctx1?kGe)rwsthd

658 # there is also the case where 2 positions are swapped and need to be swapped back 

659 

660 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 10 ctx1:Ge,)rstud

661 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 9 ctx1:Ge)rstud

662 str(i) + ' ' + str(path)) 

663 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 9 ctx1:Ge)rstud

664 data.iloc[i], data.iloc[i + 1] = b, a 9 ctx1:Ge)rstud

665 # if none of those 3 cases apply, raise an error 

666 else: 

667 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1 ctx1,

668 data.sort_values('re_ts', inplace=True) 1 ctx1,

669 data.reset_index(inplace=True) 1 ctx1,

670 if drop_first is not False: 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

671 data.drop(data.loc[:drop_first].index, inplace=True) 6 ctx1:Ge)ca

672 data = data.reindex() 6 ctx1:Ge)ca

673 return data 30 ctx1b:?kNGe,)yrwDstcfmhzAuBijaqxCd

674 

675 

676def _groom_wheel_data_lt5(data, label='file ', path=''): (empty)

677 """ 

678 The whole purpose of this function is to account for variability and corruption in 

679 the wheel position files. There are many possible errors described below, but 

680 nothing excludes getting new ones. 

681 """ 

682 data = _clean_wheel_dataframe(data, label, path) 15 ctx1b:?kNGe,)cijaqd

683 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 15 ctx1b:?kNGe,)cijaqd

684 # check if the time scale is in ms 

685 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 15 ctx1b:?kNGe,)cijaqd

686 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds 

687 if data['re_ts'].iloc[-1] / (sess_len_sec + 1e-6) < 1e5: # should be 1e6 normally 15 ctx1b:?kNGe,)cijaqd

688 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx1Ne,ca

689 'RE firmware needs upgrading and wheel velocity is potentially inaccurate') 

690 data['re_ts'] = data['re_ts'] * 1000 5 ctx1Ne,ca

691 return data 15 ctx1b:?kNGe,)cijaqd

692 

693 

694def _groom_wheel_data_ge5(data, label='file ', path=''): (empty)

695 """ 

696 The whole purpose of this function is to account for variability and corruption in 

697 the wheel position files. There are many possible errors described below, but 

698 nothing excludes getting new ones. 

699 """ 

700 data = _clean_wheel_dataframe(data, label, path) 25 ctx1:kNGe,)yrwDstfmhzAuBaqxCd

701 # check if the time scale is in ms 

702 if (data['re_ts'].iloc[-1] - data['re_ts'].iloc[0]) / 1e6 < 20: 25 ctx1:kNGe,)yrwDstfmhzAuBaqxCd

703 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 5 ctx1,)qxd

704 'RE firmware needs upgrading and wheel velocity is potentially inaccurate') 

705 data['re_ts'] = data['re_ts'] * 1000 5 ctx1,)qxd

706 return data 25 ctx1:kNGe,)yrwDstfmhzAuBaqxCd

707 

708 

709def save_bool(save, dataset_type): (empty)

710 if isinstance(save, bool): 

711 out = save 

712 elif isinstance(save, list): 

713 out = (dataset_type in save) or (Path(dataset_type).stem in save) 

714 if out: 

715 _logger.debug('extracting' + dataset_type) 

716 return out 

717 

718 

719def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5, (empty)

720 return_index=False): 

721 """ 

722 Attempts to find matching timestamps in 2 time-series that have an offset, are drifting, 

723 and are most likely incomplete: sizes don't have to match, some pulses may be missing