Coverage for ibllib/io/raw_data_loaders.py: 88%
450 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-08 17:16 +0100
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-08 17:16 +0100
1"""
2Raw Data Loader functions for PyBpod rig.
4Module contains one loader function per raw datafile.
5"""
6import re
7import json
8import logging
9import wave
10from collections import OrderedDict
11from datetime import datetime
12from pathlib import Path, PureWindowsPath
13from typing import Union
15from dateutil import parser as dateparser
16from packaging import version
17import numpy as np
18import pandas as pd
20from iblutil.io import jsonable
21from ibllib.io.video import assert_valid_label
22from ibllib.time import uncycle_pgts, convert_pgts, date2isostr
24_logger = logging.getLogger(__name__)
27def trial_times_to_times(raw_trial):
28 """
29 Parse and convert all trial timestamps to "absolute" time.
30 Float64 seconds from session start.
32 0---BpodStart---TrialStart0---------TrialEnd0-----TrialStart1---TrialEnd1...0---ts0---ts1---
33 tsN...absTS = tsN + TrialStartN - BpodStart
35 Bpod timestamps are in microseconds (µs)
36 PyBpod timestamps are is seconds (s)
38 :param raw_trial: raw trial data
39 :type raw_trial: dict
40 :return: trial data with modified timestamps
41 :rtype: dict
42 """
43 ts_bs = raw_trial['behavior_data']['Bpod start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
44 ts_ts = raw_trial['behavior_data']['Trial start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
45 # ts_te = raw_trial['behavior_data']['Trial end timestamp']
47 def convert(ts): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
48 return ts + ts_ts - ts_bs 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
50 converted_events = {} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
51 for k, v in raw_trial['behavior_data']['Events timestamps'].items(): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
52 converted_events.update({k: [convert(i) for i in v]}) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
53 raw_trial['behavior_data']['Events timestamps'] = converted_events 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
55 converted_states = {} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
56 for k, v in raw_trial['behavior_data']['States timestamps'].items(): 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
57 converted_states.update({k: [[convert(i) for i in x] for x in v]}) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
58 raw_trial['behavior_data']['States timestamps'] = converted_states 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
60 shift = raw_trial['behavior_data']['Bpod start timestamp'] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
61 raw_trial['behavior_data']['Bpod start timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
62 raw_trial['behavior_data']['Trial start timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
63 raw_trial['behavior_data']['Trial end timestamp'] -= shift 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
64 assert raw_trial['behavior_data']['Bpod start timestamp'] == 0 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
65 return raw_trial 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
68def load_bpod(session_path, task_collection='raw_behavior_data'):
69 """
70 Load both settings and data from bpod (.json and .jsonable)
72 :param session_path: Absolute path of session folder
73 :param task_collection: Collection within sesison path with behavior data
74 :return: dict settings and list of dicts data
75 """
76 return load_settings(session_path, task_collection), load_data(session_path, task_collection) 1b=?@c'pqa%
79def load_data(session_path: Union[str, Path], task_collection='raw_behavior_data', time='absolute'):
80 """
81 Load PyBpod data files (.jsonable).
83 Bpod timestamps are in microseconds (µs)
84 PyBpod timestamps are is seconds (s)
86 :param session_path: Absolute path of session folder
87 :type session_path: str, Path
88 :return: A list of len ntrials each trial being a dictionary
89 :rtype: list of dicts
90 """
91 if session_path is None: 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
92 _logger.warning('No data loaded: session_path is None')
93 return
94 path = Path(session_path).joinpath(task_collection) 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
95 path = next(path.glob('_iblrig_taskData.raw*.jsonable'), None) 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
96 if not path: 1bfUVWXYJZ01234T5678K9eL=?@whMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
97 _logger.warning('No data loaded: could not find raw data file') 1=?@
98 return None 1=?@
99 data = jsonable.read(path) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
100 if time == 'absolute': 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
101 data = [trial_times_to_times(t) for t in data] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
102 return data 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBni'tlupqNa%(grvd
105def load_camera_frameData(session_path, camera: str = 'left', raw: bool = False) -> pd.DataFrame:
106 """Loads binary frame data from Bonsai camera recording workflow.
108 Args:
109 session_path (StrPath): Path to session folder
110 camera (str, optional): Load FramsData for specific camera. Defaults to 'left'.
111 raw (bool, optional): Whether to return raw or parsed data. Defaults to False.
113 Returns:
114 parsed: (raw=False, Default)
115 pandas.DataFrame: 4 int64 columns: {
116 Timestamp, # float64 (seconds from session start)
117 embeddedTimeStamp, # float64 (seconds from session start)
118 embeddedFrameCounter, # int64 (Frame number from session start)
119 embeddedGPIOPinState # object (State of each of the 4 GPIO pins as a
120 # list of numpy boolean arrays
121 # e.g. np.array([True, False, False, False])
122 }
123 raw:
124 pandas.DataFrame: 4 int64 columns: {
125 Timestamp, # UTC ticks from BehaviorPC
126 # (100's of ns since midnight 1/1/0001)
127 embeddedTimeStamp, # Camera timestamp (Needs unclycling and conversion)
128 embeddedFrameCounter, # Frame counter (int)
129 embeddedGPIOPinState # GPIO pin state integer representation of 4 pins
130 }
131 """
132 camera = assert_valid_label(camera) 2lb. E C
133 fpath = Path(session_path).joinpath("raw_video_data") 2lb. E C
134 fpath = next(fpath.glob(f"_iblrig_{camera}Camera.frameData*.bin"), None) 2lb. E C
135 assert fpath, f"{fpath}\nFile not Found: Could not find bin file for cam <{camera}>" 2lb. E C
136 rdata = np.fromfile(fpath, dtype=np.float64) 2lb. E C
137 assert rdata.size % 4 == 0, "Dimension mismatch: bin file length is not mod 4" 2lb. E C
138 rows = int(rdata.size / 4) 2lb. E C
139 data = np.reshape(rdata.astype(np.int64), (rows, 4)) 2lb. E C
140 df_dict = dict.fromkeys( 2lb. E C
141 ["Timestamp", "embeddedTimeStamp", "embeddedFrameCounter", "embeddedGPIOPinState"]
142 )
143 df = pd.DataFrame(data, columns=df_dict.keys()) 2lb. E C
144 if raw: 2lb. E C
145 return df 2lb
147 df_dict["Timestamp"] = (data[:, 0] - data[0, 0]) / 10_000_000 # in seconds from first frame 2lb. E C
148 camerats = uncycle_pgts(convert_pgts(data[:, 1])) 2lb. E C
149 df_dict["embeddedTimeStamp"] = camerats - camerats[0] # in seconds from first frame 2lb. E C
150 df_dict["embeddedFrameCounter"] = data[:, 2] - data[0, 2] # from start 2lb. E C
151 gpio = (np.right_shift(np.tile(data[:, 3], (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 2lb. E C
152 df_dict["embeddedGPIOPinState"] = [np.array(x) for x in gpio.tolist()] 2lb. E C
154 parsed_df = pd.DataFrame.from_dict(df_dict) 2lb. E C
155 return parsed_df 2lb. E C
158def load_camera_ssv_times(session_path, camera: str):
159 """
160 Load the bonsai frame and camera timestamps from Camera.timestamps.ssv
162 NB: For some sessions the frame times are in the first column, in others the order is reversed.
163 NB: If using the new bin file the bonsai_times is a float in seconds since first frame
164 :param session_path: Absolute path of session folder
165 :param camera: Name of the camera to load, e.g. 'left'
166 :return: array of datetimes, array of frame times in seconds
167 """
168 camera = assert_valid_label(camera) 2nbmbc R s D y obS E - C z A H a
169 video_path = Path(session_path).joinpath('raw_video_data') 2nbmbc R s D y obS E - C z A H a
170 if next(video_path.glob(f'_iblrig_{camera}Camera.frameData*.bin'), None): 2nbmbc R s D y obS E - C z A H a
171 df = load_camera_frameData(session_path, camera=camera) 1EC
172 return df['Timestamp'].values, df['embeddedTimeStamp'].values 1EC
174 file = next(video_path.glob(f'_iblrig_{camera.lower()}Camera.timestamps*.ssv'), None) 2nbmbc R s D y obS E - C z A H a
175 if not file: 2nbmbc R s D y obS E - C z A H a
176 file = str(video_path.joinpath(f'_iblrig_{camera.lower()}Camera.timestamps.ssv')) 2mb
177 raise FileNotFoundError(file + ' not found') 2mb
178 # NB: Numpy has deprecated support for non-naive timestamps.
179 # Converting them is extremely slow: 6000 timestamps takes 0.8615s vs 0.0352s.
180 # from datetime import timezone
181 # c = {0: lambda x: datetime.fromisoformat(x).astimezone(timezone.utc).replace(tzinfo=None)}
183 # Determine the order of the columns by reading one line and testing whether the first value
184 # is an integer or not.
185 with open(file, 'r') as f: 2nbmbc R s D y obS E - C z A H a
186 line = f.readline() 2nbmbc R s D y obS E - C z A H a
187 type_map = OrderedDict(bonsai='<M8[ns]', camera='<u4') 2nbmbc R s D y obS E - C z A H a
188 try: 2nbmbc R s D y obS E - C z A H a
189 int(line.split(' ')[1]) 2nbmbc R s D y obS E - C z A H a
190 except ValueError: 2mbc s y z A H a
191 type_map.move_to_end('bonsai') 2mbc s y z A H a
192 ssv_params = dict(names=type_map.keys(), dtype=','.join(type_map.values()), delimiter=' ') 2nbmbc R s D y obS E - C z A H a
193 ssv_times = np.genfromtxt(file, **ssv_params) # np.loadtxt is slower for some reason 2nbmbc R s D y obS E - C z A H a
194 bonsai_times = ssv_times['bonsai'] 2nbmbc R s D y obS E - C z A H a
195 camera_times = uncycle_pgts(convert_pgts(ssv_times['camera'])) 2nbmbc R s D y obS E - C z A H a
196 return bonsai_times, camera_times 2nbmbc R s D y obS E - C z A H a
199def load_embedded_frame_data(session_path, label: str, raw=False):
200 """
201 Load the embedded frame count and GPIO for a given session. If the file doesn't exist,
202 or is empty, None values are returned.
203 :param session_path: Absolute path of session folder
204 :param label: The specific video to load, one of ('left', 'right', 'body')
205 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is
206 returned starting from 0 and the GPIO is returned as a dict of indices
207 :return: The frame count, GPIO
208 """
209 count = load_camera_frame_count(session_path, label, raw=raw) 2] c F kb, G R s D y S E - C z A H a
210 gpio = load_camera_gpio(session_path, label, as_dicts=not raw) 2] c F kb, G R s D y S E - C z A H a
211 return count, gpio 2] c F kb, G R s D y S E - C z A H a
214def load_camera_frame_count(session_path, label: str, raw=True):
215 """
216 Load the embedded frame count for a given session. If the file doesn't exist, or is empty,
217 a None value is returned.
218 :param session_path: Absolute path of session folder
219 :param label: The specific video to load, one of ('left', 'right', 'body')
220 :param raw: If True the raw data are returned without preprocessing, otherwise frame count is
221 returned starting from 0
222 :return: The frame count
223 """
224 if session_path is None: 2pb] c F kb, G R s D y S E - C z A H a
225 return 2pb
227 label = assert_valid_label(label) 2pb] c F kb, G R s D y S E - C z A H a
228 video_path = Path(session_path).joinpath('raw_video_data') 2pb] c F kb, G R s D y S E - C z A H a
229 if next(video_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), None): 2pb] c F kb, G R s D y S E - C z A H a
230 df = load_camera_frameData(session_path, camera=label) 1EC
231 return df['embeddedFrameCounter'].values 1EC
233 # Load frame count
234 glob = video_path.glob(f'_iblrig_{label}Camera.frame_counter*.bin') 2pb] c F kb, G R s D y S E - C z A H a
235 count_file = next(glob, None) 2pb] c F kb, G R s D y S E - C z A H a
236 count = np.fromfile(count_file, dtype=np.float64).astype(int) if count_file else [] 2pb] c F kb, G R s D y S E - C z A H a
237 if len(count) == 0: 2pb] c F kb, G R s D y S E - C z A H a
238 return 2pbc F kb, G s a
239 if not raw: 2pb] R D y S E - C z A H
240 count -= count[0] # start from zero 2pb] R y S - C z A H
241 return count 2pb] R D y S E - C z A H
244def load_camera_gpio(session_path, label: str, as_dicts=False):
245 """
246 Load the GPIO for a given session. If the file doesn't exist, or is empty, a None value is
247 returned.
249 The raw binary file contains uint32 values (saved as doubles) where the first 4 bits
250 represent the state of each of the 4 GPIO pins. The array is expanded to an n x 4 array by
251 shifting each bit to the end and checking whether it is 0 (low state) or 1 (high state).
253 :param session_path: Absolute path of session folder
254 :param label: The specific video to load, one of ('left', 'right', 'body')
255 :param as_dicts: If False the raw data are returned boolean array with shape (n_frames, n_pins)
256 otherwise GPIO is returned as a list of dictionaries with keys ('indices', 'polarities').
257 :return: An nx4 boolean array where columns represent state of GPIO pins 1-4.
258 If as_dicts is True, a list of dicts is returned with keys ('indices', 'polarities'),
259 or None if the dictionary is empty.
260 """
261 if session_path is None: 2. ] c F kb, G R s D y S E - C z A H a
262 return 1.
263 raw_path = Path(session_path).joinpath('raw_video_data') 2. ] c F kb, G R s D y S E - C z A H a
264 label = assert_valid_label(label) 2. ] c F kb, G R s D y S E - C z A H a
266 # Load pin state
267 if next(raw_path.glob(f'_iblrig_{label}Camera.frameData*.bin'), False): 2. ] c F kb, G R s D y S E - C z A H a
268 df = load_camera_frameData(session_path, camera=label, raw=False) 1.EC
269 gpio = np.array([x for x in df['embeddedGPIOPinState'].values]) 1.EC
270 if len(gpio) == 0: 1.EC
271 return [None] * 4 if as_dicts else None
272 else:
273 GPIO_file = next(raw_path.glob(f'_iblrig_{label}Camera.GPIO*.bin'), None) 2. ] c F kb, G R s D y S - C z A H a
274 # This deals with missing and empty files the same
275 gpio = np.fromfile(GPIO_file, dtype=np.float64).astype(np.uint32) if GPIO_file else [] 2. ] c F kb, G R s D y S - C z A H a
276 # Check values make sense (4 pins = 16 possible values)
277 if not np.isin(gpio, np.left_shift(np.arange(2 ** 4, dtype=np.uint32), 32 - 4)).all(): 2. ] c F kb, G R s D y S - C z A H a
278 _logger.warning('Unexpected GPIO values; decoding may fail') 1.
279 if len(gpio) == 0: 2. ] c F kb, G R s D y S - C z A H a
280 return [None] * 4 if as_dicts else None 2. c F kb, G s a
281 # 4 pins represented as uint32
282 # For each pin, shift its bit to the end and check the bit is set
283 gpio = (np.right_shift(np.tile(gpio, (4, 1)).T, np.arange(31, 27, -1)) & 0x1) == 1 1.]RDyS-CzAH
285 if as_dicts: 1.]RDySE-CzAH
286 if not gpio.any(): 1.]RySE-CzAH
287 _logger.error('No GPIO changes') 1.
288 return [None] * 4 1.
289 # Find state changes for each pin and construct a dict of indices and polarities for each
290 edges = np.vstack((gpio[0, :], np.diff(gpio.astype(int), axis=0))) 1.]RySE-CzAH
291 # gpio = [(ind := np.where(edges[:, i])[0], edges[ind, i]) for i in range(4)]
292 # gpio = [dict(zip(('indices', 'polarities'), x)) for x in gpio_] # py3.8
293 gpio = [{'indices': np.where(edges[:, i])[0], 1.]RySE-CzAH
294 'polarities': edges[edges[:, i] != 0, i]}
295 for i in range(4)]
296 # Replace empty dicts with None
297 gpio = [None if x['indices'].size == 0 else x for x in gpio] 1.]RySE-CzAH
299 return gpio 1.]RDySE-CzAH
302def _read_settings_json_compatibility_enforced(settings):
303 """
304 Patch iblrig settings for compatibility across rig versions.
306 Parameters
307 ----------
308 settings : pathlib.Path, dict
309 Either a _iblrig_taskSettings.raw.json file path or the loaded settings.
311 Returns
312 -------
313 dict
314 The task settings patched for compatibility.
315 """
316 if isinstance(settings, dict): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
317 md = settings.copy()
318 else:
319 with open(settings) as js: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
320 md = json.load(js) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
321 if 'IS_MOCK' not in md: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
322 md['IS_MOCK'] = False 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ ~ ab) * + bbcbdbw h M o P j k Q # F , G s y ! z A m n i O ' t l u ` p q N a % ( g r v d
323 # Many v < 8 sessions had both version and version tag keys. v > 8 have a version tag.
324 # Some sessions have neither key. From v8 onwards we will use IBLRIG_VERSION to test rig
325 # version, however some places may still use the version tag.
326 if 'IBLRIG_VERSION_TAG' not in md.keys(): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
327 md['IBLRIG_VERSION_TAG'] = md.get('IBLRIG_VERSION', '') 2_ = ? @ ~ ab) * + bbcbdbQ i
328 if 'IBLRIG_VERSION' not in md.keys(): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
329 md['IBLRIG_VERSION'] = md['IBLRIG_VERSION_TAG'] 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} [ ~ ab) * + bbcbdbw h M o P ebx j k Q # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d
330 elif all([md['IBLRIG_VERSION'], md['IBLRIG_VERSION_TAG']]): 1_=?@ciO
331 # This may not be an issue; not sure what the intended difference between these keys was
332 assert md['IBLRIG_VERSION'] == md['IBLRIG_VERSION_TAG'], 'version and version tag mismatch' 1_=?@ciO
333 # Test version can be parsed. If not, log an error and set the version to nothing
334 try: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
335 version.parse(md['IBLRIG_VERSION'] or '0') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
336 except version.InvalidVersion as ex:
337 _logger.error('%s in iblrig settings, this may affect extraction', ex)
338 # try a more relaxed version parse
339 laxed_parse = re.search(r'^\d+\.\d+\.\d+', md['IBLRIG_VERSION'])
340 # Set the tag as the invalid version
341 md['IBLRIG_VERSION_TAG'] = md['IBLRIG_VERSION']
342 # overwrite version with either successfully parsed one or an empty string
343 md['IBLRIG_VERSION'] = laxed_parse.group() if laxed_parse else ''
344 if 'device_sound' not in md: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
345 # sound device must be defined in version 8 and later # FIXME this assertion will cause tests to break
346 assert version.parse(md['IBLRIG_VERSION'] or '0') < version.parse('8.0.0') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d
347 # in v7 we must infer the device from the sampling frequency if SD is None
348 if 'sounddevice' in md.get('SD', ''): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d
349 device = 'xonar' 1bwoPjk#FGy!zAO'a%gd
350 else:
351 freq_map = {192000: 'xonar', 96000: 'harp', 44100: 'sysdefault'} 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbh M ebx , R s D S E fbgbhbm B n t l u ` p q N a % ( g r v d
352 device = freq_map.get(md.get('SOUND_SAMPLE_FREQ'), 'unknown') 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbh M ebx , R s D S E fbgbhbm B n t l u ` p q N a % ( g r v d
353 md['device_sound'] = {'OUTPUT': device} 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k # F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d
354 # 2018-12-05 Version 3.2.3 fixes (permanent fixes in IBL_RIG from 3.2.4 on)
355 if md['IBLRIG_VERSION'] == '': 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
356 pass 2T jb~ ab) * + bbcbdbQ # m d
357 elif version.parse(md['IBLRIG_VERSION']) >= version.parse('8.0.0'): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ w h M o P ebx j k c F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
358 md['SESSION_NUMBER'] = str(md['SESSION_NUMBER']).zfill(3) 1i
359 md['PYBPOD_BOARD'] = md['RIG_NAME'] 1i
360 md['PYBPOD_CREATOR'] = (md['ALYX_USER'], '') 1i
361 md['SESSION_DATE'] = md['SESSION_START_TIME'][:10] 1i
362 md['SESSION_DATETIME'] = md['SESSION_START_TIME'] 1i
363 elif version.parse(md['IBLRIG_VERSION']) <= version.parse('3.2.3'): 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | } _ [ = ? @ w h M o P ebx j k c F , G R s D y S E ! z A fbgbhbm B n O ' t l u ` p q N a % ( g r v d
364 if 'LAST_TRIAL_DATA' in md.keys(): 1a
365 md.pop('LAST_TRIAL_DATA') 1a
366 if 'weighings' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1a
367 md['PYBPOD_SUBJECT_EXTRA'].pop('weighings') 1a
368 if 'water_administration' in md['PYBPOD_SUBJECT_EXTRA'].keys(): 1a
369 md['PYBPOD_SUBJECT_EXTRA'].pop('water_administration') 1a
370 if 'IBLRIG_COMMIT_HASH' not in md.keys(): 1a
371 md['IBLRIG_COMMIT_HASH'] = 'f9d8905647dbafe1f9bdf78f73b286197ae2647b' 1a
372 # parse the date format to Django supported ISO
373 dt = dateparser.parse(md['SESSION_DATETIME']) 1a
374 md['SESSION_DATETIME'] = date2isostr(dt) 1a
375 # add the weight key if it doesn't already exist
376 if 'SUBJECT_WEIGHT' not in md: 1a
377 md['SUBJECT_WEIGHT'] = None 1a
378 return md 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
381def load_settings(session_path: Union[str, Path], task_collection='raw_behavior_data'):
382 """
383 Load PyBpod Settings files (.json).
385 [description]
387 :param session_path: Absolute path of session folder
388 :type session_path: str, Path
389 :return: Settings dictionary
390 :rtype: dict
391 """
392 if session_path is None: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
393 _logger.warning("No data loaded: session_path is None") 1[
394 return 1[
395 path = Path(session_path).joinpath(task_collection) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
396 path = next(path.glob("_iblrig_taskSettings.raw*.json"), None) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
397 if not path: 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
398 _logger.warning("No data loaded: could not find raw settings file") 1^[=O`
399 return None 1^[=O`
400 settings = _read_settings_json_compatibility_enforced(path) 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
401 return settings 2b f U V W X Y J Z 0 1 2 3 4 T 5 6 7 8 K 9 $ I e L ^ | jb} _ [ = ? @ ~ ab) * + bbcbdbw h M o P ebx j k c Q # F , G R s D y S E ! z A fbgbhbm B n i O ' t l u ` p q N a % ( g r v d
404def load_stim_position_screen(session_path, task_collection='raw_behavior_data'):
405 path = Path(session_path).joinpath(task_collection)
406 path = next(path.glob("_iblrig_stimPositionScreen.raw*.csv"), None)
408 data = pd.read_csv(path, sep=',', header=None, on_bad_lines='skip')
409 data.columns = ['contrast', 'position', 'bns_ts']
410 data['bns_ts'] = pd.to_datetime(data['bns_ts'])
411 return data
414def load_encoder_events(session_path, task_collection='raw_behavior_data', settings=False):
415 """
416 Load Rotary Encoder (RE) events raw data file.
418 Assumes that a folder called "raw_behavior_data" exists in folder.
420 Events number correspond to following bpod states:
421 1: correct / hide_stim
422 2: stim_on
423 3: closed_loop
424 4: freeze_error / freeze_correct
426 >>> data.columns
427 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'
428 'sm_ev', # State Machine Event 'numpy.int64'
429 'bns_ts'] # Bonsai Timestamp (int) 'pandas.Timestamp'
430 # pd.to_datetime(data.bns_ts) to work in datetimes
432 :param session_path: [description]
433 :type session_path: [type]
434 :return: dataframe w/ 3 cols and (ntrials * 3) lines
435 :rtype: Pandas.DataFrame
436 """
437 if session_path is None: 1bf$ewhoxjkcmBnitlupqagrvd
438 return
439 path = Path(session_path).joinpath(task_collection) 1bf$ewhoxjkcmBnitlupqagrvd
440 path = next(path.glob("_iblrig_encoderEvents.raw*.ssv"), None) 1bf$ewhoxjkcmBnitlupqagrvd
441 if not settings: 1bf$ewhoxjkcmBnitlupqagrvd
442 settings = load_settings(session_path, task_collection=task_collection) 1bf$ewhoxjkcmBnitlupqagrvd
443 if settings is None or not settings.get('IBLRIG_VERSION'): 1bf$ewhoxjkcmBnitlupqagrvd
444 settings = {'IBLRIG_VERSION': '100.0.0'} 1d
445 # auto-detect old files when version is not labeled
446 with open(path) as fid: 1d
447 line = fid.readline() 1d
448 if line.startswith('Event') and 'StateMachine' in line: 1d
449 settings = {'IBLRIG_VERSION': '0.0.0'} 1d
450 if not path: 1bf$ewhoxjkcmBnitlupqagrvd
451 return None
452 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 1bf$ewhoxjkcmBnitlupqagrvd
453 return _load_encoder_events_file_ge5(path) 1f$ewhoxjkmBnitluagrvd
454 else:
455 return _load_encoder_events_file_lt5(path) 1bf$ecpqagd
458def _load_encoder_ssv_file(file_path, **kwargs):
459 file_path = Path(file_path) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
460 if file_path.stat().st_size == 0: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
461 _logger.error(f"{file_path.name} is an empty file. ")
462 raise ValueError(f"{file_path.name} is an empty file. ABORT EXTRACTION. ")
463 return pd.read_csv(file_path, sep=' ', header=None, on_bad_lines='skip', **kwargs) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
466def _load_encoder_positions_file_lt5(file_path):
467 """
468 File loader without the session overhead
469 :param file_path:
470 :return: dataframe of encoder events
471 """
472 data = _load_encoder_ssv_file(file_path, 1b;{fIe/cpqagd
473 names=['_', 're_ts', 're_pos', 'bns_ts', '__'],
474 usecols=['re_ts', 're_pos', 'bns_ts'])
475 return _groom_wheel_data_lt5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 1b;{fIe/cpqagd
478def _load_encoder_positions_file_ge5(file_path):
479 """
480 File loader without the session overhead
481 :param file_path:
482 :return: dataframe of encoder events
483 """
484 data = _load_encoder_ssv_file(file_path, 1;fIe/whoxjkmBnitluagrvd
485 names=['re_ts', 're_pos', '_'],
486 usecols=['re_ts', 're_pos'])
487 return _groom_wheel_data_ge5(data, label='_iblrig_encoderPositions.raw.ssv', path=file_path) 1;fIe/whoxjkmBnitluagrvd
490def _load_encoder_events_file_lt5(file_path):
491 """
492 File loader without the session overhead
493 :param file_path:
494 :return: dataframe of encoder events
495 """
496 data = _load_encoder_ssv_file(file_path, 1bf$e:cpqagd
497 names=['_', 're_ts', '__', 'sm_ev', 'bns_ts', '___'],
498 usecols=['re_ts', 'sm_ev', 'bns_ts'])
499 return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 1bf$e:cpqagd
502def _load_encoder_events_file_ge5(file_path):
503 """
504 File loader without the session overhead
505 :param file_path:
506 :return: dataframe of encoder events
507 """
508 data = _load_encoder_ssv_file(file_path, 1f$e:whoxjkmBnitluagrvd
509 names=['re_ts', 'sm_ev', '_'],
510 usecols=['re_ts', 'sm_ev'])
511 return _groom_wheel_data_ge5(data, label='_iblrig_encoderEvents.raw.ssv', path=file_path) 1f$e:whoxjkmBnitluagrvd
514def load_encoder_positions(session_path, task_collection='raw_behavior_data', settings=False):
515 """
516 Load Rotary Encoder (RE) positions from raw data file within a session path.
518 Assumes that a folder called "raw_behavior_data" exists in folder.
519 Positions are RE ticks [-512, 512] == [-180º, 180º]
520 0 == trial stim init position
521 Positive nums are rightwards movements (mouse) or RE CW (mouse)
523 Variable line number, depends on movements.
525 Raw datafile Columns:
526 Position, RE timestamp, RE Position, Bonsai Timestamp
528 Position is always equal to 'Position' so this column was dropped.
530 >>> data.columns
531 >>> ['re_ts', # Rotary Encoder Timestamp (ms) 'numpy.int64'
532 're_pos', # Rotary Encoder position (ticks) 'numpy.int64'
533 'bns_ts'] # Bonsai Timestamp 'pandas.Timestamp'
534 # pd.to_datetime(data.bns_ts) to work in datetimes
536 :param session_path: Absolute path of session folder
537 :type session_path: str
538 :return: dataframe w/ 3 cols and N positions
539 :rtype: Pandas.DataFrame
540 """
541 if session_path is None: 1bfIewhoxjkcsmBnitlupqagrvd
542 return
543 path = Path(session_path).joinpath(task_collection) 1bfIewhoxjkcsmBnitlupqagrvd
544 path = next(path.glob("_iblrig_encoderPositions.raw*.ssv"), None) 1bfIewhoxjkcsmBnitlupqagrvd
545 if not settings: 1bfIewhoxjkcsmBnitlupqagrvd
546 settings = load_settings(session_path, task_collection=task_collection) 1bfIewhoxjkcsmBnitlupqagrvd
547 if settings is None or not settings.get('IBLRIG_VERSION'): 1bfIewhoxjkcsmBnitlupqagrvd
548 settings = {'IBLRIG_VERSION': '100.0.0'} 1d
549 # auto-detect old files when version is not labeled
550 with open(path) as fid: 1d
551 line = fid.readline() 1d
552 if line.startswith('Position'): 1d
553 settings = {'IBLRIG_VERSION': '0.0.0'} 1d
554 if not path: 1bfIewhoxjkcsmBnitlupqagrvd
555 _logger.warning("No data loaded: could not find raw encoderPositions file") 1sa
556 return None 1sa
557 if version.parse(settings['IBLRIG_VERSION']) >= version.parse('5.0.0'): 1bfIewhoxjkcmBnitlupqagrvd
558 return _load_encoder_positions_file_ge5(path) 1fIewhoxjkmBnitluagrvd
559 else:
560 return _load_encoder_positions_file_lt5(path) 1bfIecpqagd
563def load_encoder_trial_info(session_path, task_collection='raw_behavior_data'):
564 """
565 Load Rotary Encoder trial info from raw data file.
567 Assumes that a folder calles "raw_behavior_data" exists in folder.
569 NOTE: Last trial probably inexistent data (Trial info is sent on trial start
570 and data is only saved on trial exit...) max(trialnum) should be N+1 if N
571 is the amount of trial data saved.
573 Raw datafile Columns:
575 >>> data.columns
576 >>> ['trial_num', # Trial Number 'numpy.int64'
577 'stim_pos_init', # Initial position of visual stim 'numpy.int64'
578 'stim_contrast', # Contrast of visual stimulus 'numpy.float64'
579 'stim_freq', # Frequency of gabor patch 'numpy.float64'
580 'stim_angle', # Angle of Gabor 0 = Vertical 'numpy.float64'
581 'stim_gain', # Wheel gain (mm/º of stim) 'numpy.float64'
582 'stim_sigma', # Size of patch 'numpy.float64'
583 'stim_phase', # Phase of gabor 'numpy.float64'
584 'bns_ts' ] # Bonsai Timestamp 'pandas.Timestamp'
585 # pd.to_datetime(data.bns_ts) to work in datetimes
587 :param session_path: Absoulte path of session folder
588 :type session_path: str
589 :return: dataframe w/ 9 cols and ntrials lines
590 :rtype: Pandas.DataFrame
591 """
592 if session_path is None: 2qb
593 return 2qb
594 path = Path(session_path).joinpath(task_collection) 2qb
595 path = next(path.glob("_iblrig_encoderTrialInfo.raw*.ssv"), None) 2qb
596 if not path: 2qb
597 return None 2qb
598 data = pd.read_csv(path, sep=' ', header=None) 2qb
599 data = data.drop([9], axis=1) 2qb
600 data.columns = ['trial_num', 'stim_pos_init', 'stim_contrast', 'stim_freq', 2qb
601 'stim_angle', 'stim_gain', 'stim_sigma', 'stim_phase', 'bns_ts']
602 # return _groom_wheel_data_lt5(data, label='_iblrig_encoderEvents.raw.ssv', path=path)
603 return data 2qb
606def load_ambient_sensor(session_path, task_collection='raw_behavior_data'):
607 """
608 Load Ambient Sensor data from session.
610 Probably could be extracted to DatasetTypes:
611 _ibl_trials.temperature_C, _ibl_trials.airPressure_mb,
612 _ibl_trials.relativeHumidity
613 Returns a list of dicts one dict per trial.
614 dict keys are:
615 dict_keys(['Temperature_C', 'AirPressure_mb', 'RelativeHumidity'])
617 :param session_path: Absoulte path of session folder
618 :type session_path: str
619 :return: list of dicts
620 :rtype: list
621 """
622 if session_path is None:
623 return
624 path = Path(session_path).joinpath(task_collection)
625 path = next(path.glob("_iblrig_ambientSensorData.raw*.jsonable"), None)
626 if not path:
627 return None
628 data = []
629 with open(path, 'r') as f:
630 for line in f:
631 data.append(json.loads(line))
632 return data
635def load_mic(session_path, task_collection='raw_behavior_data'):
636 """
637 Load Microphone wav file to np.array of len nSamples
639 :param session_path: Absolute path of session folder
640 :type session_path: str
641 :return: An array of values of the sound waveform
642 :rtype: numpy.array
643 """
644 if session_path is None:
645 return
646 path = Path(session_path).joinpath(task_collection)
647 path = next(path.glob("_iblrig_micData.raw*.wav"), None)
648 if not path:
649 return None
650 fp = wave.open(path)
651 nchan = fp.getnchannels()
652 N = fp.getnframes()
653 dstr = fp.readframes(N * nchan)
654 data = np.frombuffer(dstr, np.int16)
655 data = np.reshape(data, (-1, nchan))
656 return data
659def _clean_wheel_dataframe(data, label, path):
660 if np.any(data.isna()): 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
661 _logger.warning(label + ' has missing/incomplete records \n %s', path) 1b{fIe:/pqgd
662 # first step is to re-interpret as numeric objects if not already done
663 for col in data.columns: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
664 if data[col].dtype == object and col not in ['bns_ts']: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
665 data[col] = pd.to_numeric(data[col], errors='coerce') 1:d
666 # then drop Nans and duplicates
667 data.dropna(inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
668 data.drop_duplicates(keep='first', inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
669 data.reset_index(inplace=True) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
670 # handle the clock resets when microseconds exceed uint32 max value
671 drop_first = False 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
672 data['re_ts'] = data['re_ts'].astype(np.double, copy=False) 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
673 if any(np.diff(data['re_ts']) < 0): 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
674 ind = np.where(np.diff(data['re_ts']) < 0)[0] 1;{fIe:/hojkcmnilad
675 for i in ind: 1;{fIe:/hojkcmnilad
676 # the first sample may be corrupt, in this case throw away
677 if i <= 1: 1;{fIe:/hojkcmnilad
678 drop_first = i 1;Ie/ca
679 _logger.warning(label + ' rotary encoder positions timestamps' 1;Ie/ca
680 ' first sample corrupt ' + str(path))
681 # if it's an uint32 wraparound, the diff should be close to 2 ** 32
682 elif 32 - np.log2(data['re_ts'][i] - data['re_ts'][i + 1]) < 0.2: 1;{fIe:/hojkmnild
683 data.loc[i + 1:, 're_ts'] = data.loc[i + 1:, 're_ts'] + 2 ** 32 1{fIe/hojkmnd
684 # there is also the case where 2 positions are swapped and need to be swapped back
686 elif data['re_ts'][i] > data['re_ts'][i + 1] > data['re_ts'][i - 1]: 1;Ie:/hjkild
687 _logger.warning(label + ' rotary encoder timestamps swapped at index: ' + 1;Ie/hjkild
688 str(i) + ' ' + str(path))
689 a, b = data.iloc[i].copy(), data.iloc[i + 1].copy() 1;Ie/hjkild
690 data.iloc[i], data.iloc[i + 1] = b, a 1;Ie/hjkild
691 # if none of those 3 cases apply, raise an error
692 else:
693 _logger.error(label + ' Rotary encoder timestamps are not sorted.' + str(path)) 1:
694 data.sort_values('re_ts', inplace=True) 1:
695 data.reset_index(inplace=True) 1:
696 if drop_first is not False: 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
697 data.drop(data.loc[:drop_first].index, inplace=True) 1;Ie/ca
698 data = data.reindex() 1;Ie/ca
699 return data 1b;{f$Ie:/whoxjkcmBnitlupqagrvd
702def _groom_wheel_data_lt5(data, label='file ', path=''):
703 """
704 The whole purpose of this function is to account for variability and corruption in
705 the wheel position files. There are many possible errors described below, but
706 nothing excludes getting new ones.
707 """
708 data = _clean_wheel_dataframe(data, label, path) 1b;{f$Ie:/cpqagd
709 data.drop(data.loc[data.bns_ts.apply(len) != 33].index, inplace=True) 1b;{f$Ie:/cpqagd
710 # check if the time scale is in ms
711 sess_len_sec = (datetime.strptime(data['bns_ts'].iloc[-1][:25], '%Y-%m-%dT%H:%M:%S.%f') - 1b;{f$Ie:/cpqagd
712 datetime.strptime(data['bns_ts'].iloc[0][:25], '%Y-%m-%dT%H:%M:%S.%f')).seconds
713 if data['re_ts'].iloc[-1] / (sess_len_sec + 1e-6) < 1e5: # should be 1e6 normally 1b;{f$Ie:/cpqagd
714 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 1$e:ca
715 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')
716 data['re_ts'] = data['re_ts'] * 1000 1$e:ca
717 return data 1b;{f$Ie:/cpqagd
720def _groom_wheel_data_ge5(data, label='file ', path=''):
721 """
722 The whole purpose of this function is to account for variability and corruption in
723 the wheel position files. There are many possible errors described below, but
724 nothing excludes getting new ones.
725 """
726 data = _clean_wheel_dataframe(data, label, path) 1;f$Ie:/whoxjkmBnitluagrvd
727 # check if the time scale is in ms
728 if (data['re_ts'].iloc[-1] - data['re_ts'].iloc[0]) / 1e6 < 20: 1;f$Ie:/whoxjkmBnitluagrvd
729 _logger.warning('Rotary encoder reset logs events in ms instead of us: ' + 1:/grd
730 'RE firmware needs upgrading and wheel velocity is potentially inaccurate')
731 data['re_ts'] = data['re_ts'] * 1000 1:/grd
732 return data 1;f$Ie:/whoxjkmBnitluagrvd
735def sync_trials_robust(t0, t1, diff_threshold=0.001, drift_threshold_ppm=200, max_shift=5,
736 return_index=False):
737 """
738 Attempts to find matching timestamps in 2 time-series that have an offset, are drifting,
739 and are most likely incomplete: sizes don't have to match, some pulses may be missing
740 in any series.
741 Only works with irregular time series as it relies on the derivative to match sync.
742 :param t0:
743 :param t1:
744 :param diff_threshold:
745 :param drift_threshold_ppm: (150)
746 :param max_shift: (200)
747 :param return_index (False)
748 :return:
749 """
750 nsync = min(t0.size, t1.size) 2iba d
751 dt0 = np.diff(t0) 2iba d
752 dt1 = np.diff(t1) 2iba d
753 ind = np.zeros_like(dt0) * np.nan 2iba d
754 i0 = 0 2iba d
755 i1 = 0 2iba d
756 cdt = np.nan # the current time difference between the two series to compute drift 2iba d
757 while i0 < (nsync - 1): 2iba d
758 # look in the next max_shift events the ones whose derivative match
759 isearch = np.arange(i1, min(max_shift + i1, dt1.size)) 2iba d
760 dec = np.abs(dt0[i0] - dt1[isearch]) < diff_threshold 2iba d
761 # another constraint is to check the dt for the maximum drift
762 if ~np.isnan(cdt): 2iba d
763 drift_ppm = np.abs((cdt - (t0[i0] - t1[isearch])) / dt1[isearch]) * 1e6 2iba d
764 dec = np.logical_and(dec, drift_ppm <= drift_threshold_ppm) 2iba d
765 # if one is found
766 if np.any(dec): 2iba d
767 ii1 = np.where(dec)[0][0] 2iba d
768 ind[i0] = i1 + ii1 2iba d
769 i1 += ii1 + 1 2iba d
770 cdt = t0[i0 + 1] - t1[i1 + ii1] 2iba d
771 i0 += 1 2iba d
772 it0 = np.where(~np.isnan(ind))[0] 2iba d
773 it1 = ind[it0].astype(int) 2iba d
774 ind0 = np.unique(np.r_[it0, it0 + 1]) 2iba d
775 ind1 = np.unique(np.r_[it1, it1 + 1]) 2iba d
776 if return_index: 2iba d
777 return t0[ind0], t1[ind1], ind0, ind1
778 else:
779 return t0[ind0], t1[ind1] 2iba d
782def load_bpod_fronts(session_path: str, data: list = False, task_collection: str = 'raw_behavior_data') -> list:
783 """load_bpod_fronts
784 Loads BNC1 and BNC2 bpod channels times and polarities from session_path
786 :param session_path: a valid session_path
787 :type session_path: str
788 :param data: pre-loaded raw data dict, defaults to False
789 :type data: list, optional
790 :return: List of dicts BNC1 and BNC2 {"times": np.array, "polarities":np.array}
791 :rtype: list
792 """
793 if not data: 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
794 data = load_data(session_path, task_collection) 1ca
796 BNC1_fronts = np.array([[np.nan, np.nan]]) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
797 BNC2_fronts = np.array([[np.nan, np.nan]]) 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
798 for tr in data: 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
799 BNC1_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
800 BNC1_fronts,
801 np.array(
802 [
803 [x, 1]
804 for x in tr["behavior_data"]["Events timestamps"].get("BNC1High", [np.nan])
805 ]
806 ),
807 axis=0,
808 )
809 BNC1_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
810 BNC1_fronts,
811 np.array(
812 [
813 [x, -1]
814 for x in tr["behavior_data"]["Events timestamps"].get("BNC1Low", [np.nan])
815 ]
816 ),
817 axis=0,
818 )
819 BNC2_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
820 BNC2_fronts,
821 np.array(
822 [
823 [x, 1]
824 for x in tr["behavior_data"]["Events timestamps"].get("BNC2High", [np.nan])
825 ]
826 ),
827 axis=0,
828 )
829 BNC2_fronts = np.append( 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
830 BNC2_fronts,
831 np.array(
832 [
833 [x, -1]
834 for x in tr["behavior_data"]["Events timestamps"].get("BNC2Low", [np.nan])
835 ]
836 ),
837 axis=0,
838 )
840 BNC1_fronts = BNC1_fronts[1:, :] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
841 BNC1_fronts = BNC1_fronts[BNC1_fronts[:, 0].argsort()] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
842 BNC2_fronts = BNC2_fronts[1:, :] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
843 BNC2_fronts = BNC2_fronts[BNC2_fronts[:, 0].argsort()] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
845 BNC1 = {"times": BNC1_fronts[:, 0], "polarities": BNC1_fronts[:, 1]} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
846 BNC2 = {"times": BNC2_fronts[:, 0], "polarities": BNC2_fronts[:, 1]} 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
848 return [BNC1, BNC2] 1bfUVWXYJZ01234T5678K9eLwhMoPxjkcQ#FGsDyC!zAHmBnitlupqNagrv
851def get_port_events(trial: dict, name: str = '') -> list:
852 """get_port_events
853 Return all event timestamps from bpod raw data trial that match 'name'
854 --> looks in trial['behavior_data']['Events timestamps']
856 :param trial: raw trial dict
857 :type trial: dict
858 :param name: name of event, defaults to ''
859 :type name: str, optional
860 :return: Sorted list of event timestamps
861 :rtype: list
862 TODO: add polarities?
863 """
864 out: list = [] 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
865 events = trial['behavior_data']['Events timestamps'] 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
866 for k in events: 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
867 if name in k: 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
868 out.extend(events[k]) 1fJKeLwhMoPxjkcQmBnitluNagrv
869 out = sorted(out) 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
871 return out 1bfJKeLwhMoPxjkcQmBnitlupqNagrv
874def load_widefield_mmap(session_path, dtype=np.uint16, shape=(540, 640), n_frames=None, mode='r'):
875 """
876 TODO Document this function
878 Parameters
879 ----------
880 session_path
882 Returns
883 -------
885 """
886 filepath = Path(session_path).joinpath('raw_widefield_data').glob('widefield.raw.*.dat')
887 filepath = next(filepath, None)
888 if not filepath:
889 _logger.warning("No data loaded: could not find raw data file")
890 return None
892 if type(dtype) is str:
893 dtype = np.dtype(dtype)
895 if n_frames is None:
896 # Get the number of samples from the file size
897 n_frames = int(filepath.stat().st_size / (np.prod(shape) * dtype.itemsize))
899 return np.memmap(str(filepath), mode=mode, dtype=dtype, shape=(int(n_frames), *shape))
902def patch_settings(session_path, collection='raw_behavior_data',
903 new_collection=None, subject=None, number=None, date=None):
904 """Modify various details in a settings file.
906 This function makes it easier to change things like subject name in a settings as it will
907 modify the subject name in the myriad paths. NB: This saves the settings into the same location
908 it was loaded from.
910 Parameters
911 ----------
912 session_path : str, pathlib.Path
913 The session path containing the settings file.
914 collection : str
915 The subfolder containing the settings file.
916 new_collection : str
917 An optional new subfolder to change in the settings paths.
918 subject : str
919 An optional new subject name to change in the settings.
920 number : str, int
921 An optional new number to change in the settings.
922 date : str, datetime.date
923 An optional date to change in the settings.
925 Returns
926 -------
927 dict
928 The modified settings.
930 Examples
931 --------
932 File is in /data/subject/2020-01-01/002/raw_behavior_data. Patch the file then move to new location.
933 >>> patch_settings('/data/subject/2020-01-01/002', number='001')
934 >>> shutil.move('/data/subject/2020-01-01/002/raw_behavior_data/', '/data/subject/2020-01-01/001/raw_behavior_data/')
936 File is moved into new collection within the same session, then patched.
937 >>> shutil.move('./subject/2020-01-01/002/raw_task_data_00/', './subject/2020-01-01/002/raw_task_data_01/')
938 >>> patch_settings('/data/subject/2020-01-01/002', collection='raw_task_data_01', new_collection='raw_task_data_01')
940 Update subject, date and number.
941 >>> new_session_path = Path('/data/foobar/2024-02-24/002')
942 >>> old_session_path = Path('/data/baz/2024-02-23/001')
943 >>> patch_settings(old_session_path, collection='raw_task_data_00',
944 ... subject=new_session_path.parts[-3], date=new_session_path.parts[-2], number=new_session_path.parts[-1])
945 >>> shutil.move(old_session_path, new_session_path)
946 """
947 settings = load_settings(session_path, collection) 1b)*+O
948 if not settings: 1b)*+O
949 raise IOError('Settings file not found') 1O
951 filename = PureWindowsPath(settings.get('SETTINGS_FILE_PATH', '_iblrig_taskSettings.raw.json')).name 1b)*+O
952 file_path = Path(session_path).joinpath(collection, filename) 1b)*+O
954 if subject: 1b)*+O
955 # Patch subject name
956 old_subject = settings['SUBJECT_NAME'] 1bO
957 settings['SUBJECT_NAME'] = subject 1bO
958 for k in settings.keys(): 1bO
959 if isinstance(settings[k], str): 1bO
960 settings[k] = settings[k].replace(f'\\Subjects\\{old_subject}', f'\\Subjects\\{subject}') 1bO
961 if 'SESSION_NAME' in settings: 1bO
962 settings['SESSION_NAME'] = '\\'.join([subject, *settings['SESSION_NAME'].split('\\')[1:]]) 1bO
963 settings.pop('PYBPOD_SUBJECT_EXTRA', None) # Get rid of Alyx subject info 1bO
965 if date: 1b)*+O
966 # Patch session datetime
967 date = str(date) 1bO
968 old_date = settings['SESSION_DATE'] 1bO
969 settings['SESSION_DATE'] = date 1bO
970 for k in settings.keys(): 1bO
971 if isinstance(settings[k], str): 1bO
972 settings[k] = settings[k].replace( 1bO
973 f'\\{settings["SUBJECT_NAME"]}\\{old_date}',
974 f'\\{settings["SUBJECT_NAME"]}\\{date}'
975 )
976 settings['SESSION_DATETIME'] = date + settings['SESSION_DATETIME'][10:] 1bO
977 if 'SESSION_END_TIME' in settings: 1bO
978 settings['SESSION_END_TIME'] = date + settings['SESSION_END_TIME'][10:] 1O
979 if 'SESSION_START_TIME' in settings: 1bO
980 settings['SESSION_START_TIME'] = date + settings['SESSION_START_TIME'][10:] 1O
982 if number: 1b)*+O
983 # Patch session number
984 old_number = settings['SESSION_NUMBER'] 1b)*+O
985 if isinstance(number, int): 1b)*+O
986 number = f'{number:03}' 1O
987 settings['SESSION_NUMBER'] = number 1b)*+O
988 for k in settings.keys(): 1b)*+O
989 if isinstance(settings[k], str): 1b)*+O
990 settings[k] = settings[k].replace( 1b)*+O
991 f'\\{settings["SESSION_DATE"]}\\{old_number}',
992 f'\\{settings["SESSION_DATE"]}\\{number}'
993 )
995 if new_collection: 1b)*+O
996 if 'SESSION_RAW_DATA_FOLDER' not in settings: 1)*+O
997 _logger.warning('SESSION_RAW_DATA_FOLDER key not in settings; collection not updated') 1O
998 else:
999 old_path = settings['SESSION_RAW_DATA_FOLDER'] 1)*+O
1000 new_path = PureWindowsPath(settings['SESSION_RAW_DATA_FOLDER']).with_name(new_collection) 1)*+O
1001 for k in settings.keys(): 1)*+O
1002 if isinstance(settings[k], str): 1)*+O
1003 settings[k] = settings[k].replace(old_path, str(new_path)) 1)*+O
1004 with open(file_path, 'w') as fp: 1b)*+O
1005 json.dump(settings, fp, indent=' ') 1b)*+O
1006 return settings 1b)*+O