Coverage for ibllib/tests/fixtures/utils.py: 96%
139 statements
« prev ^ index » next coverage.py v7.5.4, created at 2024-07-08 17:16 +0100
1#!/usr/bin/env python
2# -*- coding:utf-8 -*-
3# @Author: Niccolò Bonacchi
4# @Date: Friday, October 9th 2020, 12:02:56 pm
5import json
6import random
7import string
8import logging
9from pathlib import Path
11from one.registration import RegistrationClient
13from ibllib.io import session_params
15_logger = logging.getLogger(__name__)
18def create_fake_session_folder(
19 root_data_path, lab="fakelab", mouse="fakemouse", date="1900-01-01", num="001", increment=True
20):
21 root_data_path = Path(root_data_path) 1adbefcgh
22 session_path = root_data_path / lab / "Subjects" / mouse / date / num 1adbefcgh
23 if session_path.exists() and increment: 1adbefcgh
24 num = str(int(num) + 1).zfill(3) 1abc
25 return create_fake_session_folder( 1abc
26 root_data_path, lab=lab, mouse=mouse, date=date, num=num, increment=increment
27 )
28 session_path = root_data_path / lab / "Subjects" / mouse / date / num 1adbefcgh
30 session_path.mkdir(exist_ok=True, parents=True) 1adbefcgh
31 return session_path 1adbefcgh
34def create_fake_raw_ephys_data_folder(session_path, populate=True):
35 """create_fake_raw_ephys_data_folder creates raw_ephys_data folder
36 can populate with empty files with expected names
38 :param session_path: [description]
39 :type session_path: [type]
40 :param populate: [description], defaults to True
41 :type populate: bool, optional
42 :return: [description]
43 :rtype: [type]
44 """
45 session_path = Path(session_path) 1ai
46 raw_ephys_data_path = session_path / "raw_ephys_data" 1ai
47 raw_ephys_data_path.mkdir(exist_ok=True, parents=True) 1ai
48 if populate: 1ai
49 file_list = [ 1ai
50 "probe00/_spikeglx_ephysData_g0_t0.imec0.sync.npy",
51 "probe00/_spikeglx_ephysData_g0_t0.imec0.lf.meta",
52 "probe00/_spikeglx_ephysData_g0_t0.imec0.ap.cbin",
53 "probe00/_spikeglx_ephysData_g0_t0.imec0.timestamps.npy",
54 "probe00/_spikeglx_ephysData_g0_t0.imec0.lf.cbin",
55 "probe00/_spikeglx_ephysData_g0_t0.imec0.ap.ch",
56 "probe00/_spikeglx_ephysData_g0_t0.imec0.wiring.json",
57 "probe00/_spikeglx_sync.times.probe00.npy",
58 "probe00/_spikeglx_sync.channels.probe00.npy",
59 "probe00/_spikeglx_sync.polarities.probe00.npy",
60 "probe01/_spikeglx_ephysData_g0_t0.imec1.sync.npy",
61 "probe01/_spikeglx_ephysData_g0_t0.imec1.lf.meta",
62 "probe01/_spikeglx_ephysData_g0_t0.imec1.timestamps.cbin",
63 "probe01/_spikeglx_ephysData_g0_t0.imec1.ap.cbin",
64 "probe01/_spikeglx_ephysData_g0_t0.imec1.lf.cbin",
65 "probe01/_spikeglx_ephysData_g0_t0.imec1.ap.ch",
66 "probe01/_spikeglx_ephysData_g0_t0.imec1.wiring.json",
67 "probe02/_spikeglx_ephysData_g0_t0.imec.ap.meta", # 3A
68 "probe02/_spikeglx_ephysData_g0_t0.imec.lf.meta",
69 "probe02/_spikeglx_ephysData_g0_t0.imec.ap.bin",
70 "probe02/_spikeglx_ephysData_g0_t0.imec.lf.bin",
71 "_spikeglx_ephysData_g0_t0.nidq.sync.npy",
72 "_spikeglx_ephysData_g0_t0.nidq.meta",
73 "_spikeglx_ephysData_g0_t0.nidq.cbin",
74 "_spikeglx_ephysData_g0_t0.nidq.ch",
75 "_spikeglx_ephysData_g0_t0.wiring.json",
76 ]
77 for f in file_list: 1ai
78 fpath = raw_ephys_data_path / Path(f) 1ai
79 fpath.parent.mkdir(parents=True, exist_ok=True) 1ai
80 fpath.touch() 1ai
82 return session_path 1ai
85def populate_raw_spikeglx(session_path,
86 model='3B', legacy=False, user_label='my_run', n_probes=2):
87 """
88 Touch file tree to emulate files saved by SpikeGLX
89 :param session_path: The raw ephys data path to place files
90 :param model: Probe model file structure ('3A' or '3B')
91 :param legacy: If true, emulate older SpikeGLX version where all files are saved
92 into a single folder
93 :param user_label: User may input any name into SpikeGLX and filenames will include this
94 :param n_probes: Number of probe datafiles to touch
95 :return:
97 Examples:
98 populate_raw_spikeglx('3A_folder', model='3A', legacy=True, n_probes=1)
99 3A_folder
100 └───raw_ephys_folder
101 my_run_probe00_g0_t0.imec.ap.bin
102 my_run_probe00_g0_t0.imec.ap.meta
103 my_run_probe00_g0_t0.imec.lf.bin
104 my_run_probe00_g0_t0.imec.lf.meta
106 populate_raw_spikeglx('3B_folder', model='3B', n_probes=3)
107 3B_folder
108 └───my_run_g0_t0
109 my_run_g0_t0.imec0.ap.bin
110 my_run_g0_t0.imec0.ap.meta
111 my_run_g0_t0.imec0.lf.bin
112 my_run_g0_t0.imec0.lf.meta
113 my_run_g0_t0.imec1.ap.bin
114 my_run_g0_t0.imec1.ap.meta
115 my_run_g0_t0.imec1.lf.bin
116 my_run_g0_t0.imec1.lf.meta
117 my_run_g0_t0.imec2.ap.bin
118 my_run_g0_t0.imec2.ap.meta
119 my_run_g0_t0.imec2.lf.bin
120 my_run_g0_t0.imec2.lf.meta
121 my_run_g0_t0.nidq.bin
122 my_run_g0_t0.nidq.meta
124 See also: http://billkarsh.github.io/SpikeGLX/help/parsing/
125 """
126 for i in range(n_probes):
127 label = (user_label + f'_probe{i:02}') if legacy and model == '3A' else user_label
128 root = session_path.joinpath('raw_ephys_folder' if legacy else f'{user_label}_g0_t0')
129 root.mkdir(exist_ok=True, parents=True)
130 for ext in ('meta', 'bin'):
131 for freq in ('lf', 'ap'):
132 filename = f'{label}_g0_t0.imec{i if model == "3B" else ""}.{freq}.{ext}'
133 root.joinpath(filename).touch()
134 if model == '3B':
135 root.joinpath(f'{label}_g0_t0.nidq.{ext}').touch()
138def create_fake_raw_video_data_folder(session_path, populate=True, write_pars_stub=False):
139 """
140 Create the folder structure for a raw video session with three cameras.
141 Creates a raw_video_data folder and optionally, touches some files and writes a experiment
142 description stub to a _devices folder.
144 Parameters
145 ----------
146 session_path : str, pathlib.Path
147 The session path in which to create the folders.
148 populate : bool
149 If true, touch some raw video files.
150 write_pars_stub : bool, str, dict
151 If true, write an experiment description stub containing behaviour settings. If a string,
152 the stub filename will contain this. If a dict, the key is used as the filename; the value,
153 the file contents.
155 Example
156 -------
157 >>> create_fake_raw_video_data_folder(session_path, populate=False, write_pars_stub=False)
158 >>> create_fake_raw_video_data_folder(session_path, write_pars_stub='hostname_19826354')
159 """
160 session_path = Path(session_path) 1abc
161 raw_video_data_path = session_path / "raw_video_data" 1abc
162 raw_video_data_path.mkdir(exist_ok=True, parents=True) 1abc
163 if populate: 1abc
164 file_list = [ 1abc
165 "_iblrig_leftCamera.raw.mp4",
166 "_iblrig_rightCamera.raw.mp4",
167 "_iblrig_bodyCamera.raw.mp4",
168 "_iblrig_leftCamera.timestamps.ssv",
169 "_iblrig_rightCamera.timestamps.ssv",
170 "_iblrig_bodyCamera.timestamps.ssv",
171 "_iblrig_leftCamera.GPIO.bin",
172 "_iblrig_rightCamera.GPIO.bin",
173 "_iblrig_bodyCamera.GPIO.bin",
174 "_iblrig_leftCamera.frame_counter.bin",
175 "_iblrig_rightCamera.frame_counter.bin",
176 "_iblrig_bodyCamera.frame_counter.bin",
177 "_iblrig_VideoCodeFiles.raw.zip",
178 ]
179 for f in file_list: 1abc
180 fpath = raw_video_data_path / Path(f) 1abc
181 fpath.parent.mkdir(parents=True, exist_ok=True) 1abc
182 fpath.touch() 1abc
184 if write_pars_stub: 1abc
185 if isinstance(write_pars_stub, dict):
186 (name, data), = write_pars_stub.items()
187 else:
188 name = write_pars_stub if isinstance(write_pars_stub, str) else 'video'
189 d = {'collection': 'raw_video_data', 'sync_label': 'frame2ttl'}
190 data = {
191 'devices': {'cameras': {k: d.copy() for k in ('body', 'left', 'right')}},
192 'version': session_params.SPEC_VERSION
193 }
194 file_device = session_path.joinpath(f'_ibl_experiment.description_{name}.yaml')
195 file_device.parent.mkdir(exist_ok=True)
196 session_params.write_yaml(file_device, data)
197 return raw_video_data_path 1abc
200def create_fake_alf_folder_dlc_data(session_path, populate=True):
201 session_path = Path(session_path)
202 alf_path = session_path / "alf"
203 alf_path.mkdir(exist_ok=True, parents=True)
204 if populate:
205 file_list = [
206 "_ibl_leftCamera.dlc.pqt",
207 "_ibl_rightCamera.dlc.pqt",
208 "_ibl_bodyCamera.dlc.pqt",
209 "_ibl_leftCamera.times.npy",
210 "_ibl_rightCamera.times.npy",
211 "_ibl_bodyCamera.times.npy",
212 "_ibl_leftCamera.features.npy",
213 "_ibl_rightCamera.features.npy",
214 ]
215 for f in file_list:
216 fpath = alf_path / Path(f)
217 fpath.parent.mkdir(parents=True, exist_ok=True)
218 fpath.touch()
221def create_fake_raw_behavior_data_folder(
222 session_path, populate=True, task="ephysCW", folder="raw_behavior_data", write_pars_stub=False
223):
224 """Create the folder structure for a raw behaviour session.
226 Creates a raw_behavior_data folder and optionally, touches some files and writes an experiment
227 description stub to a `_devices` folder.
229 Parameters
230 ----------
231 session_path : pathlib.Path
232 The session path in which to create the folders.
233 populate : bool
234 If true, touch some raw behaviour files.
235 task : str
236 The name of the task protocol, if 'ephys' or 'passive' extra files are touched.
237 write_pars_stub : bool, str, dict
238 If true, write an experiment description stub containing behaviour settings. If a string,
239 the stub will be named as such. If a dict, the key is used as the filename; the value,
240 the file contents.
242 Returns
243 -------
244 pathlib.Path
245 The raw behaviour data path.
246 """
247 raw_behavior_data_path = session_path / folder 1adbefcgh
248 raw_behavior_data_path.mkdir(exist_ok=True, parents=True) 1adbefcgh
249 ephysCW = [ 1adbefcgh
250 "_iblrig_ambientSensorData.raw.jsonable",
251 "_iblrig_encoderEvents.raw.ssv",
252 "_iblrig_encoderPositions.raw.ssv",
253 "_iblrig_encoderTrialInfo.raw.ssv",
254 "_iblrig_micData.raw.wav",
255 "_iblrig_stimPositionScreen.raw.csv",
256 "_iblrig_syncSquareUpdate.raw.csv",
257 "_iblrig_taskCodeFiles.raw.zip",
258 "_iblrig_taskData.raw.jsonable",
259 "_iblrig_taskSettings.raw.json",
260 "online_plot.png",
261 ]
262 passiveCW = [ 1adbefcgh
263 "_iblrig_encoderEvents.raw.ssv",
264 "_iblrig_encoderPositions.raw.ssv",
265 "_iblrig_encoderTrialInfo.raw.ssv",
266 "_iblrig_RFMapStim.raw.bin",
267 "_iblrig_stimPositionScreen.raw.csv",
268 "_iblrig_syncSquareUpdate.raw.csv",
269 "_iblrig_taskCodeFiles.raw.zip",
270 "_iblrig_taskSettings.raw.json",
271 ]
273 if populate: 1adbefcgh
274 file_list = [] 1adbefcgh
275 if "ephys" in task: 1adbefcgh
276 file_list = ephysCW 1adbefcgh
277 elif "passive" in task:
278 file_list = passiveCW
279 (session_path / "passive_data_for_ephys.flag").touch()
280 else:
281 print(f"Not implemented: Task {task}")
283 for f in file_list: 1adbefcgh
284 fpath = raw_behavior_data_path / Path(f) 1adbefcgh
285 fpath.parent.mkdir(parents=True, exist_ok=True) 1adbefcgh
286 fpath.touch() 1adbefcgh
288 if write_pars_stub: 1adbefcgh
289 if isinstance(write_pars_stub, dict):
290 (name, data), = write_pars_stub.items()
291 else:
292 name = write_pars_stub if isinstance(write_pars_stub, str) else 'behaviour'
293 data = {
294 'devices': {'microphone': {'microphone': {'collection': folder, 'sync_label': None}}},
295 'procedures': ['Behavior training/tasks'],
296 'projects': ['ibl_neuropixel_brainwide_01'],
297 'tasks': [{task: {'collection': folder}}],
298 'version': session_params.SPEC_VERSION
299 }
300 if 'ephys' in task:
301 data['tasks'][0][task]['sync_label'] = 'frame2ttl'
302 else:
303 data['sync'] = {'bpod': {'collection': 'raw_behavior_data', 'extension': 'jsonable'}}
304 data['tasks'][0][task]['sync_label'] = 'bpod'
306 file_device = session_path.joinpath(f'_ibl_experiment.description_{name}.yaml')
307 file_device.parent.mkdir(exist_ok=True)
308 session_params.write_yaml(file_device, data)
310 return raw_behavior_data_path 1adbefcgh
313def populate_task_settings(fpath: Path, patch: dict):
314 """
315 Populate a task settings JSON file.
317 Parameters
318 ----------
319 fpath : pathlib.Path
320 A path to a raw task settings folder or the full settings file path.
321 patch : dict
322 The settings dict to write to file.
324 Returns
325 -------
326 pathlib.Path
327 The full settings file path.
328 """
329 if fpath.is_dir(): 1akl
330 fpath /= '_iblrig_taskSettings.raw.json'
331 with fpath.open('w') as f: 1akl
332 json.dump(patch, f, indent=1) 1akl
333 return fpath 1akl
336def create_fake_complete_ephys_session(
337 root_data_path, lab="fakelab", mouse="fakemouse", date="1900-01-01", num="001", increment=True
338):
339 session_path = create_fake_session_folder(
340 root_data_path, lab=lab, mouse=mouse, date=date, num=num, increment=increment
341 )
342 _mouse, _date, _num = session_path.parts[-3:]
343 create_fake_raw_ephys_data_folder(session_path, populate=True)
344 create_fake_raw_video_data_folder(session_path, populate=True)
345 create_fake_raw_behavior_data_folder(session_path, populate=True, task="ephys")
346 create_fake_raw_behavior_data_folder(
347 session_path, populate=True, task="passive", folder="raw_passive_data"
348 )
349 fpath = Path(session_path) / "raw_passive_data" / "_iblrig_taskSettings.raw.json"
350 passive_settings = {
351 "CORRESPONDING_EPHYS_SESSION":
352 f"C:\\some\\root\\folder\\Subjects\\{_mouse}\\{_date}\\{_num}"
353 }
354 populate_task_settings(fpath, passive_settings)
355 if session_path.joinpath("passive_data_for_ephys.flag").exists():
356 session_path.joinpath("passive_data_for_ephys.flag").unlink()
358 return session_path
361def create_fake_ephys_recording_bad_passive_transfer_sessions(
362 root_data_path, lab="fakelab", mouse="fakemouse", date="1900-01-01", num="001", increment=True
363):
364 session_path = create_fake_session_folder(
365 root_data_path, lab=lab, mouse=mouse, date=date, num=num, increment=increment
366 )
367 _mouse, _date, _num = session_path.parts[-3:]
368 create_fake_raw_ephys_data_folder(session_path, populate=True)
369 create_fake_raw_video_data_folder(session_path, populate=True)
370 create_fake_raw_behavior_data_folder(session_path, populate=True, task="ephys")
372 passive_session_path = create_fake_session_folder(
373 root_data_path, lab=lab, mouse=mouse, date=date, num=num, increment=increment
374 )
375 create_fake_raw_behavior_data_folder(passive_session_path, populate=True, task="passive")
376 fpath = Path(passive_session_path) / "raw_behavior_data" / "_iblrig_taskSettings.raw.json"
377 passive_settings = {
378 "CORRESPONDING_EPHYS_SESSION":
379 f"C:\\some\\root\\folder\\Subjects\\{_mouse}\\{_date}\\{_num}"
380 }
381 populate_task_settings(fpath, passive_settings)
383 return session_path, passive_session_path
386def register_new_session(one, subject=None, date=None):
387 """
388 Register a new test session.
390 NB: This creates the session path on disk, using `one.cache_dir`.
392 Parameters
393 ----------
394 one : one.api.OneAlyx
395 An instance of ONE.
396 subject : str
397 The subject name. If None, a new random subject is created.
398 date : str
399 An ISO date string. If None, a random one is created.
401 Returns
402 -------
403 pathlib.Path
404 New local session path.
405 uuid.UUID
406 The experiment UUID.
407 """
408 if not date: 1aj
409 date = f'20{random.randint(0, 99):02}-{random.randint(1, 12):02}-{random.randint(1, 28):02}' 1aj
410 if not subject: 1aj
411 subject = ''.join(random.choices(string.ascii_letters, k=10))
412 one.alyx.rest('subjects', 'create', data={'lab': 'mainenlab', 'nickname': subject})
414 session_path, eid = RegistrationClient(one).create_new_session(subject, date=str(date)[:10]) 1aj
415 _logger.debug('Registered session %s with eid %s', session_path, eid) 1aj
416 return session_path, eid 1aj