Coverage for ibllib/pipes/behavior_tasks.py: 85%

293 statements  

« prev     ^ index     » next       coverage.py v7.9.1, created at 2025-07-02 18:55 +0100

1"""Standard task protocol extractor dynamic pipeline tasks.""" 

2import logging 

3import traceback 

4 

5from packaging import version 

6import one.alf.io as alfio 

7from one.alf.path import session_path_parts 

8from one.api import ONE 

9 

10from ibllib.oneibl.registration import get_lab 

11from ibllib.oneibl.data_handlers import ServerDataHandler, SDSCDataHandler, ExpectedDataset 

12from ibllib.pipes import base_tasks 

13from ibllib.io.raw_data_loaders import load_settings, load_bpod_fronts 

14from ibllib.qc.task_extractors import TaskQCExtractor 

15from ibllib.qc.task_metrics import HabituationQC, TaskQC, update_dataset_qc 

16from ibllib.io.extractors.ephys_passive import PassiveChoiceWorld 

17from ibllib.io.extractors.bpod_trials import get_bpod_extractor 

18from ibllib.io.extractors.ephys_fpga import FpgaTrials, FpgaTrialsHabituation, get_sync_and_chn_map 

19from ibllib.io.extractors.mesoscope import TimelineTrials, TimelineTrialsHabituation 

20from ibllib.pipes import training_status 

21from ibllib.plots.figures import BehaviourPlots 

22 

23_logger = logging.getLogger(__name__) 

24 

25 

26class HabituationRegisterRaw(base_tasks.RegisterRawDataTask, base_tasks.BehaviourTask): 

27 priority = 100 

28 job_size = 'small' 

29 

30 @property 

31 def signature(self): 

32 signature = { 1E

33 'input_files': [], 

34 'output_files': [ 

35 ('_iblrig_taskData.raw.*', self.collection, True), 

36 ('_iblrig_taskSettings.raw.*', self.collection, True), 

37 ('_iblrig_encoderEvents.raw*', self.collection, False), 

38 ('_iblrig_encoderPositions.raw*', self.collection, False), 

39 ('_iblrig_encoderTrialInfo.raw*', self.collection, False), 

40 ('_iblrig_stimPositionScreen.raw*', self.collection, False), 

41 ('_iblrig_syncSquareUpdate.raw*', self.collection, False), 

42 ('_iblrig_ambientSensorData.raw*', self.collection, False) 

43 ] 

44 } 

45 return signature 1E

46 

47 

48class HabituationTrialsBpod(base_tasks.BehaviourTask): 

49 priority = 90 

50 job_size = 'small' 

51 

52 @property 

53 def signature(self): 

54 signature = { 1ogt

55 'input_files': [ 

56 ('_iblrig_taskData.raw.*', self.collection, True), 

57 ('_iblrig_taskSettings.raw.*', self.collection, True), 

58 ], 

59 'output_files': [ 

60 ('*trials.contrastLeft.npy', self.output_collection, True), 

61 ('*trials.contrastRight.npy', self.output_collection, True), 

62 ('*trials.feedback_times.npy', self.output_collection, True), 

63 ('*trials.feedbackType.npy', self.output_collection, True), 

64 ('*trials.goCue_times.npy', self.output_collection, True), 

65 ('*trials.goCueTrigger_times.npy', self.output_collection, True), 

66 ('*trials.intervals.npy', self.output_collection, True), 

67 ('*trials.rewardVolume.npy', self.output_collection, True), 

68 ('*trials.stimOff_times.npy', self.output_collection, True), 

69 ('*trials.stimOn_times.npy', self.output_collection, True), 

70 ('*trials.stimOffTrigger_times.npy', self.output_collection, False), 

71 ('*trials.stimOnTrigger_times.npy', self.output_collection, False), 

72 ] 

73 } 

74 return signature 1ogt

75 

76 def _run(self, update=True, save=True): 

77 """Extracts an iblrig training session.""" 

78 trials, output_files = self.extract_behaviour(save=save) 1ot

79 

80 if trials is None: 1ot

81 return None 1t

82 if self.one is None or self.one.offline: 1o

83 return output_files 1o

84 

85 # Run the task QC 

86 self.run_qc(trials, update=update) 

87 return output_files 

88 

89 def extract_behaviour(self, **kwargs): 

90 settings = load_settings(self.session_path, self.collection) 1ogbt

91 if version.parse(settings['IBLRIG_VERSION'] or '100.0.0') < version.parse('5.0.0'): 1ogbt

92 _logger.error('No extraction of legacy habituation sessions') 1t

93 return None, None 1t

94 self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection) 1ogb

95 self.extractor.default_path = self.output_collection 1ogb

96 return self.extractor.extract(task_collection=self.collection, **kwargs) 1ogb

97 

98 def run_qc(self, trials_data=None, update=True): 

99 trials_data = self._assert_trials_data(trials_data) # validate trials data 1b

100 

101 # Compile task data for QC 

102 ns = 'task' if self.protocol_number is None else f'task_{self.protocol_number:02}' 1b

103 qc = HabituationQC(self.session_path, namespace=ns, one=self.one) 1b

104 qc.extractor = TaskQCExtractor(self.session_path) 1b

105 

106 # Update extractor fields 

107 qc.extractor.data = qc.extractor.rename_data(trials_data.copy()) 1b

108 qc.extractor.frame_ttls = self.extractor.frame2ttl # used in iblapps QC viewer 1b

109 qc.extractor.audio_ttls = self.extractor.audio # used in iblapps QC viewer 1b

110 qc.extractor.settings = self.extractor.settings 1b

111 

112 qc.run(update=update) 1b

113 return qc 1b

114 

115 

116class HabituationTrialsNidq(HabituationTrialsBpod): 

117 priority = 90 

118 job_size = 'small' 

119 

120 @property 

121 def signature(self): 

122 signature = super().signature 1g

123 signature['input_files'] = [ 1g

124 ('_iblrig_taskData.raw.*', self.collection, True), 

125 ('_iblrig_taskSettings.raw.*', self.collection, True), 

126 (f'_{self.sync_namespace}_sync.channels.npy', self.sync_collection, True), 

127 (f'_{self.sync_namespace}_sync.polarities.npy', self.sync_collection, True), 

128 (f'_{self.sync_namespace}_sync.times.npy', self.sync_collection, True), 

129 ('*wiring.json', self.sync_collection, False), 

130 ('*.meta', self.sync_collection, True)] 

131 return signature 1g

132 

133 def extract_behaviour(self, save=True, **kwargs): 

134 """Extract the habituationChoiceWorld trial data using NI DAQ clock.""" 

135 # Extract Bpod trials 

136 bpod_trials, _ = super().extract_behaviour(save=False, **kwargs) 1b

137 

138 # Sync Bpod trials to FPGA 

139 sync, chmap = get_sync_and_chn_map(self.session_path, self.sync_collection) 1b

140 self.extractor = FpgaTrialsHabituation( 1b

141 self.session_path, bpod_trials=bpod_trials, bpod_extractor=self.extractor) 

142 

143 # NB: The stimOff times are called stimCenter times for habituation choice world 

144 outputs, files = self.extractor.extract( 1b

145 save=save, sync=sync, chmap=chmap, path_out=self.session_path.joinpath(self.output_collection), 

146 task_collection=self.collection, protocol_number=self.protocol_number, **kwargs) 

147 return outputs, files 1b

148 

149 def run_qc(self, trials_data=None, update=True, **_): 

150 """Run and update QC. 

151 

152 This adds the bpod TTLs to the QC object *after* the QC is run in the super call method. 

153 The raw Bpod TTLs are not used by the QC however they are used in the iblapps QC plot. 

154 """ 

155 qc = super().run_qc(trials_data=trials_data, update=update) 1b

156 qc.extractor.bpod_ttls = self.extractor.bpod 1b

157 return qc 1b

158 

159 

160class HabituationTrialsTimeline(HabituationTrialsNidq): 

161 """Behaviour task extractor with DAQdata.raw NPY datasets.""" 

162 @property 

163 def signature(self): 

164 signature = super().signature 1g

165 signature['input_files'] = [ 1g

166 ('_iblrig_taskData.raw.*', self.collection, True), 

167 ('_iblrig_taskSettings.raw.*', self.collection, True), 

168 (f'_{self.sync_namespace}_DAQdata.raw.npy', self.sync_collection, True), 

169 (f'_{self.sync_namespace}_DAQdata.timestamps.npy', self.sync_collection, True), 

170 (f'_{self.sync_namespace}_DAQdata.meta.json', self.sync_collection, True), 

171 ] 

172 return signature 1g

173 

174 def extract_behaviour(self, save=True, **kwargs): 

175 """Extract the Bpod trials data and Timeline acquired signals.""" 

176 # First determine the extractor from the task protocol 

177 bpod_trials, _ = HabituationTrialsBpod.extract_behaviour(self, save=False, **kwargs) 1g

178 

179 # Sync Bpod trials to DAQ 

180 self.extractor = TimelineTrialsHabituation(self.session_path, bpod_trials=bpod_trials, bpod_extractor=self.extractor) 1g

181 save_path = self.session_path / self.output_collection 1g

182 if not self._spacer_support(self.extractor.settings): 1g

183 _logger.warning('Protocol spacers not supported; setting protocol_number to None') 

184 self.protocol_number = None 

185 

186 # NB: The stimOff times are called stimCenter times for habituation choice world 

187 dsets, out_files = self.extractor.extract( 1g

188 save=save, path_out=save_path, sync_collection=self.sync_collection, 

189 task_collection=self.collection, protocol_number=self.protocol_number, **kwargs) 

190 

191 return dsets, out_files 1g

192 

193 

194class TrialRegisterRaw(base_tasks.RegisterRawDataTask, base_tasks.BehaviourTask): 

195 priority = 100 

196 job_size = 'small' 

197 

198 @property 

199 def signature(self): 

200 signature = { 1aFGHcijklmhn

201 'input_files': [], 

202 'output_files': [ 

203 ('_iblrig_taskData.raw.*', self.collection, True), 

204 ('_iblrig_taskSettings.raw.*', self.collection, True), 

205 ('_iblrig_encoderEvents.raw*', self.collection, False), 

206 ('_iblrig_encoderPositions.raw*', self.collection, False), 

207 ('_iblrig_encoderTrialInfo.raw*', self.collection, False), 

208 ('_iblrig_stimPositionScreen.raw*', self.collection, False), 

209 ('_iblrig_syncSquareUpdate.raw*', self.collection, False), 

210 ('_iblrig_ambientSensorData.raw*', self.collection, False) 

211 ] 

212 } 

213 return signature 1aFGHcijklmhn

214 

215 

216class PassiveRegisterRaw(base_tasks.RegisterRawDataTask, base_tasks.BehaviourTask): 

217 priority = 100 

218 job_size = 'small' 

219 

220 @property 

221 def signature(self): 

222 signature = { 1aIijklmn

223 'input_files': [], 

224 'output_files': [('_iblrig_taskSettings.raw.*', self.collection, True), 

225 ('_iblrig_encoderEvents.raw*', self.collection, False), 

226 ('_iblrig_encoderPositions.raw*', self.collection, False), 

227 ('_iblrig_encoderTrialInfo.raw*', self.collection, False), 

228 ('_iblrig_stimPositionScreen.raw*', self.collection, False), 

229 ('_iblrig_syncSquareUpdate.raw*', self.collection, False), 

230 ('_iblrig_RFMapStim.raw*', self.collection, True)] 

231 } 

232 return signature 1aIijklmn

233 

234 

235class PassiveTaskNidq(base_tasks.BehaviourTask): 

236 priority = 90 

237 job_size = 'small' 

238 

239 @property 

240 def signature(self): 

241 signature = { 1aABijklmnC

242 'input_files': [('_iblrig_taskSettings.raw*', self.collection, True), 

243 ('_iblrig_RFMapStim.raw*', self.collection, True), 

244 (f'_{self.sync_namespace}_sync.channels.*', self.sync_collection, True), 

245 (f'_{self.sync_namespace}_sync.polarities.*', self.sync_collection, True), 

246 (f'_{self.sync_namespace}_sync.times.*', self.sync_collection, True), 

247 ('*.wiring.json', self.sync_collection, False), 

248 ('*.meta', self.sync_collection, False)], 

249 'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, False), 

250 ('_ibl_passivePeriods.intervalsTable.csv', self.output_collection, True), 

251 ('_ibl_passiveRFM.times.npy', self.output_collection, True), 

252 ('_ibl_passiveStims.table.csv', self.output_collection, False)] 

253 } 

254 return signature 1aABijklmnC

255 

256 def _run(self, **kwargs): 

257 """returns a list of pathlib.Paths. """ 

258 data, paths = PassiveChoiceWorld(self.session_path).extract( 1ABC

259 sync_collection=self.sync_collection, task_collection=self.collection, save=True, 

260 path_out=self.session_path.joinpath(self.output_collection), protocol_number=self.protocol_number) 

261 

262 if any(x is None for x in paths): 1ABC

263 self.status = -1 

264 

265 return paths 1ABC

266 

267 

268class PassiveTaskTimeline(base_tasks.BehaviourTask, base_tasks.MesoscopeTask): 

269 """TODO should be mesoscope invariant, using wiring file""" 

270 priority = 90 

271 job_size = 'small' 

272 

273 @property 

274 def signature(self): 

275 signature = { 

276 'input_files': [('_iblrig_taskSettings.raw*', self.collection, True), 

277 ('_iblrig_RFMapStim.raw*', self.collection, True), 

278 (f'_{self.sync_namespace}_sync.channels.*', self.sync_collection, False), 

279 (f'_{self.sync_namespace}_sync.polarities.*', self.sync_collection, False), 

280 (f'_{self.sync_namespace}_sync.times.*', self.sync_collection, False)], 

281 'output_files': [('_ibl_passiveGabor.table.csv', self.output_collection, False), 

282 ('_ibl_passivePeriods.intervalsTable.csv', self.output_collection, True), 

283 ('_ibl_passiveRFM.times.npy', self.output_collection, True), 

284 ('_ibl_passiveStims.table.csv', self.output_collection, False)] 

285 } 

286 return signature 

287 

288 def _run(self, **kwargs): 

289 """returns a list of pathlib.Paths. 

290 This class exists to load the sync file and set the protocol_number to None 

291 """ 

292 settings = load_settings(self.session_path, self.collection) 

293 ver = settings.get('IBLRIG_VERSION') or '100.0.0' 

294 if ver == '100.0.0' or version.parse(ver) <= version.parse('7.1.0'): 

295 _logger.warning('Protocol spacers not supported; setting protocol_number to None') 

296 self.protocol_number = None 

297 

298 sync, chmap = self.load_sync() 

299 data, paths = PassiveChoiceWorld(self.session_path).extract( 

300 sync_collection=self.sync_collection, task_collection=self.collection, save=True, 

301 path_out=self.session_path.joinpath(self.output_collection), 

302 protocol_number=self.protocol_number, sync=sync, sync_map=chmap) 

303 

304 if any(x is None for x in paths): 

305 self.status = -1 

306 

307 return paths 

308 

309 

310class ChoiceWorldTrialsBpod(base_tasks.BehaviourTask): 

311 priority = 90 

312 job_size = 'small' 

313 extractor = None 

314 """ibllib.io.extractors.base.BaseBpodTrialsExtractor: An instance of the Bpod trials extractor.""" 

315 

316 @property 

317 def signature(self): 

318 signature = { 1JKpqrcs

319 'input_files': [ 

320 ('_iblrig_taskData.raw.*', self.collection, True), 

321 ('_iblrig_taskSettings.raw.*', self.collection, True), 

322 ('_iblrig_encoderEvents.raw*', self.collection, True), 

323 ('_iblrig_encoderPositions.raw*', self.collection, True)], 

324 'output_files': [ 

325 ('*trials.goCueTrigger_times.npy', self.output_collection, True), 

326 ('*trials.stimOffTrigger_times.npy', self.output_collection, False), 

327 ('*trials.stimOnTrigger_times.npy', self.output_collection, False), 

328 ('*trials.table.pqt', self.output_collection, True), 

329 ('*wheel.position.npy', self.output_collection, True), 

330 ('*wheel.timestamps.npy', self.output_collection, True), 

331 ('*wheelMoves.intervals.npy', self.output_collection, True), 

332 ('*wheelMoves.peakAmplitude.npy', self.output_collection, True) 

333 ] 

334 } 

335 return signature 1JKpqrcs

336 

337 def _run(self, update=True, save=True, **kwargs): 

338 """Extracts an iblrig training session.""" 

339 trials, output_files = self.extract_behaviour(save=save) 1pfqrcds

340 if trials is None: 1pfqrcds

341 return None 

342 if self.one is None or self.one.offline: 1pfqrcds

343 return output_files 1pfqrds

344 

345 # Run the task QC 

346 qc = self.run_qc(trials, update=update, **kwargs) 1c

347 if update and not self.one.offline: 1c

348 on_server = self.location == 'server' and isinstance(self.data_handler, ServerDataHandler) 1c

349 on_sdsc = self.location == 'sdsc' and isinstance(self.data_handler, SDSCDataHandler) 1c

350 if not (on_server or on_sdsc): 1c

351 _logger.warning('Updating dataset QC only supported on local servers and SDSC') 

352 else: 

353 # On SDSC the lab is in the session path; on local servers we use Alyx to get the lab 

354 lab = get_lab(self.session_path, self.one.alyx) if on_server else self.session_path.lab 1c

355 # registered_dsets = self.register_datasets(labs=lab) 

356 datasets = self.data_handler.uploadData(output_files, self.version, labs=lab) 1c

357 update_dataset_qc(qc, datasets, self.one) 1c

358 

359 return output_files 1c

360 

361 def extract_behaviour(self, **kwargs): 

362 self.extractor = get_bpod_extractor(self.session_path, task_collection=self.collection) 1aDpfqrcdbeuvwxsyz

363 _logger.info('Bpod trials extractor: %s.%s', 1aDpfqrcdbeuvwxsyz

364 self.extractor.__module__, self.extractor.__class__.__name__) 

365 self.extractor.default_path = self.output_collection 1aDpfqrcdbeuvwxsyz

366 return self.extractor.extract(task_collection=self.collection, **kwargs) 1aDpfqrcdbeuvwxsyz

367 

368 def run_qc(self, trials_data=None, update=True, QC=None): 

369 """ 

370 Run the task QC. 

371 

372 Parameters 

373 ---------- 

374 trials_data : dict 

375 The complete extracted task data. 

376 update : bool 

377 If True, updates the session QC fields on Alyx. 

378 QC : ibllib.qc.task_metrics.TaskQC 

379 An optional QC class to instantiate. 

380 

381 Returns 

382 ------- 

383 ibllib.qc.task_metrics.TaskQC 

384 The task QC object. 

385 """ 

386 trials_data = self._assert_trials_data(trials_data) # validate trials data 1ac

387 

388 # Compile task data for QC 

389 qc_extractor = TaskQCExtractor(self.session_path) 1ac

390 qc_extractor.data = qc_extractor.rename_data(trials_data) 1ac

391 if not QC: 1ac

392 QC = HabituationQC if type(self.extractor).__name__ == 'HabituationTrials' else TaskQC 1ac

393 _logger.debug('Running QC with %s.%s', QC.__module__, QC.__name__) 1ac

394 namespace = 'task' if self.protocol_number is None else f'task_{self.protocol_number:02}' 1ac

395 qc = QC(self.session_path, namespace=namespace, one=self.one, log=_logger) 1ac

396 if QC is not HabituationQC: 1ac

397 qc_extractor.wheel_encoding = 'X1' 1ac

398 qc_extractor.settings = self.extractor.settings 1ac

399 qc_extractor.frame_ttls, qc_extractor.audio_ttls = load_bpod_fronts( 1ac

400 self.session_path, task_collection=self.collection) 

401 qc.extractor = qc_extractor 1ac

402 

403 # Aggregate and update Alyx QC fields 

404 qc.run(update=update) 1ac

405 return qc 1ac

406 

407 

408class ChoiceWorldTrialsNidq(ChoiceWorldTrialsBpod): 

409 priority = 90 

410 job_size = 'small' 

411 

412 @property 

413 def signature(self): 

414 I = ExpectedDataset.input # noqa 1afijklmhnd

415 ns = self.sync_namespace 1afijklmhnd

416 # Neuropixels 3A sync data are kept in individual probe collections 

417 v3A = ( 1afijklmhnd

418 I(f'_{ns}_sync.channels.probe??.npy', f'{self.sync_collection}/probe??', True, unique=False) & 

419 I(f'_{ns}_sync.polarities.probe??.npy', f'{self.sync_collection}/probe??', True, unique=False) & 

420 I(f'_{ns}_sync.times.probe??.npy', f'{self.sync_collection}/probe??', True, unique=False) & 

421 I(f'_{ns}_*.ap.meta', f'{self.sync_collection}/probe??', True, unique=False) & 

422 I(f'_{ns}_*wiring.json', f'{self.sync_collection}/probe??', False, unique=False) 

423 ) 

424 # Neuropixels 3B sync data are kept in probe-independent datasets 

425 v3B = ( 1afijklmhnd

426 I(f'_{ns}_sync.channels.npy', self.sync_collection, True) & 

427 I(f'_{ns}_sync.polarities.npy', self.sync_collection, True) & 

428 I(f'_{ns}_sync.times.npy', self.sync_collection, True) & 

429 I(f'_{ns}_*.meta', self.sync_collection, True) & 

430 I(f'_{ns}_*wiring.json', self.sync_collection, False) 

431 ) 

432 signature = { 1afijklmhnd

433 'input_files': [ 

434 ('_iblrig_taskData.raw.*', self.collection, True), 

435 ('_iblrig_taskSettings.raw.*', self.collection, True), 

436 ('_iblrig_encoderEvents.raw*', self.collection, True), 

437 ('_iblrig_encoderPositions.raw*', self.collection, True), 

438 v3B | (~v3B & v3A) # either 3B datasets OR 3A datasets must be present 

439 ], 

440 'output_files': [ 

441 ('*trials.goCueTrigger_times.npy', self.output_collection, True), 

442 ('*trials.intervals_bpod.npy', self.output_collection, False), 

443 ('*trials.stimOff_times.npy', self.output_collection, False), 

444 ('*trials.stimOffTrigger_times.npy', self.output_collection, False), 

445 ('*trials.stimOnTrigger_times.npy', self.output_collection, False), 

446 ('*trials.table.pqt', self.output_collection, True), 

447 ('*wheel.position.npy', self.output_collection, True), 

448 ('*wheel.timestamps.npy', self.output_collection, True), 

449 ('*wheelMoves.intervals.npy', self.output_collection, True), 

450 ('*wheelMoves.peakAmplitude.npy', self.output_collection, True) 

451 ] 

452 } 

453 return signature 1afijklmhnd

454 

455 def _behaviour_criterion(self, update=True, truncate_to_pass=True): 

456 """ 

457 Computes and update the behaviour criterion on Alyx 

458 """ 

459 from brainbox.behavior import training 

460 

461 trials = alfio.load_object(self.session_path.joinpath(self.output_collection), 'trials').to_df() 

462 good_enough, _ = training.criterion_delay( 

463 n_trials=trials.shape[0], 

464 perf_easy=training.compute_performance_easy(trials), 

465 ) 

466 if truncate_to_pass and not good_enough: 

467 n_trials = trials.shape[0] 

468 while not good_enough and n_trials > 400: 

469 n_trials -= 1 

470 good_enough, _ = training.criterion_delay( 

471 n_trials=n_trials, 

472 perf_easy=training.compute_performance_easy(trials[:n_trials]), 

473 ) 

474 

475 if update: 

476 eid = self.one.path2eid(self.session_path, query_type='remote') 

477 self.one.alyx.json_field_update( 

478 "sessions", eid, "extended_qc", {"behavior": int(good_enough)} 

479 ) 

480 

481 def extract_behaviour(self, save=True, **kwargs): 

482 # Extract Bpod trials 

483 bpod_trials, _ = super().extract_behaviour(save=False, **kwargs) 1fbeuvwxyz

484 

485 # Sync Bpod trials to FPGA 

486 sync, chmap = get_sync_and_chn_map(self.session_path, self.sync_collection) 1fbeuvwxyz

487 self.extractor = FpgaTrials(self.session_path, bpod_trials=bpod_trials, bpod_extractor=self.extractor) 1fbeuvwxyz

488 outputs, files = self.extractor.extract( 1fbeuvwxyz

489 save=save, sync=sync, chmap=chmap, path_out=self.session_path.joinpath(self.output_collection), 

490 task_collection=self.collection, protocol_number=self.protocol_number, **kwargs) 

491 return outputs, files 1fbeuvwxyz

492 

493 def run_qc(self, trials_data=None, update=False, plot_qc=False, QC=None): 

494 trials_data = self._assert_trials_data(trials_data) # validate trials data 1be

495 

496 # Compile task data for QC 

497 qc_extractor = TaskQCExtractor(self.session_path) 1be

498 qc_extractor.data = qc_extractor.rename_data(trials_data.copy()) 1be

499 if not QC: 1be

500 QC = HabituationQC if type(self.extractor).__name__ == 'HabituationTrials' else TaskQC 1be

501 _logger.debug('Running QC with %s.%s', QC.__module__, QC.__name__) 1be

502 namespace = 'task' if self.protocol_number is None else f'task_{self.protocol_number:02}' 1be

503 qc = QC(self.session_path, namespace=namespace, one=self.one, log=_logger) 1be

504 if QC is not HabituationQC: 1be

505 # Add Bpod wheel data 

506 wheel_ts_bpod = self.extractor.bpod2fpga(self.extractor.bpod_trials['wheel_timestamps']) 1be

507 qc_extractor.data['wheel_timestamps_bpod'] = wheel_ts_bpod 1be

508 qc_extractor.data['wheel_position_bpod'] = self.extractor.bpod_trials['wheel_position'] 1be

509 qc_extractor.wheel_encoding = 'X4' 1be

510 qc_extractor.frame_ttls = self.extractor.frame2ttl 1be

511 qc_extractor.audio_ttls = self.extractor.audio 1be

512 qc_extractor.bpod_ttls = self.extractor.bpod 1be

513 qc_extractor.settings = self.extractor.settings 1be

514 qc.extractor = qc_extractor 1be

515 

516 # Aggregate and update Alyx QC fields 

517 qc.run(update=update) 1be

518 

519 if plot_qc: 1be

520 _logger.info('Creating Trials QC plots') 

521 try: 

522 session_id = self.one.path2eid(self.session_path) 

523 plot_task = BehaviourPlots( 

524 session_id, self.session_path, one=self.one, task_collection=self.output_collection) 

525 _ = plot_task.run() 

526 self.plot_tasks.append(plot_task) 

527 except Exception: 

528 _logger.error('Could not create Trials QC Plot') 

529 _logger.error(traceback.format_exc()) 

530 self.status = -1 

531 return qc 1be

532 

533 def _run(self, update=True, plot_qc=True, save=True): 

534 output_files = super()._run(update=update, save=save, plot_qc=plot_qc) 1fd

535 if update and not self.one.offline: 1fd

536 self._behaviour_criterion(update=update) 

537 

538 return output_files 1fd

539 

540 

541class ChoiceWorldTrialsTimeline(ChoiceWorldTrialsNidq): 

542 """Behaviour task extractor with DAQdata.raw NPY datasets.""" 

543 @property 

544 def signature(self): 

545 signature = super().signature 1hd

546 signature['input_files'] = [ 1hd

547 ('_iblrig_taskData.raw.*', self.collection, True), 

548 ('_iblrig_taskSettings.raw.*', self.collection, True), 

549 ('_iblrig_encoderEvents.raw*', self.collection, True), 

550 ('_iblrig_encoderPositions.raw*', self.collection, True), 

551 (f'_{self.sync_namespace}_DAQdata.raw.npy', self.sync_collection, True), 

552 (f'_{self.sync_namespace}_DAQdata.timestamps.npy', self.sync_collection, True), 

553 (f'_{self.sync_namespace}_DAQdata.meta.json', self.sync_collection, True), 

554 ] 

555 if self.protocol: 1hd

556 extractor = get_bpod_extractor(self.session_path, protocol=self.protocol) 1hd

557 if extractor.save_names: 1hd

558 signature['output_files'] = [(fn, self.output_collection, True) 1hd

559 for fn in filter(None, extractor.save_names)] 

560 return signature 1hd

561 

562 def extract_behaviour(self, save=True, **kwargs): 

563 """Extract the Bpod trials data and Timeline acquired signals.""" 

564 # First determine the extractor from the task protocol 

565 bpod_trials, _ = ChoiceWorldTrialsBpod.extract_behaviour(self, save=False, **kwargs) 1d

566 

567 # Sync Bpod trials to DAQ 

568 self.extractor = TimelineTrials(self.session_path, bpod_trials=bpod_trials, bpod_extractor=self.extractor) 1d

569 save_path = self.session_path / self.output_collection 1d

570 if not self._spacer_support(self.extractor.settings): 1d

571 _logger.warning('Protocol spacers not supported; setting protocol_number to None') 1d

572 self.protocol_number = None 1d

573 

574 dsets, out_files = self.extractor.extract( 1d

575 save=save, path_out=save_path, sync_collection=self.sync_collection, 

576 task_collection=self.collection, protocol_number=self.protocol_number, **kwargs) 

577 

578 return dsets, out_files 1d

579 

580 

581class TrainingStatus(base_tasks.BehaviourTask): 

582 priority = 90 

583 job_size = 'small' 

584 

585 @property 

586 def signature(self): 

587 signature = { 1c

588 'input_files': [ 

589 ('_iblrig_taskData.raw.*', self.collection, True), 

590 ('_iblrig_taskSettings.raw.*', self.collection, True), 

591 ('*trials.table.pqt', self.output_collection, True)], 

592 'output_files': [] 

593 } 

594 return signature 1c

595 

596 def _run(self, upload=True): 

597 """ 

598 Extracts training status for subject. 

599 """ 

600 

601 lab = get_lab(self.session_path, self.one.alyx) 1c

602 if lab == 'cortexlab' and 'cortexlab' in self.one.alyx.base_url: 1c

603 _logger.info('Switching from cortexlab Alyx to IBL Alyx for training status queries.') 

604 one = ONE(base_url='https://alyx.internationalbrainlab.org') 

605 else: 

606 one = self.one 1c

607 

608 df = training_status.get_latest_training_information(self.session_path, one) 1c

609 if df is not None: 1c

610 training_status.make_plots( 1c

611 self.session_path, self.one, df=df, save=True, upload=upload, task_collection=self.collection) 

612 # Update status map in JSON field of subjects endpoint 

613 if self.one and not self.one.offline: 1c

614 _logger.debug('Updating JSON field of subjects endpoint') 1c

615 status = (df.set_index('date')[['training_status', 'session_path']].drop_duplicates( 1c

616 subset='training_status', keep='first').to_dict()) 

617 date, sess = status.items() 1c

618 data = {'trained_criteria': {v.replace(' ', '_'): (k, self.one.path2eid(sess[1][k])) 1c

619 for k, v in date[1].items()}} 

620 _, subject, *_ = session_path_parts(self.session_path) 1c

621 self.one.alyx.json_field_update('subjects', subject, data=data) 1c

622 output_files = [] 1c

623 return output_files 1c