Coverage for ibllib/pipes/base_tasks.py: 95%

1"""Abstract base classes for dynamic pipeline tasks.""" 

2import logging (empty)

3from pathlib import Path (empty)

4 

5from packaging import version (empty)

6from one.webclient import no_cache (empty)

7from iblutil.util import flatten, ensure_list (empty)

8import matplotlib.image (empty)

9from skimage.io import ImageCollection, imread (empty)

10 

11from ibllib.pipes.tasks import Task (empty)

12import ibllib.io.session_params as sess_params (empty)

13from ibllib.qc.base import sign_off_dict, SIGN_OFF_CATEGORIES (empty)

14from ibllib.io.raw_daq_loaders import load_timeline_sync_and_chmap (empty)

15from ibllib.oneibl.data_handlers import update_collections (empty)

16 

17_logger = logging.getLogger(__name__) (empty)

18 

19 

20class DynamicTask(Task): (empty)

21 

22 def __init__(self, session_path, **kwargs): (empty)

23 super().__init__(session_path, **kwargs) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

24 self.session_params = self.read_params_file() 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

25 

26 # TODO Which should be default? 

27 # Sync collection 

28 self.sync_collection = self.get_sync_collection(kwargs.get('sync_collection', None)) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

29 # Sync type 

30 self.sync = self.get_sync(kwargs.get('sync', None)) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

31 # Sync extension 

32 self.sync_ext = self.get_sync_extension(kwargs.get('sync_ext', None)) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

33 # Sync namespace 

34 self.sync_namespace = self.get_sync_namespace(kwargs.get('sync_namespace', None)) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

35 

36 def get_sync_collection(self, sync_collection=None): (empty)

37 return sync_collection if sync_collection else sess_params.get_sync_collection(self.session_params) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

38 

39 def get_sync(self, sync=None): (empty)

40 return sync if sync else sess_params.get_sync_label(self.session_params) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

41 

42 def get_sync_extension(self, sync_ext=None): (empty)

43 return sync_ext if sync_ext else sess_params.get_sync_extension(self.session_params) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

44 

45 def get_sync_namespace(self, sync_namespace=None): (empty)

46 return sync_namespace if sync_namespace else sess_params.get_sync_namespace(self.session_params) 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

47 

48 def get_protocol(self, protocol=None, task_collection=None): (empty)

49 return protocol if protocol else sess_params.get_task_protocol(self.session_params, task_collection) 6 ctx1bckjl,

50 

51 def get_task_collection(self, collection=None): (empty)

52 if not collection: 9 ctx1u-.bckjl,

53 collection = sess_params.get_task_collection(self.session_params) 5 ctx1bckjl

54 # If inferring the collection from the experiment description, assert only one returned 

55 assert collection is None or isinstance(collection, str) or len(collection) == 1 9 ctx1u-.bckjl,

56 return collection 9 ctx1u-.bckjl,

57 

58 def get_device_collection(self, device, device_collection=None): (empty)

59 if device_collection: 56 ctx2a x ] ^ ubu _ ` { - . b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % ( ) * | } + 1 3 4 ~ fb: ; = ? , @ [ abbbv cbdbeb

60 return device_collection 56 ctx2a x ] ^ ubu _ ` { - . b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % ( ) * | } + 1 3 4 ~ fb: ; = ? , @ [ abbbv cbdbeb

61 collection_map = sess_params.get_collections(self.session_params['devices']) 1 ctx2ub

62 return collection_map.get(device) 1 ctx2ub

63 

64 def read_params_file(self): (empty)

65 """Read the session parameters file. 

66 

67 Returns 

68 ------- 

69 dict 

70 The session parameters dictionary, or an empty dictionary if the file does not exist. 

71 """ 

72 if not self.session_path: 106 ctx2a 0 X E t w y D d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

73 return {} 4 ctx1twyD

74 return sess_params.read_params(self.session_path) or {} 102 ctx2a 0 X E d Y x ] ^ F G ' g H u _ ` { - . I o p J q K L M r N O s b h i e f c k n j l m 5 6 7 8 9 ! # 2 $ % hbibjbkblb( ) * | } + 1 3 4 ~ z C fbmbnbobpbqbrb/ : ; = ? , @ [ abbbv cbdbebZ A B P Q R S T U V W

75 

76 

77class BehaviourTask(DynamicTask): (empty)

78 

79 extractor = None (empty)

80 """ibllib.io.extractors.base.BaseBpodExtractor: A trials extractor object.""" (empty)

81 

82 def __init__(self, session_path, **kwargs): (empty)

83 super().__init__(session_path, **kwargs) 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

84 

85 self.collection = self.get_task_collection(kwargs.get('collection', None)) 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

86 # Task type (protocol) 

87 self.protocol = self.get_protocol(kwargs.get('protocol', None), task_collection=self.collection) 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

88 

89 self.protocol_number = self.get_protocol_number(kwargs.get('protocol_number'), task_protocol=self.protocol) 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

90 

91 self.output_collection = 'alf' 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

92 # Do not use kwargs.get('number', None) -- this will return None if number is 0 

93 if self.protocol_number is not None: 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

94 self.output_collection += f'/task_{self.protocol_number:02}' 11 ctx1aXExuecnjCZ

95 

96 def get_protocol(self, protocol=None, task_collection=None): (empty)

97 """ 

98 Return the task protocol name. 

99 

100 This returns the task protocol based on the task collection. If `protocol` is not None, this 

101 acts as an identity function. If both `task_collection` and `protocol` are None, returns 

102 the protocol defined in the experiment description file only if a single protocol was run. 

103 If the `task_collection` is not None, the associated protocol name is returned. 

104 

105 

106 Parameters 

107 ---------- 

108 protocol : str 

109 A task protocol name. If not None, the same value is returned. 

110 task_collection : str 

111 The task collection whose protocol name to return. May be None if only one protocol run. 

112 

113 Returns 

114 ------- 

115 str, None 

116 The task protocol name, or None, if no protocol found. 

117 

118 Raises 

119 ------ 

120 ValueError 

121 For session with multiple task protocols, a task collection must be passed. 

122 """ 

123 if protocol: 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

124 return protocol 20 ctx1a0XEwxubhiefcknjlmCZ

125 protocol = sess_params.get_task_protocol(self.session_params, task_collection) or None 30 ctx1atwyDFGIopJqKLMrNOszABPQRSTUVW

126 if isinstance(protocol, set): 30 ctx1atwyDFGIopJqKLMrNOszABPQRSTUVW

127 if len(protocol) == 1: 1 ctx1w

128 protocol = next(iter(protocol)) 1 ctx1w

129 else: 

130 raise ValueError('Multiple task protocols for session. Task collection must be explicitly defined.') 1 ctx1w

131 return protocol 30 ctx1atwyDFGIopJqKLMrNOszABPQRSTUVW

132 

133 def get_task_collection(self, collection=None): (empty)

134 """ 

135 Return the task collection. 

136 

137 If `collection` is not None, this acts as an identity function. Otherwise loads it from 

138 the experiment description if only one protocol was run. 

139 

140 Parameters 

141 ---------- 

142 collection : str 

143 A task collection. If not None, the same value is returned. 

144 

145 Returns 

146 ------- 

147 str, None 

148 The task collection, or None if no task protocols were run. 

149 

150 Raises 

151 ------ 

152 AssertionError 

153 Raised if multiple protocols were run and collection is None, or if experiment 

154 description file is improperly formatted. 

155 

156 """ 

157 if not collection: 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

158 collection = sess_params.get_task_collection(self.session_params) 5 ctx1EtwyD

159 # If inferring the collection from the experiment description, assert only one returned 

160 assert collection is None or isinstance(collection, str) or len(collection) == 1 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

161 return collection 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

162 

163 def get_protocol_number(self, number=None, task_protocol=None): (empty)

164 """ 

165 Return the task protocol number. 

166 

167 Numbering starts from 0. If the 'protocol_number' field is missing from the experiment 

168 description, None is returned. If `task_protocol` is None, the first protocol number if n 

169 protocols == 1, otherwise raises an AssertionError. 

170 

171 NB: :func:`ibllib.pipes.dynamic_pipeline.make_pipeline` will determine the protocol number 

172 from the order of the tasks in the experiment description if the task collection follows 

173 the pattern 'raw_task_data_XX'. If the task protocol does not follow this pattern, the 

174 experiment description file should explicitly define the number with the 'protocol_number' 

175 field. 

176 

177 Parameters 

178 ---------- 

179 number : int 

180 The protocol number. If not None, the same value is returned. 

181 task_protocol : str 

182 The task protocol name. 

183 

184 Returns 

185 ------- 

186 int, None 

187 The task protocol number, if defined. 

188 """ 

189 if number is None: # Do not use "if not number" as that will return True if number is 0 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

190 number = sess_params.get_task_protocol_number(self.session_params, task_protocol) 40 ctx1a0twyDxFGIopJqKLMrNOsbhiefklmzABPQRSTUVW

191 elif not isinstance(number, int): 10 ctx1XEyxucnjCZ

192 number = int(number) 1 ctx1y

193 # If inferring the number from the experiment description, assert only one returned (or something went wrong) 

194 assert number is None or isinstance(number, int), 'ambiguous protocol number; no task protocol defined' 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

195 return number 48 ctx1a0XEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCZABPQRSTUVW

196 

197 @staticmethod (empty)

198 def _spacer_support(settings): (empty)

199 """ 

200 Spacer support was introduced in v7.1 for iblrig v7 and v8.0.1 in v8. 

201 

202 Parameters 

203 ---------- 

204 settings : dict 

205 The task settings dict. 

206 

207 Returns 

208 ------- 

209 bool 

210 True if task spacers are to be expected. 

211 """ 

212 v = version.parse 3 ctx2vbz C

213 ver = v(settings.get('IBLRIG_VERSION') or '100.0.0') 3 ctx2vbz C

214 return ver not in (v('100.0.0'), v('8.0.0')) and ver >= v('7.1.0') 3 ctx2vbz C

215 

216 def extract_behaviour(self, save=True): (empty)

217 """Extract trials data. 

218 

219 This is an abstract method called by `_run` and `run_qc` methods. Subclasses should return 

220 the extracted trials data and a list of output files. This method should also save the 

221 trials extractor object to the :prop:`extractor` property for use by `run_qc`. 

222 

223 Parameters 

224 ---------- 

225 save : bool 

226 Whether to save the extracted data as ALF datasets. 

227 

228 Returns 

229 ------- 

230 dict 

231 A dictionary of trials data. 

232 list of pathlib.Path 

233 A list of output file paths if save == true. 

234 """ 

235 return None, None 

236 

237 def run_qc(self, trials_data=None, update=True): (empty)

238 """Run task QC. 

239 

240 Subclass method should return the QC object. This just validates the trials_data is not 

241 None. 

242 

243 Parameters 

244 ---------- 

245 trials_data : dict 

246 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

247 update : bool 

248 If true, update Alyx with the QC outcome. 

249 

250 Returns 

251 ------- 

252 ibllib.qc.task_metrics.TaskQC 

253 A TaskQC object replete with task data and computed metrics. 

254 """ 

255 self._assert_trials_data(trials_data) 

256 return None 

257 

258 def _assert_trials_data(self, trials_data=None): (empty)

259 """Check trials data available. 

260 

261 Called by :meth:`run_qc`, this extracts the trial data if `trials_data` is None, and raises 

262 if :meth:`extract_behaviour` returns None. 

263 

264 Parameters 

265 ---------- 

266 trials_data : dict, None 

267 A dictionary of extracted trials data or None. 

268 

269 Returns 

270 ------- 

271 trials_data : dict 

272 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

273 """ 

274 if not self.extractor or trials_data is None: 5 ctx1atbAB

275 trials_data, _ = self.extract_behaviour(save=False) 2 ctx1at

276 if not (trials_data and self.extractor): 5 ctx1atbAB

277 raise ValueError('No trials data and/or extractor found') 1 ctx1t

278 return trials_data 5 ctx1atbAB

279 

280 

281class VideoTask(DynamicTask): (empty)

282 

283 def __init__(self, session_path, cameras, **kwargs): (empty)

284 super().__init__(session_path, cameras=cameras, **kwargs) 19 ctx1abhiefcknjlm:;=?,@[

285 self.cameras = cameras 19 ctx1abhiefcknjlm:;=?,@[

286 self.device_collection = self.get_device_collection('cameras', kwargs.get('device_collection', 'raw_video_data')) 19 ctx1abhiefcknjlm:;=?,@[

287 # self.collection = self.get_task_collection(kwargs.get('collection', None)) 

288 

289 def extract_camera(self, save=True): (empty)

290 """Extract trials data. 

291 

292 This is an abstract method called by `_run` and `run_qc` methods. Subclasses should return 

293 the extracted trials data and a list of output files. This method should also save the 

294 trials extractor object to the :prop:`extractor` property for use by `run_qc`. 

295 

296 Parameters 

297 ---------- 

298 save : bool 

299 Whether to save the extracted data as ALF datasets. 

300 

301 Returns 

302 ------- 

303 dict 

304 A dictionary of trials data. 

305 list of pathlib.Path 

306 A list of output file paths if save == true. 

307 """ 

308 return None, None 

309 

310 def run_qc(self, camera_data=None, update=True): (empty)

311 """Run camera QC. 

312 

313 Subclass method should return the QC object. This just validates the trials_data is not 

314 None. 

315 

316 Parameters 

317 ---------- 

318 camera_data : dict 

319 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

320 update : bool 

321 If true, update Alyx with the QC outcome. 

322 

323 Returns 

324 ------- 

325 ibllib.qc.task_metrics.TaskQC 

326 A TaskQC object replete with task data and computed metrics. 

327 """ 

328 self._assert_trials_data(camera_data) 

329 return None 

330 

331 

332class AudioTask(DynamicTask): (empty)

333 

334 def __init__(self, session_path, **kwargs): (empty)

335 super().__init__(session_path, **kwargs) 15 ctx1au{-.bhiefckjlm

336 self.device_collection = self.get_device_collection('microphone', kwargs.get('device_collection', 'raw_behavior_data')) 15 ctx1au{-.bhiefckjlm

337 

338 

339class EphysTask(DynamicTask): (empty)

340 

341 def __init__(self, session_path, **kwargs): (empty)

342 super().__init__(session_path, **kwargs) 16 ctx1ahiefc56789!#2$%

343 

344 self.pname = self.get_pname(kwargs.get('pname', None)) 16 ctx1ahiefc56789!#2$%

345 self.nshanks, self.pextra = self.get_nshanks(kwargs.get('nshanks', None)) 16 ctx1ahiefc56789!#2$%

346 self.device_collection = self.get_device_collection('neuropixel', kwargs.get('device_collection', 'raw_ephys_data')) 16 ctx1ahiefc56789!#2$%

347 

348 def get_pname(self, pname): (empty)

349 # pname can be a list or a string 

350 pname = self.kwargs.get('pname', pname) 16 ctx1ahiefc56789!#2$%

351 

352 return pname 16 ctx1ahiefc56789!#2$%

353 

354 def get_nshanks(self, nshanks=None): (empty)

355 nshanks = self.kwargs.get('nshanks', nshanks) 16 ctx1ahiefc56789!#2$%

356 if nshanks is not None: 16 ctx1ahiefc56789!#2$%

357 pextra = [chr(97 + int(shank)) for shank in range(nshanks)] 4 ctx1fc#2

358 else: 

359 pextra = [] 15 ctx1ahiefc56789!2$%

360 

361 return nshanks, pextra 16 ctx1ahiefc56789!#2$%

362 

363 

364class WidefieldTask(DynamicTask): (empty)

365 def __init__(self, session_path, **kwargs): (empty)

366 super().__init__(session_path, **kwargs) 10 ctx2a _ ` m abbbv cbdbeb

367 

368 self.device_collection = self.get_device_collection('widefield', kwargs.get('device_collection', 'raw_widefield_data')) 10 ctx2a _ ` m abbbv cbdbeb

369 

370 

371class MesoscopeTask(DynamicTask): (empty)

372 def __init__(self, session_path, **kwargs): (empty)

373 super().__init__(session_path, **kwargs) 15 ctx1ax]^n()*|}+134~

374 

375 self.device_collection = self.get_device_collection( 15 ctx1ax]^n()*|}+134~

376 'mesoscope', kwargs.get('device_collection', 'raw_imaging_data_[0-9]*')) 

377 

378 def get_signatures(self, **kwargs): (empty)

379 """ 

380 From the template signature of the task, create the exact list of inputs and outputs to expect based on the 

381 available device collection folders 

382 

383 Necessary because we don't know in advance how many device collection folders ("imaging bouts") to expect 

384 """ 

385 self.session_path = Path(self.session_path) 9 ctx2sbtb( ) * + 1 3 4

386 # Glob for all device collection (raw imaging data) folders 

387 raw_imaging_folders = [p.name for p in self.session_path.glob(self.device_collection)] 9 ctx2sbtb( ) * + 1 3 4

388 super().get_signatures(**kwargs) # Set inputs and outputs 9 ctx2sbtb( ) * + 1 3 4

389 # For all inputs and outputs that are part of the device collection, expand to one file per folder 

390 # All others keep unchanged 

391 self.input_files = [ 9 ctx2sbtb( ) * + 1 3 4

392 update_collections(x, raw_imaging_folders, self.device_collection, exact_match=True) for x in self.input_files] 

393 self.output_files = [ 9 ctx2sbtb( ) * + 1 3 4

394 update_collections(x, raw_imaging_folders, self.device_collection, exact_match=True) for x in self.output_files] 

395 

396 def load_sync(self): (empty)

397 """ 

398 Load the sync and channel map. 

399 

400 This method may be expanded to support other raw DAQ data formats. 

401 

402 Returns 

403 ------- 

404 one.alf.io.AlfBunch 

405 A dictionary with keys ('times', 'polarities', 'channels'), containing the sync pulses 

406 and the corresponding channel numbers. 

407 dict 

408 A map of channel names and their corresponding indices. 

409 """ 

410 alf_path = self.session_path / self.sync_collection 2 ctx134

411 if self.get_sync_namespace() == 'timeline': 2 ctx134

412 # Load the sync and channel map from the raw DAQ data 

413 sync, chmap = load_timeline_sync_and_chmap(alf_path) 2 ctx134

414 else: 

415 raise NotImplementedError 

416 return sync, chmap 2 ctx134

417 

418 

419class RegisterRawDataTask(DynamicTask): (empty)

420 """ 

421 Base register raw task. 

422 To rename files 

423 1. input and output must have the same length 

424 2. output files must have full filename 

425 """ 

426 

427 priority = 100 (empty)

428 job_size = 'small' (empty)

429 

430 def rename_files(self, symlink_old=False): (empty)

431 

432 # If either no inputs or no outputs are given, we don't do any renaming 

433 if not all(map(len, (self.input_files, self.output_files))): 12 ctx2Y H gbo p q r s b 1 / v

434 return 10 ctx1YHopqrsb1/

435 

436 # Otherwise we need to make sure there is one to one correspondence for renaming files 

437 assert len(self.input_files) == len(self.output_files) 3 ctx2Y gbv

438 

439 for before, after in zip(self.input_files, self.output_files): 3 ctx2Y gbv

440 ok, old_paths, missing = before.find_files(self.session_path) 3 ctx2Y gbv

441 if not old_paths: 3 ctx2Y gbv

442 if ok: # if the file doesn't exist and it is not required we are okay to continue 1 ctx1Y

443 continue 

444 else: 

445 raise FileNotFoundError(f'file(s) {", ".join(missing)} not found') 1 ctx1Y

446 new_paths = list(map(self.session_path.joinpath, ensure_list(after.glob_pattern))) 3 ctx2Y gbv

447 assert len(old_paths) == len(new_paths) 3 ctx2Y gbv

448 for old_path, new_path in zip(old_paths, new_paths): 3 ctx2Y gbv

449 if old_path == new_path: 3 ctx2Y gbv

450 continue 

451 new_path.parent.mkdir(parents=True, exist_ok=True) 3 ctx2Y gbv

452 _logger.debug('%s -> %s', old_path.relative_to(self.session_path), new_path.relative_to(self.session_path)) 3 ctx2Y gbv

453 old_path.replace(new_path) 3 ctx2Y gbv

454 if symlink_old: 3 ctx2Y gbv

455 old_path.symlink_to(new_path) 1 ctx1v

456 

457 @staticmethod (empty)

458 def _is_animated_gif(snapshot: Path) -> bool: (empty)

459 """ 

460 Test if image is an animated GIF file. 

461 

462 Parameters 

463 ---------- 

464 snapshot : pathlib.Path 

465 An image filepath to test. 

466 

467 Returns 

468 ------- 

469 bool 

470 True if image is an animated GIF. 

471 

472 Notes 

473 ----- 

474 This could be achieved more succinctly with `from PIL import Image; Image.open(snapshot).is_animated`, 

475 however despite being an indirect dependency, the Pillow library is not in the requirements, 

476 whereas skimage is. 

477 """ 

478 return snapshot.suffix == '.gif' and len(ImageCollection(str(snapshot))) > 1 1 ctx1d

479 

480 @staticmethod (empty)

481 def _save_as_png(snapshot: Path) -> Path: (empty)

482 """ 

483 Save an image to PNG format. 

484 

485 Parameters 

486 ---------- 

487 snapshot : pathlib.Path 

488 An image filepath to convert. 

489 

490 Returns 

491 ------- 

492 pathlib.Path 

493 The new PNG image filepath. 

494 """ 

495 img = imread(snapshot, as_gray=True) 1 ctx1d

496 matplotlib.image.imsave(snapshot.with_suffix('.png'), img, cmap='gray') 1 ctx1d

497 return snapshot.with_suffix('.png') 1 ctx1d

498 

499 def register_snapshots(self, unlink=False, collection=None): (empty)

500 """ 

501 Register any photos in the snapshots folder to the session. Typically imaging users will 

502 take numerous photos for reference. Supported extensions: .jpg, .jpeg, .png, .tif, .tiff 

503 

504 If a .txt file with the same name exists in the same location, the contents will be added 

505 to the note text. 

506 

507 Parameters 

508 ---------- 

509 unlink : bool 

510 If true, files are deleted after upload. 

511 collection : str, list, optional 

512 Location of 'snapshots' folder relative to the session path. If None, uses 

513 'device_collection' attribute (if exists) or root session path. 

514 

515 Returns 

516 ------- 

517 list of dict 

518 The newly registered Alyx notes. 

519 

520 Notes 

521 ----- 

522 - Animated GIF files are not resized and therefore may take up significant space on the database. 

523 - TIFF files are converted to PNG format before upload. The original file is not replaced. 

524 - JPEG and PNG files are resized by Alyx. 

525 """ 

526 assert self.one and not self.one.offline, f'{self.__class__.__name__} requires an online ONE instance' 3 ctx1d'g

527 if not self.one.alyx.is_logged_in: 3 ctx1d'g

528 # Register snapshot requires the user field to be set, which may happen before a REST 

529 # query is made. To avoid the user field being None, we authenticate here. If the 

530 # token is cached this will simply set the user and token properties. 

531 self.one.alyx.authenticate() 1 ctx1'

532 

533 collection = getattr(self, 'device_collection', None) if collection is None else collection 3 ctx1d'g

534 collection = collection or '' # If not defined, use no collection 3 ctx1d'g

535 if collection and '*' in collection: 3 ctx1d'g

536 collection = [p.name for p in self.session_path.glob(collection)] 1 ctx1g

537 # Check whether folders on disk contain '*'; this is to stop an infinite recursion 

538 assert not any('*' in c for c in collection), 'folders containing asterisks not supported' 1 ctx1g

539 # If more than one collection exists, register snapshots in each collection 

540 if collection and not isinstance(collection, str): 3 ctx1d'g

541 return flatten(filter(None, [self.register_snapshots(unlink, c) for c in collection])) 1 ctx1g

542 snapshots_path = self.session_path.joinpath(*filter(None, (collection, 'snapshots'))) 3 ctx1d'g

543 if not snapshots_path.exists(): 3 ctx1d'g

544 return 1 ctx1'

545 

546 eid = self.one.path2eid(self.session_path, query_type='remote') 2 ctx1dg

547 if not eid: 2 ctx1dg

548 _logger.warning('Failed to upload snapshots: session not found on Alyx') 

549 return 

550 note = dict(user=self.one.alyx.user, content_type='session', object_id=eid, text='') 2 ctx1dg

551 

552 notes = [] 2 ctx1dg

553 exts = ('.jpg', '.jpeg', '.png', '.tif', '.tiff', '.gif') 2 ctx1dg

554 for snapshot in filter(lambda x: x.suffix.lower() in exts, snapshots_path.glob('*.*')): 2 ctx1dg

555 if snapshot.suffix in ('.tif', '.tiff') and not snapshot.with_suffix('.png').exists(): 2 ctx1dg

556 _logger.debug('converting "%s" to png...', snapshot.relative_to(self.session_path)) 2 ctx1dg

557 snapshot = self._save_as_png(snapshot_tif := snapshot) 2 ctx1dg

558 if unlink: 2 ctx1dg

559 snapshot_tif.unlink() 1 ctx1d

560 _logger.info('Uploading "%s"...', snapshot.relative_to(self.session_path)) 2 ctx1dg

561 if snapshot.with_suffix('.txt').exists(): 2 ctx1dg

562 with open(snapshot.with_suffix('.txt'), 'r') as txt_file: 1 ctx1g

563 note['text'] = txt_file.read().strip() 1 ctx1g

564 else: 

565 note['text'] = '' 2 ctx1dg

566 note['width'] = 'orig' if self._is_animated_gif(snapshot) else None 2 ctx1dg

567 with open(snapshot, 'rb') as img_file: 2 ctx1dg

568 files = {'image': img_file} 2 ctx1dg

569 notes.append(self.one.alyx.rest('notes', 'create', data=note, files=files)) 2 ctx1dg

570 if unlink: 2 ctx1dg

571 snapshot.unlink() 1 ctx1d

572 # If nothing else in the snapshots folder, delete the folder 

573 if unlink and next(snapshots_path.rglob('*'), None) is None: 2 ctx1dg

574 snapshots_path.rmdir() 1 ctx1d

575 _logger.info('%i snapshots uploaded to Alyx', len(notes)) 2 ctx1dg

576 return notes 2 ctx1dg

577 

578 def _run(self, **kwargs): (empty)

579 self.rename_files(**kwargs) 10 ctx1Hopqrsb1/v

580 if not self.output_files: 10 ctx1Hopqrsb1/v

581 return [] 1 ctx11

582 

583 # FIXME Can be done with Task.assert_expected_outputs 

584 ok, out_files, missing = map(flatten, zip(*map(lambda x: x.find_files(self.session_path), self.output_files))) 9 ctx1Hopqrsb/v

585 if not ok: 9 ctx1Hopqrsb/v

586 _logger.error('The following expected are missing: %s', ', '.join(missing)) 

587 self.status = -1 

588 

589 return out_files 9 ctx1Hopqrsb/v

590 

591 

592class ExperimentDescriptionRegisterRaw(RegisterRawDataTask): (empty)

593 """dict of list: custom sign off keys corresponding to specific devices""" 

594 sign_off_categories = SIGN_OFF_CATEGORIES (empty)

595 

596 @property (empty)

597 def signature(self): (empty)

598 signature = { 10 ctx1aHbhiefcnm

599 'input_files': [], 

600 'output_files': [('*experiment.description.yaml', '', True)] 

601 } 

602 return signature 10 ctx1aHbhiefcnm

603 

604 def _run(self, **kwargs): (empty)

605 # Register experiment description file 

606 out_files = super(ExperimentDescriptionRegisterRaw, self)._run(**kwargs) 2 ctx1Hb

607 if not self.one.offline and self.status == 0: 2 ctx1Hb

608 with no_cache(self.one.alyx): # Ensure we don't load the cached JSON response 2 ctx1Hb

609 eid = self.one.path2eid(self.session_path, query_type='remote') 2 ctx1Hb

610 exp_dec = sess_params.read_params(out_files[0]) 2 ctx1Hb

611 data = sign_off_dict(exp_dec, sign_off_categories=self.sign_off_categories) 2 ctx1Hb

612 self.one.alyx.json_field_update('sessions', eid, data=data) 2 ctx1Hb

613 return out_files 2 ctx1Hb