Coverage for ibllib/pipes/base_tasks.py: 95%

1"""Abstract base classes for dynamic pipeline tasks.""" 

2import logging (empty)

3from pathlib import Path (empty)

4 

5from packaging import version (empty)

6from one.webclient import no_cache (empty)

7from iblutil.util import flatten, ensure_list (empty)

8import matplotlib.image (empty)

9from skimage.io import ImageCollection, imread (empty)

10 

11from ibllib.pipes.tasks import Task (empty)

12import ibllib.io.session_params as sess_params (empty)

13from ibllib.qc.base import sign_off_dict, SIGN_OFF_CATEGORIES (empty)

14from ibllib.io.raw_daq_loaders import load_timeline_sync_and_chmap (empty)

15from ibllib.oneibl.data_handlers import update_collections (empty)

16 

17_logger = logging.getLogger(__name__) (empty)

18 

19 

20class DynamicTask(Task): (empty)

21 

22 def __init__(self, session_path, **kwargs): (empty)

23 super().__init__(session_path, **kwargs) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

24 self.session_params = self.read_params_file() 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

25 

26 # TODO Which should be default? 

27 # Sync collection 

28 self.sync_collection = self.get_sync_collection(kwargs.get('sync_collection', None)) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

29 # Sync type 

30 self.sync = self.get_sync(kwargs.get('sync', None)) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

31 # Sync extension 

32 self.sync_ext = self.get_sync_extension(kwargs.get('sync_ext', None)) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

33 # Sync namespace 

34 self.sync_namespace = self.get_sync_namespace(kwargs.get('sync_namespace', None)) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

35 

36 def get_sync_collection(self, sync_collection=None): (empty)

37 return sync_collection if sync_collection else sess_params.get_sync_collection(self.session_params) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

38 

39 def get_sync(self, sync=None): (empty)

40 return sync if sync else sess_params.get_sync_label(self.session_params) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

41 

42 def get_sync_extension(self, sync_ext=None): (empty)

43 return sync_ext if sync_ext else sess_params.get_sync_extension(self.session_params) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

44 

45 def get_sync_namespace(self, sync_namespace=None): (empty)

46 return sync_namespace if sync_namespace else sess_params.get_sync_namespace(self.session_params) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

47 

48 def get_protocol(self, protocol=None, task_collection=None): (empty)

49 return protocol if protocol else sess_params.get_task_protocol(self.session_params, task_collection) 6 ctx1bckjl+

50 

51 def get_task_collection(self, collection=None): (empty)

52 if not collection: 9 ctx1u,-bckjl+

53 collection = sess_params.get_task_collection(self.session_params) 5 ctx1bckjl

54 # If inferring the collection from the experiment description, assert only one returned 

55 assert collection is None or isinstance(collection, str) or len(collection) == 1 9 ctx1u,-bckjl+

56 return collection 9 ctx1u,-bckjl+

57 

58 def get_device_collection(self, device, device_collection=None): (empty)

59 if device_collection: 56 ctx2a x [ ] tbu ^ _ ` , - b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ ' ( ) { | * 0 2 3 } eb/ : ; = + ? @ ~ abv bbcbdb

60 return device_collection 56 ctx2a x [ ] tbu ^ _ ` , - b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ ' ( ) { | * 0 2 3 } eb/ : ; = + ? @ ~ abv bbcbdb

61 collection_map = sess_params.get_collections(self.session_params['devices']) 1 ctx2tb

62 return collection_map.get(device) 1 ctx2tb

63 

64 def read_params_file(self): (empty)

65 params = sess_params.read_params(self.session_path) 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

66 

67 if params is None: 105 ctx2a Z W E t w y D d X x [ ] F G % g H u ^ _ ` , - I o p J q K L M r N O s b h i e f c k n j l m 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | * 0 2 3 } z C ebkblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbY A B P Q R S T U V

68 return {} 84 ctx2a Z W E t w y D d X x [ ] F G % g ^ _ ` , - I o p J q K L M r N O s 4 5 6 7 8 9 ! 1 # $ fbgbhbibjb' ( ) { | } kblbmbnbobpb. / : ; = + ? @ ~ abv bbcbdbA B P Q R S T U V

69 

70 # TODO figure out the best way 

71 # if params is None and self.one: 

72 # # Try to read params from alyx or try to download params file 

73 # params = self.one.load_dataset(self.one.path2eid(self.session_path), 'params.yml') 

74 # params = self.one.alyx.rest() 

75 

76 return params 23 ctx2a x H u b h i e f c k n j l m * 0 2 3 z C ebY

77 

78 

79class BehaviourTask(DynamicTask): (empty)

80 

81 extractor = None (empty)

82 """ibllib.io.extractors.base.BaseBpodExtractor: A trials extractor object.""" (empty)

83 

84 def __init__(self, session_path, **kwargs): (empty)

85 super().__init__(session_path, **kwargs) 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

86 

87 self.collection = self.get_task_collection(kwargs.get('collection', None)) 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

88 # Task type (protocol) 

89 self.protocol = self.get_protocol(kwargs.get('protocol', None), task_collection=self.collection) 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

90 

91 self.protocol_number = self.get_protocol_number(kwargs.get('protocol_number'), task_protocol=self.protocol) 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

92 

93 self.output_collection = 'alf' 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

94 # Do not use kwargs.get('number', None) -- this will return None if number is 0 

95 if self.protocol_number is not None: 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

96 self.output_collection += f'/task_{self.protocol_number:02}' 11 ctx1aWExuecnjCY

97 

98 def get_protocol(self, protocol=None, task_collection=None): (empty)

99 """ 

100 Return the task protocol name. 

101 

102 This returns the task protocol based on the task collection. If `protocol` is not None, this 

103 acts as an identity function. If both `task_collection` and `protocol` are None, returns 

104 the protocol defined in the experiment description file only if a single protocol was run. 

105 If the `task_collection` is not None, the associated protocol name is returned. 

106 

107 

108 Parameters 

109 ---------- 

110 protocol : str 

111 A task protocol name. If not None, the same value is returned. 

112 task_collection : str 

113 The task collection whose protocol name to return. May be None if only one protocol run. 

114 

115 Returns 

116 ------- 

117 str, None 

118 The task protocol name, or None, if no protocol found. 

119 

120 Raises 

121 ------ 

122 ValueError 

123 For session with multiple task protocols, a task collection must be passed. 

124 """ 

125 if protocol: 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

126 return protocol 20 ctx1aZWEwxubhiefcknjlmCY

127 protocol = sess_params.get_task_protocol(self.session_params, task_collection) or None 29 ctx1atwyDFGIopJqKLMrNOszABPQRSTUV

128 if isinstance(protocol, set): 29 ctx1atwyDFGIopJqKLMrNOszABPQRSTUV

129 if len(protocol) == 1: 1 ctx1w

130 protocol = next(iter(protocol)) 1 ctx1w

131 else: 

132 raise ValueError('Multiple task protocols for session. Task collection must be explicitly defined.') 1 ctx1w

133 return protocol 29 ctx1atwyDFGIopJqKLMrNOszABPQRSTUV

134 

135 def get_task_collection(self, collection=None): (empty)

136 """ 

137 Return the task collection. 

138 

139 If `collection` is not None, this acts as an identity function. Otherwise loads it from 

140 the experiment description if only one protocol was run. 

141 

142 Parameters 

143 ---------- 

144 collection : str 

145 A task collection. If not None, the same value is returned. 

146 

147 Returns 

148 ------- 

149 str, None 

150 The task collection, or None if no task protocols were run. 

151 

152 Raises 

153 ------ 

154 AssertionError 

155 Raised if multiple protocols were run and collection is None, or if experiment 

156 description file is improperly formatted. 

157 

158 """ 

159 if not collection: 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

160 collection = sess_params.get_task_collection(self.session_params) 5 ctx1EtwyD

161 # If inferring the collection from the experiment description, assert only one returned 

162 assert collection is None or isinstance(collection, str) or len(collection) == 1 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

163 return collection 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

164 

165 def get_protocol_number(self, number=None, task_protocol=None): (empty)

166 """ 

167 Return the task protocol number. 

168 

169 Numbering starts from 0. If the 'protocol_number' field is missing from the experiment 

170 description, None is returned. If `task_protocol` is None, the first protocol number if n 

171 protocols == 1, otherwise raises an AssertionError. 

172 

173 NB: :func:`ibllib.pipes.dynamic_pipeline.make_pipeline` will determine the protocol number 

174 from the order of the tasks in the experiment description if the task collection follows 

175 the pattern 'raw_task_data_XX'. If the task protocol does not follow this pattern, the 

176 experiment description file should explicitly define the number with the 'protocol_number' 

177 field. 

178 

179 Parameters 

180 ---------- 

181 number : int 

182 The protocol number. If not None, the same value is returned. 

183 task_protocol : str 

184 The task protocol name. 

185 

186 Returns 

187 ------- 

188 int, None 

189 The task protocol number, if defined. 

190 """ 

191 if number is None: # Do not use "if not number" as that will return True if number is 0 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

192 number = sess_params.get_task_protocol_number(self.session_params, task_protocol) 39 ctx1aZtwyDxFGIopJqKLMrNOsbhiefklmzABPQRSTUV

193 elif not isinstance(number, int): 10 ctx1WEyxucnjCY

194 number = int(number) 1 ctx1y

195 # If inferring the number from the experiment description, assert only one returned (or something went wrong) 

196 assert number is None or isinstance(number, int), 'ambiguous protocol number; no task protocol defined' 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

197 return number 47 ctx1aZWEtwyDxFGuIopJqKLMrNOsbhiefcknjlmzCYABPQRSTUV

198 

199 @staticmethod (empty)

200 def _spacer_support(settings): (empty)

201 """ 

202 Spacer support was introduced in v7.1 for iblrig v7 and v8.0.1 in v8. 

203 

204 Parameters 

205 ---------- 

206 settings : dict 

207 The task settings dict. 

208 

209 Returns 

210 ------- 

211 bool 

212 True if task spacers are to be expected. 

213 """ 

214 v = version.parse 3 ctx2ubz C

215 ver = v(settings.get('IBLRIG_VERSION') or '100.0.0') 3 ctx2ubz C

216 return ver not in (v('100.0.0'), v('8.0.0')) and ver >= v('7.1.0') 3 ctx2ubz C

217 

218 def extract_behaviour(self, save=True): (empty)

219 """Extract trials data. 

220 

221 This is an abstract method called by `_run` and `run_qc` methods. Subclasses should return 

222 the extracted trials data and a list of output files. This method should also save the 

223 trials extractor object to the :prop:`extractor` property for use by `run_qc`. 

224 

225 Parameters 

226 ---------- 

227 save : bool 

228 Whether to save the extracted data as ALF datasets. 

229 

230 Returns 

231 ------- 

232 dict 

233 A dictionary of trials data. 

234 list of pathlib.Path 

235 A list of output file paths if save == true. 

236 """ 

237 return None, None 

238 

239 def run_qc(self, trials_data=None, update=True): (empty)

240 """Run task QC. 

241 

242 Subclass method should return the QC object. This just validates the trials_data is not 

243 None. 

244 

245 Parameters 

246 ---------- 

247 trials_data : dict 

248 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

249 update : bool 

250 If true, update Alyx with the QC outcome. 

251 

252 Returns 

253 ------- 

254 ibllib.qc.task_metrics.TaskQC 

255 A TaskQC object replete with task data and computed metrics. 

256 """ 

257 self._assert_trials_data(trials_data) 

258 return None 

259 

260 def _assert_trials_data(self, trials_data=None): (empty)

261 """Check trials data available. 

262 

263 Called by :meth:`run_qc`, this extracts the trial data if `trials_data` is None, and raises 

264 if :meth:`extract_behaviour` returns None. 

265 

266 Parameters 

267 ---------- 

268 trials_data : dict, None 

269 A dictionary of extracted trials data or None. 

270 

271 Returns 

272 ------- 

273 trials_data : dict 

274 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

275 """ 

276 if not self.extractor or trials_data is None: 5 ctx1atbAB

277 trials_data, _ = self.extract_behaviour(save=False) 2 ctx1at

278 if not (trials_data and self.extractor): 5 ctx1atbAB

279 raise ValueError('No trials data and/or extractor found') 1 ctx1t

280 return trials_data 5 ctx1atbAB

281 

282 

283class VideoTask(DynamicTask): (empty)

284 

285 def __init__(self, session_path, cameras, **kwargs): (empty)

286 super().__init__(session_path, cameras=cameras, **kwargs) 19 ctx1abhiefcknjlm/:;=+?@

287 self.cameras = cameras 19 ctx1abhiefcknjlm/:;=+?@

288 self.device_collection = self.get_device_collection('cameras', kwargs.get('device_collection', 'raw_video_data')) 19 ctx1abhiefcknjlm/:;=+?@

289 # self.collection = self.get_task_collection(kwargs.get('collection', None)) 

290 

291 def extract_camera(self, save=True): (empty)

292 """Extract trials data. 

293 

294 This is an abstract method called by `_run` and `run_qc` methods. Subclasses should return 

295 the extracted trials data and a list of output files. This method should also save the 

296 trials extractor object to the :prop:`extractor` property for use by `run_qc`. 

297 

298 Parameters 

299 ---------- 

300 save : bool 

301 Whether to save the extracted data as ALF datasets. 

302 

303 Returns 

304 ------- 

305 dict 

306 A dictionary of trials data. 

307 list of pathlib.Path 

308 A list of output file paths if save == true. 

309 """ 

310 return None, None 

311 

312 def run_qc(self, camera_data=None, update=True): (empty)

313 """Run camera QC. 

314 

315 Subclass method should return the QC object. This just validates the trials_data is not 

316 None. 

317 

318 Parameters 

319 ---------- 

320 camera_data : dict 

321 A dictionary of extracted trials data. The output of :meth:`extract_behaviour`. 

322 update : bool 

323 If true, update Alyx with the QC outcome. 

324 

325 Returns 

326 ------- 

327 ibllib.qc.task_metrics.TaskQC 

328 A TaskQC object replete with task data and computed metrics. 

329 """ 

330 self._assert_trials_data(camera_data) 

331 return None 

332 

333 

334class AudioTask(DynamicTask): (empty)

335 

336 def __init__(self, session_path, **kwargs): (empty)

337 super().__init__(session_path, **kwargs) 15 ctx1au`,-bhiefckjlm

338 self.device_collection = self.get_device_collection('microphone', kwargs.get('device_collection', 'raw_behavior_data')) 15 ctx1au`,-bhiefckjlm

339 

340 

341class EphysTask(DynamicTask): (empty)

342 

343 def __init__(self, session_path, **kwargs): (empty)

344 super().__init__(session_path, **kwargs) 16 ctx1ahiefc456789!1#$

345 

346 self.pname = self.get_pname(kwargs.get('pname', None)) 16 ctx1ahiefc456789!1#$

347 self.nshanks, self.pextra = self.get_nshanks(kwargs.get('nshanks', None)) 16 ctx1ahiefc456789!1#$

348 self.device_collection = self.get_device_collection('neuropixel', kwargs.get('device_collection', 'raw_ephys_data')) 16 ctx1ahiefc456789!1#$

349 

350 def get_pname(self, pname): (empty)

351 # pname can be a list or a string 

352 pname = self.kwargs.get('pname', pname) 16 ctx1ahiefc456789!1#$

353 

354 return pname 16 ctx1ahiefc456789!1#$

355 

356 def get_nshanks(self, nshanks=None): (empty)

357 nshanks = self.kwargs.get('nshanks', nshanks) 16 ctx1ahiefc456789!1#$

358 if nshanks is not None: 16 ctx1ahiefc456789!1#$

359 pextra = [chr(97 + int(shank)) for shank in range(nshanks)] 4 ctx1fc!1

360 else: 

361 pextra = [] 15 ctx1ahiefc4567891#$

362 

363 return nshanks, pextra 16 ctx1ahiefc456789!1#$

364 

365 

366class WidefieldTask(DynamicTask): (empty)

367 def __init__(self, session_path, **kwargs): (empty)

368 super().__init__(session_path, **kwargs) 10 ctx2a ^ _ m ~ abv bbcbdb

369 

370 self.device_collection = self.get_device_collection('widefield', kwargs.get('device_collection', 'raw_widefield_data')) 10 ctx2a ^ _ m ~ abv bbcbdb

371 

372 

373class MesoscopeTask(DynamicTask): (empty)

374 def __init__(self, session_path, **kwargs): (empty)

375 super().__init__(session_path, **kwargs) 15 ctx1ax[]n'(){|*023}

376 

377 self.device_collection = self.get_device_collection( 15 ctx1ax[]n'(){|*023}

378 'mesoscope', kwargs.get('device_collection', 'raw_imaging_data_[0-9]*')) 

379 

380 def get_signatures(self, **kwargs): (empty)

381 """ 

382 From the template signature of the task, create the exact list of inputs and outputs to expect based on the 

383 available device collection folders 

384 

385 Necessary because we don't know in advance how many device collection folders ("imaging bouts") to expect 

386 """ 

387 self.session_path = Path(self.session_path) 9 ctx2rbsb' ( ) * 0 2 3

388 # Glob for all device collection (raw imaging data) folders 

389 raw_imaging_folders = [p.name for p in self.session_path.glob(self.device_collection)] 9 ctx2rbsb' ( ) * 0 2 3

390 super().get_signatures(**kwargs) # Set inputs and outputs 9 ctx2rbsb' ( ) * 0 2 3

391 # For all inputs and outputs that are part of the device collection, expand to one file per folder 

392 # All others keep unchanged 

393 self.input_files = [update_collections(x, raw_imaging_folders, self.device_collection) for x in self.input_files] 9 ctx2rbsb' ( ) * 0 2 3

394 self.output_files = [update_collections(x, raw_imaging_folders, self.device_collection) for x in self.output_files] 9 ctx2rbsb' ( ) * 0 2 3

395 

396 def load_sync(self): (empty)

397 """ 

398 Load the sync and channel map. 

399 

400 This method may be expanded to support other raw DAQ data formats. 

401 

402 Returns 

403 ------- 

404 one.alf.io.AlfBunch 

405 A dictionary with keys ('times', 'polarities', 'channels'), containing the sync pulses 

406 and the corresponding channel numbers. 

407 dict 

408 A map of channel names and their corresponding indices. 

409 """ 

410 alf_path = self.session_path / self.sync_collection 2 ctx123

411 if self.get_sync_namespace() == 'timeline': 2 ctx123

412 # Load the sync and channel map from the raw DAQ data 

413 sync, chmap = load_timeline_sync_and_chmap(alf_path) 2 ctx123

414 else: 

415 raise NotImplementedError 

416 return sync, chmap 2 ctx123

417 

418 

419class RegisterRawDataTask(DynamicTask): (empty)

420 """ 

421 Base register raw task. 

422 To rename files 

423 1. input and output must have the same length 

424 2. output files must have full filename 

425 """ 

426 

427 priority = 100 (empty)

428 job_size = 'small' (empty)

429 

430 def rename_files(self, symlink_old=False): (empty)

431 

432 # If either no inputs or no outputs are given, we don't do any renaming 

433 if not all(map(len, (self.input_files, self.output_files))): 12 ctx2X H qbo p q r s b 0 . v

434 return 10 ctx1XHopqrsb0.

435 

436 # Otherwise we need to make sure there is one to one correspondence for renaming files 

437 assert len(self.input_files) == len(self.output_files) 3 ctx2X qbv

438 

439 for before, after in zip(self.input_files, self.output_files): 3 ctx2X qbv

440 ok, old_paths, missing = before.find_files(self.session_path) 3 ctx2X qbv

441 if not old_paths: 3 ctx2X qbv

442 if ok: # if the file doesn't exist and it is not required we are okay to continue 1 ctx1X

443 continue 

444 else: 

445 raise FileNotFoundError(f'file(s) {", ".join(missing)} not found') 1 ctx1X

446 new_paths = list(map(self.session_path.joinpath, ensure_list(after.glob_pattern))) 3 ctx2X qbv

447 assert len(old_paths) == len(new_paths) 3 ctx2X qbv

448 for old_path, new_path in zip(old_paths, new_paths): 3 ctx2X qbv

449 if old_path == new_path: 3 ctx2X qbv

450 continue 

451 new_path.parent.mkdir(parents=True, exist_ok=True) 3 ctx2X qbv

452 _logger.debug('%s -> %s', old_path.relative_to(self.session_path), new_path.relative_to(self.session_path)) 3 ctx2X qbv

453 old_path.replace(new_path) 3 ctx2X qbv

454 if symlink_old: 3 ctx2X qbv

455 old_path.symlink_to(new_path) 1 ctx1v

456 

457 @staticmethod (empty)

458 def _is_animated_gif(snapshot: Path) -> bool: (empty)

459 """ 

460 Test if image is an animated GIF file. 

461 

462 Parameters 

463 ---------- 

464 snapshot : pathlib.Path 

465 An image filepath to test. 

466 

467 Returns 

468 ------- 

469 bool 

470 True if image is an animated GIF. 

471 

472 Notes 

473 ----- 

474 This could be achieved more succinctly with `from PIL import Image; Image.open(snapshot).is_animated`, 

475 however despite being an indirect dependency, the Pillow library is not in the requirements, 

476 whereas skimage is. 

477 """ 

478 return snapshot.suffix == '.gif' and len(ImageCollection(str(snapshot))) > 1 1 ctx1d

479 

480 @staticmethod (empty)

481 def _save_as_png(snapshot: Path) -> Path: (empty)

482 """ 

483 Save an image to PNG format. 

484 

485 Parameters 

486 ---------- 

487 snapshot : pathlib.Path 

488 An image filepath to convert. 

489 

490 Returns 

491 ------- 

492 pathlib.Path 

493 The new PNG image filepath. 

494 """ 

495 img = imread(snapshot, as_gray=True) 1 ctx1d

496 matplotlib.image.imsave(snapshot.with_suffix('.png'), img, cmap='gray') 1 ctx1d

497 return snapshot.with_suffix('.png') 1 ctx1d

498 

499 def register_snapshots(self, unlink=False, collection=None): (empty)

500 """ 

501 Register any photos in the snapshots folder to the session. Typically imaging users will 

502 take numerous photos for reference. Supported extensions: .jpg, .jpeg, .png, .tif, .tiff 

503 

504 If a .txt file with the same name exists in the same location, the contents will be added 

505 to the note text. 

506 

507 Parameters 

508 ---------- 

509 unlink : bool 

510 If true, files are deleted after upload. 

511 collection : str, list, optional 

512 Location of 'snapshots' folder relative to the session path. If None, uses 

513 'device_collection' attribute (if exists) or root session path. 

514 

515 Returns 

516 ------- 

517 list of dict 

518 The newly registered Alyx notes. 

519 

520 Notes 

521 ----- 

522 - Animated GIF files are not resized and therefore may take up significant space on the database. 

523 - TIFF files are converted to PNG format before upload. The original file is not replaced. 

524 - JPEG and PNG files are resized by Alyx. 

525 """ 

526 assert self.one and not self.one.offline, f'{self.__class__.__name__} requires an online ONE instance' 3 ctx1d%g

527 if not self.one.alyx.is_logged_in: 3 ctx1d%g

528 # Register snapshot requires the user field to be set, which may happen before a REST 

529 # query is made. To avoid the user field being None, we authenticate here. If the 

530 # token is cached this will simply set the user and token properties. 

531 self.one.alyx.authenticate() 1 ctx1%

532 

533 collection = getattr(self, 'device_collection', None) if collection is None else collection 3 ctx1d%g

534 collection = collection or '' # If not defined, use no collection 3 ctx1d%g

535 if collection and '*' in collection: 3 ctx1d%g

536 collection = [p.name for p in self.session_path.glob(collection)] 1 ctx1g

537 # Check whether folders on disk contain '*'; this is to stop an infinite recursion 

538 assert not any('*' in c for c in collection), 'folders containing asterisks not supported' 1 ctx1g

539 # If more than one collection exists, register snapshots in each collection 

540 if collection and not isinstance(collection, str): 3 ctx1d%g

541 return flatten(filter(None, [self.register_snapshots(unlink, c) for c in collection])) 1 ctx1g

542 snapshots_path = self.session_path.joinpath(*filter(None, (collection, 'snapshots'))) 3 ctx1d%g

543 if not snapshots_path.exists(): 3 ctx1d%g

544 return 1 ctx1%

545 

546 eid = self.one.path2eid(self.session_path, query_type='remote') 2 ctx1dg

547 if not eid: 2 ctx1dg

548 _logger.warning('Failed to upload snapshots: session not found on Alyx') 

549 return 

550 note = dict(user=self.one.alyx.user, content_type='session', object_id=eid, text='') 2 ctx1dg

551 

552 notes = [] 2 ctx1dg

553 exts = ('.jpg', '.jpeg', '.png', '.tif', '.tiff', '.gif') 2 ctx1dg

554 for snapshot in filter(lambda x: x.suffix.lower() in exts, snapshots_path.glob('*.*')): 2 ctx1dg

555 if snapshot.suffix in ('.tif', '.tiff') and not snapshot.with_suffix('.png').exists(): 2 ctx1dg

556 _logger.debug('converting "%s" to png...', snapshot.relative_to(self.session_path)) 2 ctx1dg

557 snapshot = self._save_as_png(snapshot_tif := snapshot) 2 ctx1dg

558 if unlink: 2 ctx1dg

559 snapshot_tif.unlink() 1 ctx1d

560 _logger.info('Uploading "%s"...', snapshot.relative_to(self.session_path)) 2 ctx1dg

561 if snapshot.with_suffix('.txt').exists(): 2 ctx1dg

562 with open(snapshot.with_suffix('.txt'), 'r') as txt_file: 1 ctx1g

563 note['text'] = txt_file.read().strip() 1 ctx1g

564 else: 

565 note['text'] = '' 2 ctx1dg

566 note['width'] = 'orig' if self._is_animated_gif(snapshot) else None 2 ctx1dg

567 with open(snapshot, 'rb') as img_file: 2 ctx1dg

568 files = {'image': img_file} 2 ctx1dg

569 notes.append(self.one.alyx.rest('notes', 'create', data=note, files=files)) 2 ctx1dg

570 if unlink: 2 ctx1dg

571 snapshot.unlink() 1 ctx1d

572 # If nothing else in the snapshots folder, delete the folder 

573 if unlink and next(snapshots_path.rglob('*'), None) is None: 2 ctx1dg

574 snapshots_path.rmdir() 1 ctx1d

575 _logger.info('%i snapshots uploaded to Alyx', len(notes)) 2 ctx1dg

576 return notes 2 ctx1dg

577 

578 def _run(self, **kwargs): (empty)

579 self.rename_files(**kwargs) 10 ctx1Hopqrsb0.v

580 if not self.output_files: 10 ctx1Hopqrsb0.v

581 return [] 1 ctx10

582 

583 # FIXME Can be done with Task.assert_expected_outputs 

584 ok, out_files, missing = map(flatten, zip(*map(lambda x: x.find_files(self.session_path), self.output_files))) 9 ctx1Hopqrsb.v

585 if not ok: 9 ctx1Hopqrsb.v

586 _logger.error('The following expected are missing: %s', ', '.join(missing)) 

587 self.status = -1 

588 

589 return out_files 9 ctx1Hopqrsb.v

590 

591 

592class ExperimentDescriptionRegisterRaw(RegisterRawDataTask): (empty)

593 """dict of list: custom sign off keys corresponding to specific devices""" 

594 sign_off_categories = SIGN_OFF_CATEGORIES (empty)

595 

596 @property (empty)

597 def signature(self): (empty)

598 signature = { 10 ctx1aHbhiefcnm

599 'input_files': [], 

600 'output_files': [('*experiment.description.yaml', '', True)] 

601 } 

602 return signature 10 ctx1aHbhiefcnm

603 

604 def _run(self, **kwargs): (empty)

605 # Register experiment description file 

606 out_files = super(ExperimentDescriptionRegisterRaw, self)._run(**kwargs) 2 ctx1Hb

607 if not self.one.offline and self.status == 0: 2 ctx1Hb

608 with no_cache(self.one.alyx): # Ensure we don't load the cached JSON response 2 ctx1Hb

609 eid = self.one.path2eid(self.session_path, query_type='remote') 2 ctx1Hb

610 exp_dec = sess_params.read_params(out_files[0]) 2 ctx1Hb

611 data = sign_off_dict(exp_dec, sign_off_categories=self.sign_off_categories) 2 ctx1Hb

612 self.one.alyx.json_field_update('sessions', eid, data=data) 2 ctx1Hb

613 return out_files 2 ctx1Hb