Coverage for ibllib/qc/camera.py: 83%

1"""Video quality control. 

2 

3This module runs a list of quality control metrics on the camera and extracted video data. 

4 

5Examples 

6-------- 

7Run right camera QC, downloading all but video file 

8 

9>>> qc = CameraQC(eid, 'right', download_data=True, stream=True) 

10>>> qc.run() 

11 

12Run left camera QC with session path, update QC field in Alyx 

13 

14>>> qc = CameraQC(session_path, 'left') 

15>>> outcome, extended = qc.run(update=True) # Returns outcome of videoQC only 

16>>> print(f'video QC = {outcome}; overall session QC = {qc.outcome}') # NB difference outcomes 

17 

18Run only video QC (no timestamp/alignment checks) on 20 frames for the body camera 

19 

20>>> qc = CameraQC(eid, 'body', n_samples=20) 

21>>> qc.load_video_data() # Quicker than loading all data 

22>>> qc.run() 

23 

24Run specific video QC check and display the plots 

25 

26>>> qc = CameraQC(eid, 'left') 

27>>> qc.load_data(download_data=True) 

28>>> qc.check_position(display=True) # NB: Not all checks make plots 

29 

30Run the QC for all cameras 

31 

32>>> qcs = run_all_qc(eid) 

33>>> qcs['left'].metrics # Dict of checks and outcomes for left camera 

34""" 

35import logging (empty)

36from inspect import getmembers, isfunction (empty)

37from pathlib import Path (empty)

38from itertools import chain (empty)

39import copy (empty)

40 

41import cv2 (empty)

42import numpy as np (empty)

43import matplotlib.pyplot as plt (empty)

44from matplotlib.patches import Rectangle (empty)

45 

46import one.alf.io as alfio (empty)

47from one.alf import spec (empty)

48from one.alf.exceptions import ALFObjectNotFound (empty)

49from iblutil.util import Bunch (empty)

50from iblutil.numerical import within_ranges (empty)

51 

52import ibllib.io.extractors.camera as camio (empty)

53from ibllib.io.extractors import ephys_fpga, training_wheel, mesoscope (empty)

54from ibllib.io.extractors.video_motion import MotionAlignment (empty)

55from ibllib.io import raw_data_loaders as raw (empty)

56from ibllib.io.raw_daq_loaders import load_timeline_sync_and_chmap (empty)

57from ibllib.io.session_params import read_params, get_sync, get_sync_namespace (empty)

58import brainbox.behavior.wheel as wh (empty)

59from ibllib.io.video import get_video_meta, get_video_frames_preload, assert_valid_label (empty)

60from . import base (empty)

61 

62_log = logging.getLogger(__name__) (empty)

63 

64try: (empty)

65 from labcams import parse_cam_log (empty)

66except ImportError: 

67 _log.warning('labcams not installed') 

68 

69 

70class CameraQC(base.QC): (empty)

71 """A class for computing camera QC metrics.""" 

72 

73 dstypes = [ (empty)

74 '_ibl_experiment.description', 

75 '_iblrig_Camera.frameData', # Replaces the next 3 datasets 

76 '_iblrig_Camera.frame_counter', 

77 '_iblrig_Camera.GPIO', 

78 '_iblrig_Camera.timestamps', 

79 '_iblrig_taskData.raw', 

80 '_iblrig_taskSettings.raw', 

81 '_iblrig_Camera.raw', 

82 'camera.times', 

83 'wheel.position', 

84 'wheel.timestamps' 

85 ] 

86 dstypes_fpga = [ (empty)

87 '_spikeglx_sync.channels', 

88 '_spikeglx_sync.polarities', 

89 '_spikeglx_sync.times', 

90 'ephysData.raw.meta' 

91 ] 

92 """Recall that for the training rig there is only one side camera at 30 Hz and 1280 x 1024 px. (empty)

93 For the recording rig there are two label cameras (left: 60 Hz, 1280 x 1024 px; 

94 right: 150 Hz, 640 x 512 px) and one body camera (30 Hz, 640 x 512 px). """ 

95 video_meta = { (empty)

96 'training': { 

97 'left': { 

98 'fps': 30, 

99 'width': 1280, 

100 'height': 1024 

101 } 

102 }, 

103 'ephys': { 

104 'left': { 

105 'fps': 60, 

106 'width': 1280, 

107 'height': 1024 

108 }, 

109 'right': { 

110 'fps': 150, 

111 'width': 640, 

112 'height': 512 

113 }, 

114 'body': { 

115 'fps': 30, 

116 'width': 640, 

117 'height': 512 

118 }, 

119 } 

120 } 

121 

122 def __init__(self, session_path_or_eid, camera, **kwargs): (empty)

123 """ 

124 Compute and view camera QC metrics. 

125 

126 :param session_path_or_eid: A session id or path 

127 :param camera: The camera to run QC on, if None QC is run for all three cameras 

128 :param n_samples: The number of frames to sample for the position and brightness QC 

129 :param stream: If true and local video files not available, the data are streamed from 

130 the remote source. 

131 :param log: A logging.Logger instance, if None the 'ibllib' logger is used 

132 :param one: An ONE instance for fetching and setting the QC on Alyx 

133 """ 

134 self.stream = kwargs.pop('stream', None) 7 ctx1gcadihb

135 self.n_samples = kwargs.pop('n_samples', 100) 7 ctx1gcadihb

136 self.sync_collection = kwargs.pop('sync_collection', None) 7 ctx1gcadihb

137 self.sync = kwargs.pop('sync_type', None) 7 ctx1gcadihb

138 self.protocol = kwargs.pop('protocol', None) 7 ctx1gcadihb

139 super().__init__(session_path_or_eid, **kwargs) 7 ctx1gcadihb

140 

141 # Data 

142 self.label = assert_valid_label(camera) 7 ctx1gcadihb

143 filename = f'_iblrig_{self.label}Camera.raw*.mp4' 7 ctx1gcadihb

144 raw_video_path = self.session_path.joinpath('raw_video_data') 7 ctx1gcadihb

145 self.video_path = next(raw_video_path.glob(filename), None) 7 ctx1gcadihb

146 

147 # If local video doesn't exist, change video path to URL 

148 if not self.video_path and self.stream is not False and self.one is not None: 7 ctx1gcadihb

149 try: 

150 self.stream = True 

151 self.video_path = self.one.path2url(raw_video_path / filename.replace('*', '')) 

152 except (StopIteration, ALFObjectNotFound): 

153 _log.error('No remote or local video file found') 

154 self.video_path = None 

155 

156 logging.disable(logging.NOTSET) 7 ctx1gcadihb

157 keys = ('count', 'pin_state', 'audio', 'fpga_times', 'wheel', 'video', 7 ctx1gcadihb

158 'frame_samples', 'timestamps', 'camera_times', 'bonsai_times') 

159 self.data = Bunch.fromkeys(keys) 7 ctx1gcadihb

160 self.frame_samples_idx = None 7 ctx1gcadihb

161 

162 # QC outcomes map 

163 self.metrics = None 7 ctx1gcadihb

164 self.outcome = spec.QC.NOT_SET 7 ctx1gcadihb

165 

166 # Specify any checks to remove 

167 if self.protocol is not None and 'habituation' in self.protocol: 7 ctx1gcadihb

168 self.checks_to_remove = ['check_wheel_alignment'] 1 ctx1d

169 else: 

170 self.checks_to_remove = [] 6 ctx1gcaihb

171 self._type = None 7 ctx1gcadihb

172 

173 @property (empty)

174 def type(self): (empty)

175 """ 

176 Returns the camera type based on the protocol. 

177 

178 :return: Returns either None, 'ephys' or 'training' 

179 """ 

180 if not self._type: 9 ctx1uspcadhbf

181 return 

182 else: 

183 return 'ephys' if 'ephys' in self._type else 'training' 9 ctx1uspcadhbf

184 

185 def load_data(self, extract_times: bool = False, load_video: bool = True) -> None: (empty)

186 """Extract the data from raw data files. 

187 

188 Extracts all the required task data from the raw data files. Missing data will raise an 

189 AssertionError. TODO This method could instead be moved to CameraExtractor classes. 

190 

191 Data keys: 

192 - count (int array): the sequential frame number (n, n+1, n+2...) 

193 - pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame 

194 - audio (float array): timestamps of audio TTL fronts 

195 - fpga_times (float array): timestamps of camera TTLs recorded by FPGA 

196 - timestamps (float array): extracted video timestamps (the camera.times ALF) 

197 - bonsai_times (datetime array): system timestamps of video PC; should be one per frame 

198 - camera_times (float array): camera frame timestamps extracted from frame headers 

199 - wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion 

200 - video (Bunch): video meta data, including dimensions and FPS 

201 - frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel) 

202 

203 :param extract_times: if True, the camera.times are re-extracted from the raw data 

204 :param load_video: if True, calls the load_video_data method 

205 """ 

206 assert self.session_path, 'no session path set' 6 ctx1cadihb

207 _log.info('Gathering data for QC') 6 ctx1cadihb

208 

209 # Get frame count and pin state 

210 self.data['count'], self.data['pin_state'] = \ 6 ctx1cadihb

211 raw.load_embedded_frame_data(self.session_path, self.label, raw=True) 

212 

213 # If there is an experiment description and there are video parameters 

214 sess_params = read_params(self.session_path) or {} 6 ctx1cadihb

215 task_collection = get_task_collection(sess_params) 6 ctx1cadihb

216 ns = get_sync_namespace(sess_params) 6 ctx1cadihb

217 self._set_sync(sess_params) 6 ctx1cadihb

218 self._update_meta_from_session_params(sess_params) 6 ctx1cadihb

219 

220 # Load the audio and raw FPGA times 

221 sync = chmap = None 6 ctx1cadihb

222 if self.sync != 'bpod' and self.sync is not None: 6 ctx1cadihb

223 self.sync_collection = self.sync_collection or 'raw_ephys_data' 2 ctx1ah

224 ns = ns or 'spikeglx' 2 ctx1ah

225 if ns == 'spikeglx': 2 ctx1ah

226 sync, chmap = ephys_fpga.get_sync_and_chn_map(self.session_path, self.sync_collection) 2 ctx1ah

227 elif ns == 'timeline': 1 ctx1h

228 sync, chmap = load_timeline_sync_and_chmap(self.session_path / self.sync_collection) 

229 else: 

230 raise NotImplementedError(f'Unknown namespace "{ns}"') 1 ctx1h

231 audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio']) 2 ctx1ah

232 self.data['audio'] = audio_ttls['times'] # Get rises 2 ctx1ah

233 # Load raw FPGA times 

234 cam_ts = camio.extract_camera_sync(sync, chmap) 2 ctx1ah

235 self.data['fpga_times'] = cam_ts[self.label] 2 ctx1ah

236 else: 

237 self.sync_collection = self.sync_collection or task_collection 5 ctx1cdihb

238 bpod_data = raw.load_data(self.session_path, task_collection) 5 ctx1cdihb

239 _, audio_ttls = raw.load_bpod_fronts( 5 ctx1cdihb

240 self.session_path, data=bpod_data, task_collection=task_collection) 

241 self.data['audio'] = audio_ttls['times'] 5 ctx1cdihb

242 

243 # Load extracted frame times 

244 alf_path = self.session_path / 'alf' 6 ctx1cadihb

245 try: 6 ctx1cadihb

246 assert not extract_times 6 ctx1cadihb

247 self.data['timestamps'] = alfio.load_object( 3 ctx1cih

248 alf_path, f'{self.label}Camera', short_keys=True)['times'] 

249 except AssertionError: # Re-extract 4 ctx1adib

250 kwargs = dict(video_path=self.video_path, save=False) 3 ctx1adb

251 if self.sync == 'bpod': 3 ctx1adb

252 extractor = camio.CameraTimestampsBpod(self.session_path) 2 ctx1db

253 kwargs['task_collection'] = task_collection 2 ctx1db

254 else: 

255 kwargs.update(sync=sync, chmap=chmap) 1 ctx1a

256 extractor = camio.CameraTimestampsFPGA(self.label, self.session_path) 1 ctx1a

257 self.data['timestamps'] = extractor.extract(**kwargs)[0] 3 ctx1adb

258 except ALFObjectNotFound: 1 ctx1i

259 _log.warning('no camera.times ALF found for session') 1 ctx1i

260 

261 # Get audio and wheel data 

262 wheel_keys = ('timestamps', 'position') 6 ctx1cadihb

263 try: 6 ctx1cadihb

264 # glob in case wheel data are in sub-collections 

265 alf_path = next(alf_path.rglob('*wheel.timestamps*')).parent 6 ctx1cadihb

266 self.data['wheel'] = alfio.load_object(alf_path, 'wheel', short_keys=True) 4 ctx1cahb

267 except (StopIteration, ALFObjectNotFound): 2 ctx1di

268 # Extract from raw data 

269 if self.sync != 'bpod' and self.sync is not None: 2 ctx1di

270 if ns == 'spikeglx': 

271 wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap) 

272 elif ns == 'timeline': 

273 extractor = mesoscope.TimelineTrials(self.session_path, sync_collection=self.sync_collection) 

274 wheel_data = extractor.extract_wheel_sync() 

275 else: 

276 raise NotImplementedError(f'Unknown namespace "{ns}"') 

277 else: 

278 if self.protocol is not None and 'habituation' in self.protocol: 2 ctx1di

279 wheel_data = training_wheel.get_wheel_position( 1 ctx1d

280 self.session_path, task_collection=task_collection) 

281 else: 

282 wheel_data = [None, None] 1 ctx1i

283 

284 self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data)) 2 ctx1di

285 

286 # Find short period of wheel motion for motion correlation. 

287 if data_for_keys(wheel_keys, self.data['wheel']) and self.data['timestamps'] is not None: 6 ctx1cadihb

288 self.data['wheel'].period = self.get_active_wheel_period(self.data['wheel']) 4 ctx1cahb

289 

290 # Load Bonsai frame timestamps 

291 try: 6 ctx1cadihb

292 ssv_times = raw.load_camera_ssv_times(self.session_path, self.label) 6 ctx1cadihb

293 self.data['bonsai_times'], self.data['camera_times'] = ssv_times 6 ctx1cadihb

294 except AssertionError: 

295 _log.warning('No Bonsai video timestamps file found') 

296 

297 # Gather information from video file 

298 if load_video: 6 ctx1cadihb

299 _log.info('Inspecting video file...') 5 ctx1cadib

300 self.load_video_data() 5 ctx1cadib

301 

302 def load_video_data(self): (empty)

303 """Get basic properties of video. 

304 

305 Updates the `data` property with video metadata such as length and frame count, as well as 

306 loading some frames to perform QC on. 

307 """ 

308 try: 6 ctx1gcadib

309 self.data['video'] = get_video_meta(self.video_path, one=self.one) 6 ctx1gcadib

310 # Sample some frames from the video file 

311 indices = np.linspace(100, self.data['video'].length - 100, self.n_samples).astype(int) 6 ctx1gcadib

312 self.frame_samples_idx = indices 6 ctx1gcadib

313 self.data['frame_samples'] = get_video_frames_preload(self.video_path, indices, 6 ctx1gcadib

314 mask=np.s_[:, :, 0]) 

315 except AssertionError: 

316 _log.error('Failed to read video file; setting outcome to CRITICAL') 

317 self._outcome = spec.QC.CRITICAL 

318 

319 @staticmethod (empty)

320 def get_active_wheel_period(wheel, duration_range=(3., 20.), display=False): (empty)

321 """ 

322 Find period of active wheel movement. 

323 

324 Attempts to find a period of movement where the wheel accelerates and decelerates for the 

325 wheel motion alignment QC. 

326 

327 :param wheel: A Bunch of wheel timestamps and position data 

328 :param duration_range: The candidates must be within min/max duration range 

329 :param display: If true, plot the selected wheel movement 

330 :return: 2-element array comprising the start and end times of the active period 

331 """ 

332 pos, ts = wh.interpolate_position(wheel.timestamps, wheel.position) 5 ctx1ncahb

333 v, acc = wh.velocity_filtered(pos, 1000) 5 ctx1ncahb

334 on, off, *_ = wh.movements(ts, acc, pos_thresh=.1, make_plots=False) 5 ctx1ncahb

335 edges = np.c_[on, off] 5 ctx1ncahb

336 indices, _ = np.where(np.logical_and( 5 ctx1ncahb

337 np.diff(edges) > duration_range[0], np.diff(edges) < duration_range[1])) 

338 if len(indices) == 0: 5 ctx1ncahb

339 _log.warning('No period of wheel movement found for motion alignment.') 1 ctx1n

340 return None 1 ctx1n

341 # Pick movement somewhere in the middle 

342 i = indices[int(indices.size / 2)] 4 ctx1cahb

343 if display: 4 ctx1cahb

344 _, (ax0, ax1) = plt.subplots(2, 1, sharex='all') 

345 mask = np.logical_and(ts > edges[i][0], ts < edges[i][1]) 

346 ax0.plot(ts[mask], pos[mask]) 

347 ax1.plot(ts[mask], acc[mask]) 

348 return edges[i] 4 ctx1cahb

349 

350 def _set_sync(self, session_params=False): (empty)

351 """Set the sync and sync_collection attributes if not already set. 

352 

353 Also set the type attribute based on the sync. NB The type attribute is for legacy sessions. 

354 

355 Parameters 

356 ---------- 

357 session_params : dict, bool 

358 The loaded experiment description file. If False, attempts to load it from the session_path. 

359 """ 

360 if session_params is False: 6 ctx1cadihb

361 if not self.session_path: 

362 raise ValueError('No session path set') 

363 session_params = read_params(self.session_path) 

364 sync, sync_dict = get_sync(session_params) 6 ctx1cadihb

365 self.sync = self.sync or sync 6 ctx1cadihb

366 self.sync_collection = self.sync_collection or sync_dict.get('collection') 6 ctx1cadihb

367 if self.sync: 6 ctx1cadihb

368 self._type = 'ephys' if self.sync == 'nidq' else 'training' 5 ctx1cadhb

369 

370 def _update_meta_from_session_params(self, sess_params): (empty)

371 """ 

372 Update the default expected video properties. 

373 

374 Use properties defined in the experiment description file, if present. This updates the 

375 `video_meta` property with the fps, width and height for the type and camera label. 

376 

377 Parameters 

378 ---------- 

379 sess_params : dict 

380 The loaded experiment description file. 

381 """ 

382 try: 6 ctx1cadihb

383 assert sess_params 6 ctx1cadihb

384 video_pars = sess_params.get('devices', {}).get('cameras', {}).get(self.label) 4 ctx1cahb

385 except (AssertionError, KeyError): 2 ctx1di

386 return 2 ctx1di

387 PROPERTIES = ('width', 'height', 'fps') 4 ctx1cahb

388 video_meta = copy.deepcopy(self.video_meta) # must re-assign as it's a class attribute 4 ctx1cahb

389 if self.type not in video_meta: 4 ctx1cahb

390 video_meta[self.type] = {} 

391 if self.label not in video_meta[self.type]: 4 ctx1cahb

392 video_meta[self.type][self.label] = {} 

393 video_meta[self.type][self.label].update( 4 ctx1cahb

394 **{k: v for k, v in video_pars.items() if k in PROPERTIES} 

395 ) 

396 self.video_meta = video_meta 4 ctx1cahb

397 

398 def run(self, update: bool = False, **kwargs) -> (str, dict): (empty)

399 """ 

400 Run video QC checks and return outcome. 

401 

402 :param update: if True, updates the session QC fields on Alyx 

403 :param download_data: if True, downloads any missing data if required 

404 :param extract_times: if True, re-extracts the camera timestamps from the raw data 

405 :returns: overall outcome as a str, a dict of checks and their outcomes 

406 """ 

407 _log.info(f'Computing QC outcome for {self.label} camera, session {self.eid}') 5 ctx1cadib

408 namespace = f'video{self.label.capitalize()}' 5 ctx1cadib

409 if all(x is None for x in self.data.values()): 5 ctx1cadib

410 self.load_data(**kwargs) 3 ctx1cai

411 if self.data['frame_samples'] is None or self.data['timestamps'] is None: 5 ctx1cadib

412 return spec.QC.NOT_SET, {} 1 ctx1i

413 if self.data['timestamps'].shape[0] == 0: 4 ctx1cadb

414 _log.error(f'No timestamps for {self.label} camera; setting outcome to CRITICAL') 

415 return spec.QC.CRITICAL, {} 

416 

417 def is_metric(x): 4 ctx1cadb

418 return isfunction(x) and x.__name__.startswith('check_') 4 ctx1cadb

419 # import importlib 

420 # classe = getattr(importlib.import_module(self.__module__), self.__name__) 

421 # print(classe) 

422 

423 checks = getmembers(self.__class__, is_metric) 4 ctx1cadb

424 checks = self._remove_check(checks) 4 ctx1cadb

425 self.metrics = {f'_{namespace}_' + k[6:]: fn(self) for k, fn in checks} 4 ctx1cadb

426 

427 values = [x if isinstance(x, spec.QC) else x[0] for x in self.metrics.values()] 4 ctx1cadb

428 outcome = max(map(spec.QC.validate, values)) 4 ctx1cadb

429 

430 if update: 4 ctx1cadb

431 extended = dict() 1 ctx1c

432 for k, v in self.metrics.items(): 1 ctx1c

433 if v is None: 1 ctx1c

434 extended[k] = spec.QC.NOT_SET.name 

435 elif isinstance(v, tuple): 1 ctx1c

436 extended[k] = tuple(i.name if isinstance(i, spec.QC) else i for i in v) 1 ctx1c

437 else: 

438 extended[k] = v.name 1 ctx1c

439 

440 self.update_extended_qc(extended) 1 ctx1c

441 self.update(outcome, namespace) 1 ctx1c

442 return outcome, self.metrics 4 ctx1cadb

443 

444 def _remove_check(self, checks): (empty)

445 """ 

446 Remove one or more check functions from QC checklist. 

447 

448 Parameters 

449 ---------- 

450 checks : list of tuple 

451 The complete list of check function name and handle. 

452 

453 Returns 

454 ------- 

455 list of tuple 

456 The list of check function name and handle, sans names in `checks_to_remove` property. 

457 """ 

458 if len(self.checks_to_remove) == 0: 4 ctx1cadb

459 return checks 3 ctx1cab

460 else: 

461 for check in self.checks_to_remove: 1 ctx1d

462 check_names = [ch[0] for ch in checks] 1 ctx1d

463 idx = check_names.index(check) 1 ctx1d

464 checks.pop(idx) 1 ctx1d

465 return checks 1 ctx1d

466 

467 def check_brightness(self, bounds=(40, 200), max_std=20, roi=True, display=False): (empty)

468 """Check that the video brightness is within a given range. 

469 

470 The mean brightness of each frame must be with the bounds provided, and the standard 

471 deviation across samples frames should be less than the given value. Assumes that the 

472 frame samples are 2D (no colour channels). 

473 

474 :param bounds: For each frame, check that: bounds[0] < M < bounds[1], 

475 where M = mean(frame). If less than 75% of sample frames outside of these bounds, the 

476 outcome is WARNING. If <75% of frames within twice the bounds, the outcome is FAIL. 

477 :param max_std: The standard deviation of the frame luminance means must be less than this 

478 :param roi: If True, check brightness on ROI of frame 

479 :param display: When True the mean frame luminance is plotted against sample frames. 

480 The sample frames with the lowest and highest mean luminance are shown. 

481 """ 

482 if self.data['frame_samples'] is None: 6 ctx1kcadbf

483 return spec.QC.NOT_SET 1 ctx1k

484 if roi is True: 6 ctx1kcadbf

485 _, h, w = self.data['frame_samples'].shape 6 ctx1kcadbf

486 if self.label == 'body': # Latter half 6 ctx1kcadbf

487 roi = (slice(None), slice(None), slice(int(w / 2), None, None)) 1 ctx1f

488 elif self.label == 'left': # Top left quadrant (~2/3, 1/2 height) 6 ctx1kcadbf

489 roi = (slice(None), slice(None, int(h / 2), None), slice(None, int(w * .66), None)) 6 ctx1kcadbf

490 else: # Top right quadrant (~2/3 width, 1/2 height) 

491 roi = (slice(None), slice(None, int(h / 2), None), slice(int(w * .66), None, None)) 1 ctx1f

492 else: 

493 roi = (slice(None), slice(None), slice(None)) 

494 brightness = self.data['frame_samples'][roi].mean(axis=(1, 2)) 6 ctx1kcadbf

495 # dims = self.data['frame_samples'].shape 

496 # brightness /= np.array((*dims[1:], 255)).prod() # Normalize 

497 

498 if display: 6 ctx1kcadbf

499 f = plt.figure() 1 ctx1k

500 gs = f.add_gridspec(2, 3) 1 ctx1k

501 indices = self.frame_samples_idx 1 ctx1k

502 # Plot mean frame luminance 

503 ax = f.add_subplot(gs[:2, :2]) 1 ctx1k

504 plt.plot(indices, brightness, label='brightness') 1 ctx1k

505 ax.set( 1 ctx1k

506 xlabel='frame #', 

507 ylabel='brightness (mean pixel)', 

508 title='Brightness') 

509 ax.hlines(bounds, 0, indices[-1], 1 ctx1k

510 colors='tab:orange', linestyles=':', label='warning bounds') 

511 ax.hlines((bounds[0] / 2, bounds[1] * 2), 0, indices[-1], 1 ctx1k

512 colors='r', linestyles=':', label='failure bounds') 

513 ax.legend() 1 ctx1k

514 # Plot min-max frames 

515 for i, idx in enumerate((np.argmax(brightness), np.argmin(brightness))): 1 ctx1k

516 a = f.add_subplot(gs[i, 2]) 1 ctx1k

517 ax.annotate('*', (indices[idx], brightness[idx]), # this is the point to label 1 ctx1k

518 textcoords='offset points', xytext=(0, 1), ha='center') 

519 frame = self.data['frame_samples'][idx][roi[1:]] 1 ctx1k

520 title = ('min' if i else 'max') + ' mean luminance = %.2f' % brightness[idx] 1 ctx1k

521 self.imshow(frame, ax=a, title=title) 1 ctx1k

522 

523 PCT_PASS = .75 # Proportion of sample frames that must pass 6 ctx1kcadbf

524 # Warning if brightness not within range (3/4 of frames must be between bounds) 

525 warn_range = np.logical_and(brightness > bounds[0], brightness < bounds[1]) 6 ctx1kcadbf

526 warn_range = 'PASS' if sum(warn_range) / self.n_samples > PCT_PASS else 'WARNING' 6 ctx1kcadbf

527 # Fail if brightness not within twice the range or std less than threshold 

528 fail_range = np.logical_and(brightness > bounds[0] / 2, brightness < bounds[1] * 2) 6 ctx1kcadbf

529 within_range = sum(fail_range) / self.n_samples > PCT_PASS 6 ctx1kcadbf

530 fail_range = 'PASS' if within_range and np.std(brightness) < max_std else 'FAIL' 6 ctx1kcadbf

531 return self.overall_outcome([warn_range, fail_range]) 6 ctx1kcadbf

532 

533 def check_file_headers(self): (empty)

534 """Check reported frame rate matches FPGA frame rate.""" 

535 if None in (self.data['video'], self.video_meta): 6 ctx1ucadbf

536 return spec.QC.NOT_SET 1 ctx1u

537 expected = self.video_meta[self.type][self.label] 6 ctx1ucadbf

538 return spec.QC.PASS if self.data['video']['fps'] == expected['fps'] else spec.QC.FAIL 6 ctx1ucadbf

539 

540 def check_framerate(self, threshold=1.): (empty)

541 """Check camera times match specified frame rate for camera. 

542 

543 :param threshold: The maximum absolute difference between timestamp sample rate and video 

544 frame rate. NB: Does not take into account dropped frames. 

545 """ 

546 if any(x is None for x in (self.data['timestamps'], self.video_meta)): 5 ctx1scadb

547 return spec.QC.NOT_SET 1 ctx1s

548 fps = self.video_meta[self.type][self.label]['fps'] 5 ctx1scadb

549 Fs = 1 / np.median(np.diff(self.data['timestamps'])) # Approx. frequency of camera 5 ctx1scadb

550 return spec.QC.PASS if abs(Fs - fps) < threshold else spec.QC.FAIL, float(round(Fs, 3)) 5 ctx1scadb

551 

552 def check_pin_state(self, display=False): (empty)

553 """Check the pin state reflects Bpod TTLs.""" 

554 if not data_for_keys(('video', 'pin_state', 'audio'), self.data): 5 ctx1lcadb

555 return spec.QC.NOT_SET 2 ctx1lc

556 size_diff = int(self.data['pin_state'].shape[0] - self.data['video']['length']) 4 ctx1ladb

557 # NB: The pin state can be high for 2 consecutive frames 

558 low2high = np.insert(np.diff(self.data['pin_state'][:, -1].astype(int)) == 1, 0, False) 4 ctx1ladb

559 # NB: Time between two consecutive TTLs can be sub-frame, so this will fail 

560 ndiff_low2high = int(self.data['audio'][::2].size - sum(low2high)) 4 ctx1ladb

561 # state_ttl_matches = ndiff_low2high == 0 

562 # Check within ms of audio times 

563 if display: 4 ctx1ladb

564 plt.Figure() 1 ctx1l

565 plt.plot(self.data['timestamps'][low2high], np.zeros(sum(low2high)), 'o', 1 ctx1l

566 label='GPIO Low -> High') 

567 plt.plot(self.data['audio'], np.zeros(self.data['audio'].size), 'rx', 1 ctx1l

568 label='Audio TTL High') 

569 plt.xlabel('FPGA frame times / s') 1 ctx1l

570 plt.gca().set(yticklabels=[]) 1 ctx1l

571 plt.gca().tick_params(left=False) 1 ctx1l

572 plt.legend() 1 ctx1l

573 

574 outcome = self.overall_outcome( 4 ctx1ladb

575 ('PASS' if size_diff == 0 else 'WARNING' if np.abs(size_diff) < 5 else 'FAIL', 

576 'PASS' if np.abs(ndiff_low2high) < 5 else 'WARNING') 

577 ) 

578 return outcome, ndiff_low2high, size_diff 4 ctx1ladb

579 

580 def check_dropped_frames(self, threshold=.1): (empty)

581 """Check how many frames were reported missing. 

582 

583 :param threshold: The maximum allowable percentage of dropped frames 

584 """ 

585 if not data_for_keys(('video', 'count'), self.data): 5 ctx1mcadb

586 return spec.QC.NOT_SET 2 ctx1mc

587 size_diff = int(self.data['count'].size - self.data['video']['length']) 4 ctx1madb

588 strict_increase = np.all(np.diff(self.data['count']) > 0) 4 ctx1madb

589 if not strict_increase: 4 ctx1madb

590 n_affected = np.sum(np.invert(strict_increase)) 1 ctx1m

591 _log.info(f'frame count not strictly increasing: ' 1 ctx1m

592 f'{n_affected} frames affected ({n_affected / strict_increase.size:.2%})') 

593 return spec.QC.CRITICAL 1 ctx1m

594 dropped = np.diff(self.data['count']).astype(int) - 1 4 ctx1madb

595 pct_dropped = (sum(dropped) / len(dropped) * 100) 4 ctx1madb

596 # Calculate overall outcome for this check 

597 outcome = self.overall_outcome( 4 ctx1madb

598 ('PASS' if size_diff == 0 else 'WARNING' if np.abs(size_diff) < 5 else 'FAIL', 

599 'PASS' if pct_dropped < threshold else 'FAIL') 

600 ) 

601 return outcome, int(sum(dropped)), size_diff 4 ctx1madb

602 

603 def check_timestamps(self): (empty)

604 """Check that the camera.times array is reasonable.""" 

605 if not data_for_keys(('timestamps', 'video'), self.data): 5 ctx1tcadb

606 return spec.QC.NOT_SET 1 ctx1t

607 # Check number of timestamps matches video 

608 length_matches = self.data['timestamps'].size == self.data['video'].length 5 ctx1tcadb

609 # Check times are strictly increasing 

610 increasing = all(np.diff(self.data['timestamps']) > 0) 5 ctx1tcadb

611 # Check times do not contain nans 

612 nanless = not np.isnan(self.data['timestamps']).any() 5 ctx1tcadb

613 return spec.QC.PASS if increasing and length_matches and nanless else spec.QC.FAIL 5 ctx1tcadb

614 

615 def check_camera_times(self): (empty)

616 """Check that the number of raw camera timestamps matches the number of video frames.""" 

617 if not data_for_keys(('bonsai_times', 'video'), self.data): 5 ctx1vcadb

618 return spec.QC.NOT_SET 1 ctx1v

619 length_match = len(self.data['camera_times']) == self.data['video'].length 5 ctx1vcadb

620 outcome = spec.QC.PASS if length_match else spec.QC.WARNING 5 ctx1vcadb

621 # 1 / np.median(np.diff(self.data.camera_times)) 

622 return outcome, len(self.data['camera_times']) - self.data['video'].length 5 ctx1vcadb

623 

624 def check_resolution(self): (empty)

625 """Check that the timestamps and video file resolution match what we expect.""" 

626 if self.data['video'] is None: 6 ctx1pcadbf

627 return spec.QC.NOT_SET 1 ctx1p

628 actual = self.data['video'] 6 ctx1pcadbf

629 expected = self.video_meta[self.type][self.label] 6 ctx1pcadbf

630 match = actual['width'] == expected['width'] and actual['height'] == expected['height'] 6 ctx1pcadbf

631 return spec.QC.PASS if match else spec.QC.FAIL 6 ctx1pcadbf

632 

633 def check_wheel_alignment(self, tolerance=(1, 2), display=False): (empty)

634 """Check wheel motion in video correlates with the rotary encoder signal. 

635 

636 Check is skipped for body camera videos as the wheel is often obstructed 

637 

638 Parameters 

639 ---------- 

640 tolerance : int, (int, int) 

641 Maximum absolute offset in frames. If two values, the maximum value is taken as the 

642 warning threshold. 

643 display : bool 

644 If true, the wheel motion energy is plotted against the rotary encoder. 

645 

646 Returns 

647 ------- 

648 one.alf.spec.QC 

649 The outcome, one of {'NOT_SET', 'FAIL', 'WARNING', 'PASS'}. 

650 int 

651 Frame offset, i.e. by how many frames the video was shifted to match the rotary encoder 

652 signal. Negative values mean the video was shifted backwards with respect to the wheel 

653 timestamps. 

654 

655 Notes 

656 ----- 

657 - A negative frame offset typically means that there were frame TTLs at the beginning that 

658 do not correspond to any video frames (sometimes the first few frames aren't saved to 

659 disk). Since 2021-09-15 the extractor should compensate for this. 

660 """ 

661 wheel_present = data_for_keys(('position', 'timestamps', 'period'), self.data['wheel']) 5 ctx1oncab

662 if not wheel_present or self.label == 'body': 5 ctx1oncab

663 return spec.QC.NOT_SET 2 ctx1on