Coverage for ibllib/qc/camera.py: 82%

1"""Video quality control. 

2 

3This module runs a list of quality control metrics on the camera and extracted video data. 

4 

5Examples 

6-------- 

7Run right camera QC, downloading all but video file 

8 

9>>> qc = CameraQC(eid, 'right', download_data=True, stream=True) 

10>>> qc.run() 

11 

12Run left camera QC with session path, update QC field in Alyx 

13 

14>>> qc = CameraQC(session_path, 'left') 

15>>> outcome, extended = qc.run(update=True) # Returns outcome of videoQC only 

16>>> print(f'video QC = {outcome}; overall session QC = {qc.outcome}') # NB difference outcomes 

17 

18Run only video QC (no timestamp/alignment checks) on 20 frames for the body camera 

19 

20>>> qc = CameraQC(eid, 'body', n_samples=20) 

21>>> qc.load_video_data() # Quicker than loading all data 

22>>> qc.run() 

23 

24Run specific video QC check and display the plots 

25 

26>>> qc = CameraQC(eid, 'left') 

27>>> qc.load_data(download_data=True) 

28>>> qc.check_position(display=True) # NB: Not all checks make plots 

29 

30Run the QC for all cameras 

31 

32>>> qcs = run_all_qc(eid) 

33>>> qcs['left'].metrics # Dict of checks and outcomes for left camera 

34""" 

35import logging (empty)

36from inspect import getmembers, isfunction (empty)

37from pathlib import Path (empty)

38from itertools import chain (empty)

39import copy (empty)

40 

41import cv2 (empty)

42import numpy as np (empty)

43import matplotlib.pyplot as plt (empty)

44from matplotlib.patches import Rectangle (empty)

45 

46import one.alf.io as alfio (empty)

47from one.alf import spec (empty)

48from one.alf.exceptions import ALFObjectNotFound (empty)

49from iblutil.util import Bunch (empty)

50from iblutil.numerical import within_ranges (empty)

51 

52import ibllib.io.extractors.camera as camio (empty)

53from ibllib.io.extractors import ephys_fpga, training_wheel, mesoscope (empty)

54from ibllib.io.extractors.video_motion import MotionAlignment (empty)

55from ibllib.io import raw_data_loaders as raw (empty)

56from ibllib.io.raw_daq_loaders import load_timeline_sync_and_chmap (empty)

57from ibllib.io.session_params import read_params, get_sync, get_sync_namespace (empty)

58import brainbox.behavior.wheel as wh (empty)

59from ibllib.io.video import get_video_meta, get_video_frames_preload, assert_valid_label (empty)

60from . import base (empty)

61 

62_log = logging.getLogger(__name__) (empty)

63 

64try: (empty)

65 from labcams import parse_cam_log (empty)

66except ImportError: 

67 _log.warning('labcams not installed') 

68 

69 

70class CameraQC(base.QC): (empty)

71 """A class for computing camera QC metrics.""" 

72 

73 dstypes = [ (empty)

74 '_ibl_experiment.description', 

75 '_iblrig_Camera.frameData', # Replaces the next 3 datasets 

76 '_iblrig_Camera.frame_counter', 

77 '_iblrig_Camera.GPIO', 

78 '_iblrig_Camera.timestamps', 

79 '_iblrig_taskData.raw', 

80 '_iblrig_taskSettings.raw', 

81 '_iblrig_Camera.raw', 

82 'camera.times', 

83 'wheel.position', 

84 'wheel.timestamps' 

85 ] 

86 dstypes_fpga = [ (empty)

87 '_spikeglx_sync.channels', 

88 '_spikeglx_sync.polarities', 

89 '_spikeglx_sync.times', 

90 'ephysData.raw.meta' 

91 ] 

92 """Recall that for the training rig there is only one side camera at 30 Hz and 1280 x 1024 px. (empty)

93 For the recording rig there are two label cameras (left: 60 Hz, 1280 x 1024 px; 

94 right: 150 Hz, 640 x 512 px) and one body camera (30 Hz, 640 x 512 px). """ 

95 video_meta = { (empty)

96 'training': { 

97 'left': { 

98 'fps': 30, 

99 'width': 1280, 

100 'height': 1024 

101 } 

102 }, 

103 'ephys': { 

104 'left': { 

105 'fps': 60, 

106 'width': 1280, 

107 'height': 1024 

108 }, 

109 'right': { 

110 'fps': 150, 

111 'width': 640, 

112 'height': 512 

113 }, 

114 'body': { 

115 'fps': 30, 

116 'width': 640, 

117 'height': 512 

118 }, 

119 } 

120 } 

121 

122 def __init__(self, session_path_or_eid, camera, **kwargs): (empty)

123 """ 

124 Compute and view camera QC metrics. 

125 

126 :param session_path_or_eid: A session id or path 

127 :param camera: The camera to run QC on, if None QC is run for all three cameras 

128 :param n_samples: The number of frames to sample for the position and brightness QC 

129 :param stream: If true and local video files not available, the data are streamed from 

130 the remote source. 

131 :param log: A logging.Logger instance, if None the 'ibllib' logger is used 

132 :param one: An ONE instance for fetching and setting the QC on Alyx 

133 """ 

134 self.stream = kwargs.pop('stream', None) 6 ctx1fachgb

135 self.n_samples = kwargs.pop('n_samples', 100) 6 ctx1fachgb

136 self.sync_collection = kwargs.pop('sync_collection', None) 6 ctx1fachgb

137 self.sync = kwargs.pop('sync_type', None) 6 ctx1fachgb

138 self.protocol = kwargs.pop('protocol', None) 6 ctx1fachgb

139 super().__init__(session_path_or_eid, **kwargs) 6 ctx1fachgb

140 

141 # Data 

142 self.label = assert_valid_label(camera) 6 ctx1fachgb

143 filename = f'_iblrig_{self.label}Camera.raw*.mp4' 6 ctx1fachgb

144 raw_video_path = self.session_path.joinpath('raw_video_data') 6 ctx1fachgb

145 self.video_path = next(raw_video_path.glob(filename), None) 6 ctx1fachgb

146 

147 # If local video doesn't exist, change video path to URL 

148 if not self.video_path and self.stream is not False and self.one is not None: 6 ctx1fachgb

149 try: 

150 self.stream = True 

151 self.video_path = self.one.path2url(raw_video_path / filename.replace('*', '')) 

152 except (StopIteration, ALFObjectNotFound): 

153 _log.error('No remote or local video file found') 

154 self.video_path = None 

155 

156 logging.disable(logging.NOTSET) 6 ctx1fachgb

157 keys = ('count', 'pin_state', 'audio', 'fpga_times', 'wheel', 'video', 6 ctx1fachgb

158 'frame_samples', 'timestamps', 'camera_times', 'bonsai_times') 

159 self.data = Bunch.fromkeys(keys) 6 ctx1fachgb

160 self.frame_samples_idx = None 6 ctx1fachgb

161 

162 # QC outcomes map 

163 self.metrics = None 6 ctx1fachgb

164 self.outcome = spec.QC.NOT_SET 6 ctx1fachgb

165 

166 # Specify any checks to remove 

167 if self.protocol is not None and 'habituation' in self.protocol: 6 ctx1fachgb

168 self.checks_to_remove = ['check_wheel_alignment'] 1 ctx1c

169 else: 

170 self.checks_to_remove = [] 5 ctx1fahgb

171 self._type = None 6 ctx1fachgb

172 

173 @property (empty)

174 def type(self): (empty)

175 """ 

176 Returns the camera type based on the protocol. 

177 

178 :return: Returns either None, 'ephys' or 'training' 

179 """ 

180 if not self._type: 8 ctx1troacgbe

181 return 

182 else: 

183 return 'ephys' if 'ephys' in self._type else 'training' 8 ctx1troacgbe

184 

185 def load_data(self, extract_times: bool = False, load_video: bool = True) -> None: (empty)

186 """Extract the data from raw data files. 

187 

188 Extracts all the required task data from the raw data files. Missing data will raise an 

189 AssertionError. TODO This method could instead be moved to CameraExtractor classes. 

190 

191 Data keys: 

192 - count (int array): the sequential frame number (n, n+1, n+2...) 

193 - pin_state (): the camera GPIO pin; records the audio TTLs; should be one per frame 

194 - audio (float array): timestamps of audio TTL fronts 

195 - fpga_times (float array): timestamps of camera TTLs recorded by FPGA 

196 - timestamps (float array): extracted video timestamps (the camera.times ALF) 

197 - bonsai_times (datetime array): system timestamps of video PC; should be one per frame 

198 - camera_times (float array): camera frame timestamps extracted from frame headers 

199 - wheel (Bunch): rotary encoder timestamps, position and period used for wheel motion 

200 - video (Bunch): video meta data, including dimensions and FPS 

201 - frame_samples (h x w x n array): array of evenly sampled frames (1 colour channel) 

202 

203 :param extract_times: if True, the camera.times are re-extracted from the raw data 

204 :param load_video: if True, calls the load_video_data method 

205 """ 

206 assert self.session_path, 'no session path set' 5 ctx1achgb

207 _log.info('Gathering data for QC') 5 ctx1achgb

208 

209 # Get frame count and pin state 

210 self.data['count'], self.data['pin_state'] = \ 5 ctx1achgb

211 raw.load_embedded_frame_data(self.session_path, self.label, raw=True) 

212 

213 # If there is an experiment description and there are video parameters 

214 sess_params = read_params(self.session_path) or {} 5 ctx1achgb

215 task_collection = get_task_collection(sess_params) 5 ctx1achgb

216 ns = get_sync_namespace(sess_params) 5 ctx1achgb

217 self._set_sync(sess_params) 5 ctx1achgb

218 self._update_meta_from_session_params(sess_params) 5 ctx1achgb

219 

220 # Load the audio and raw FPGA times 

221 sync = chmap = None 5 ctx1achgb

222 if self.sync != 'bpod' and self.sync is not None: 5 ctx1achgb

223 self.sync_collection = self.sync_collection or 'raw_ephys_data' 2 ctx1ag

224 ns = ns or 'spikeglx' 2 ctx1ag

225 if ns == 'spikeglx': 2 ctx1ag

226 sync, chmap = ephys_fpga.get_sync_and_chn_map(self.session_path, self.sync_collection) 2 ctx1ag

227 elif ns == 'timeline': 1 ctx1g

228 sync, chmap = load_timeline_sync_and_chmap(self.session_path / self.sync_collection) 

229 else: 

230 raise NotImplementedError(f'Unknown namespace "{ns}"') 1 ctx1g

231 audio_ttls = ephys_fpga.get_sync_fronts(sync, chmap['audio']) 2 ctx1ag

232 self.data['audio'] = audio_ttls['times'] # Get rises 2 ctx1ag

233 # Load raw FPGA times 

234 cam_ts = camio.extract_camera_sync(sync, chmap) 2 ctx1ag

235 self.data['fpga_times'] = cam_ts[self.label] 2 ctx1ag

236 else: 

237 self.sync_collection = self.sync_collection or task_collection 4 ctx1chgb

238 bpod_data = raw.load_data(self.session_path, task_collection) 4 ctx1chgb

239 _, audio_ttls = raw.load_bpod_fronts( 4 ctx1chgb

240 self.session_path, data=bpod_data, task_collection=task_collection) 

241 self.data['audio'] = audio_ttls['times'] 4 ctx1chgb

242 

243 # Load extracted frame times 

244 alf_path = self.session_path / 'alf' 5 ctx1achgb

245 try: 5 ctx1achgb

246 assert not extract_times 5 ctx1achgb

247 self.data['timestamps'] = alfio.load_object( 2 ctx1hg

248 alf_path, f'{self.label}Camera', short_keys=True)['times'] 

249 except AssertionError: # Re-extract 4 ctx1achb

250 kwargs = dict(video_path=self.video_path, save=False) 3 ctx1acb

251 if self.sync == 'bpod': 3 ctx1acb

252 extractor = camio.CameraTimestampsBpod(self.session_path) 2 ctx1cb

253 kwargs['task_collection'] = task_collection 2 ctx1cb

254 else: 

255 kwargs.update(sync=sync, chmap=chmap) 1 ctx1a

256 extractor = camio.CameraTimestampsFPGA(self.label, self.session_path) 1 ctx1a

257 self.data['timestamps'] = extractor.extract(**kwargs)[0] 3 ctx1acb

258 except ALFObjectNotFound: 1 ctx1h

259 _log.warning('no camera.times ALF found for session') 1 ctx1h

260 

261 # Get audio and wheel data 

262 wheel_keys = ('timestamps', 'position') 5 ctx1achgb

263 try: 5 ctx1achgb

264 # glob in case wheel data are in sub-collections 

265 alf_path = next(alf_path.rglob('*wheel.timestamps*')).parent 5 ctx1achgb

266 self.data['wheel'] = alfio.load_object(alf_path, 'wheel', short_keys=True) 3 ctx1agb

267 except (StopIteration, ALFObjectNotFound): 2 ctx1ch

268 # Extract from raw data 

269 if self.sync != 'bpod' and self.sync is not None: 2 ctx1ch

270 if ns == 'spikeglx': 

271 wheel_data = ephys_fpga.extract_wheel_sync(sync, chmap) 

272 elif ns == 'timeline': 

273 extractor = mesoscope.TimelineTrials(self.session_path, sync_collection=self.sync_collection) 

274 wheel_data = extractor.extract_wheel_sync() 

275 else: 

276 raise NotImplementedError(f'Unknown namespace "{ns}"') 

277 else: 

278 if self.protocol is not None and 'habituation' in self.protocol: 2 ctx1ch

279 wheel_data = training_wheel.get_wheel_position( 1 ctx1c

280 self.session_path, task_collection=task_collection) 

281 else: 

282 wheel_data = [None, None] 1 ctx1h

283 

284 self.data['wheel'] = Bunch(zip(wheel_keys, wheel_data)) 2 ctx1ch

285 

286 # Find short period of wheel motion for motion correlation. 

287 if data_for_keys(wheel_keys, self.data['wheel']) and self.data['timestamps'] is not None: 5 ctx1achgb

288 self.data['wheel'].period = self.get_active_wheel_period(self.data['wheel']) 3 ctx1agb

289 

290 # Load Bonsai frame timestamps 

291 try: 5 ctx1achgb

292 ssv_times = raw.load_camera_ssv_times(self.session_path, self.label) 5 ctx1achgb

293 self.data['bonsai_times'], self.data['camera_times'] = ssv_times 5 ctx1achgb

294 except AssertionError: 

295 _log.warning('No Bonsai video timestamps file found') 

296 

297 # Gather information from video file 

298 if load_video: 5 ctx1achgb

299 _log.info('Inspecting video file...') 4 ctx1achb

300 self.load_video_data() 4 ctx1achb

301 

302 def load_video_data(self): (empty)

303 """Get basic properties of video. 

304 

305 Updates the `data` property with video metadata such as length and frame count, as well as 

306 loading some frames to perform QC on. 

307 """ 

308 try: 5 ctx1fachb

309 self.data['video'] = get_video_meta(self.video_path, one=self.one) 5 ctx1fachb

310 # Sample some frames from the video file 

311 indices = np.linspace(100, self.data['video'].length - 100, self.n_samples).astype(int) 5 ctx1fachb

312 self.frame_samples_idx = indices 5 ctx1fachb

313 self.data['frame_samples'] = get_video_frames_preload(self.video_path, indices, 5 ctx1fachb

314 mask=np.s_[:, :, 0]) 

315 except AssertionError: 

316 _log.error('Failed to read video file; setting outcome to CRITICAL') 

317 self._outcome = spec.QC.CRITICAL 

318 

319 @staticmethod (empty)

320 def get_active_wheel_period(wheel, duration_range=(3., 20.), display=False): (empty)

321 """ 

322 Find period of active wheel movement. 

323 

324 Attempts to find a period of movement where the wheel accelerates and decelerates for the 

325 wheel motion alignment QC. 

326 

327 :param wheel: A Bunch of wheel timestamps and position data 

328 :param duration_range: The candidates must be within min/max duration range 

329 :param display: If true, plot the selected wheel movement 

330 :return: 2-element array comprising the start and end times of the active period 

331 """ 

332 pos, ts = wh.interpolate_position(wheel.timestamps, wheel.position) 4 ctx1magb

333 v, acc = wh.velocity_filtered(pos, 1000) 4 ctx1magb

334 on, off, *_ = wh.movements(ts, acc, pos_thresh=.1, make_plots=False) 4 ctx1magb

335 edges = np.c_[on, off] 4 ctx1magb

336 indices, _ = np.where(np.logical_and( 4 ctx1magb

337 np.diff(edges) > duration_range[0], np.diff(edges) < duration_range[1])) 

338 if len(indices) == 0: 4 ctx1magb

339 _log.warning('No period of wheel movement found for motion alignment.') 1 ctx1m

340 return None 1 ctx1m

341 # Pick movement somewhere in the middle 

342 i = indices[int(indices.size / 2)] 3 ctx1agb

343 if display: 3 ctx1agb

344 _, (ax0, ax1) = plt.subplots(2, 1, sharex='all') 

345 mask = np.logical_and(ts > edges[i][0], ts < edges[i][1]) 

346 ax0.plot(ts[mask], pos[mask]) 

347 ax1.plot(ts[mask], acc[mask]) 

348 return edges[i] 3 ctx1agb

349 

350 def _set_sync(self, session_params=False): (empty)

351 """Set the sync and sync_collection attributes if not already set. 

352 

353 Also set the type attribute based on the sync. NB The type attribute is for legacy sessions. 

354 

355 Parameters 

356 ---------- 

357 session_params : dict, bool 

358 The loaded experiment description file. If False, attempts to load it from the session_path. 

359 """ 

360 if session_params is False: 5 ctx1achgb

361 if not self.session_path: 

362 raise ValueError('No session path set') 

363 session_params = read_params(self.session_path) 

364 sync, sync_dict = get_sync(session_params) 5 ctx1achgb

365 self.sync = self.sync or sync 5 ctx1achgb

366 self.sync_collection = self.sync_collection or sync_dict.get('collection') 5 ctx1achgb

367 if self.sync: 5 ctx1achgb

368 self._type = 'ephys' if self.sync == 'nidq' else 'training' 4 ctx1acgb

369 

370 def _update_meta_from_session_params(self, sess_params): (empty)

371 """ 

372 Update the default expected video properties. 

373 

374 Use properties defined in the experiment description file, if present. This updates the 

375 `video_meta` property with the fps, width and height for the type and camera label. 

376 

377 Parameters 

378 ---------- 

379 sess_params : dict 

380 The loaded experiment description file. 

381 """ 

382 try: 5 ctx1achgb

383 assert sess_params 5 ctx1achgb

384 video_pars = sess_params.get('devices', {}).get('cameras', {}).get(self.label) 3 ctx1agb

385 except (AssertionError, KeyError): 2 ctx1ch

386 return 2 ctx1ch

387 PROPERTIES = ('width', 'height', 'fps') 3 ctx1agb

388 video_meta = copy.deepcopy(self.video_meta) # must re-assign as it's a class attribute 3 ctx1agb

389 if self.type not in video_meta: 3 ctx1agb

390 video_meta[self.type] = {} 

391 if self.label not in video_meta[self.type]: 3 ctx1agb

392 video_meta[self.type][self.label] = {} 

393 video_meta[self.type][self.label].update( 3 ctx1agb

394 **{k: v for k, v in video_pars.items() if k in PROPERTIES} 

395 ) 

396 self.video_meta = video_meta 3 ctx1agb

397 

398 def run(self, update: bool = False, **kwargs) -> (str, dict): (empty)

399 """ 

400 Run video QC checks and return outcome. 

401 

402 :param update: if True, updates the session QC fields on Alyx 

403 :param download_data: if True, downloads any missing data if required 

404 :param extract_times: if True, re-extracts the camera timestamps from the raw data 

405 :returns: overall outcome as a str, a dict of checks and their outcomes 

406 """ 

407 _log.info(f'Computing QC outcome for {self.label} camera, session {self.eid}') 4 ctx1achb

408 namespace = f'video{self.label.capitalize()}' 4 ctx1achb

409 if all(x is None for x in self.data.values()): 4 ctx1achb

410 self.load_data(**kwargs) 2 ctx1ah

411 if self.data['frame_samples'] is None or self.data['timestamps'] is None: 4 ctx1achb

412 return spec.QC.NOT_SET, {} 1 ctx1h

413 if self.data['timestamps'].shape[0] == 0: 3 ctx1acb

414 _log.error(f'No timestamps for {self.label} camera; setting outcome to CRITICAL') 

415 return spec.QC.CRITICAL, {} 

416 

417 def is_metric(x): 3 ctx1acb

418 return isfunction(x) and x.__name__.startswith('check_') 3 ctx1acb

419 # import importlib 

420 # classe = getattr(importlib.import_module(self.__module__), self.__name__) 

421 # print(classe) 

422 

423 checks = getmembers(self.__class__, is_metric) 3 ctx1acb

424 checks = self._remove_check(checks) 3 ctx1acb

425 self.metrics = {f'_{namespace}_' + k[6:]: fn(self) for k, fn in checks} 3 ctx1acb

426 

427 values = [x if isinstance(x, spec.QC) else x[0] for x in self.metrics.values()] 3 ctx1acb

428 outcome = max(map(spec.QC.validate, values)) 3 ctx1acb

429 

430 if update: 3 ctx1acb

431 extended = dict() 

432 for k, v in self.metrics.items(): 

433 if v is None: 

434 extended[k] = spec.QC.NOT_SET.name 

435 elif isinstance(v, tuple): 

436 extended[k] = tuple(i.name if isinstance(i, spec.QC) else i for i in v) 

437 else: 

438 extended[k] = v.name 

439 

440 self.update_extended_qc(extended) 

441 self.update(outcome, namespace) 

442 return outcome, self.metrics 3 ctx1acb

443 

444 def _remove_check(self, checks): (empty)

445 """ 

446 Remove one or more check functions from QC checklist. 

447 

448 Parameters 

449 ---------- 

450 checks : list of tuple 

451 The complete list of check function name and handle. 

452 

453 Returns 

454 ------- 

455 list of tuple 

456 The list of check function name and handle, sans names in `checks_to_remove` property. 

457 """ 

458 if len(self.checks_to_remove) == 0: 3 ctx1acb

459 return checks 2 ctx1ab

460 else: 

461 for check in self.checks_to_remove: 1 ctx1c

462 check_names = [ch[0] for ch in checks] 1 ctx1c

463 idx = check_names.index(check) 1 ctx1c

464 checks.pop(idx) 1 ctx1c

465 return checks 1 ctx1c

466 

467 def check_brightness(self, bounds=(40, 200), max_std=20, roi=True, display=False): (empty)

468 """Check that the video brightness is within a given range. 

469 

470 The mean brightness of each frame must be with the bounds provided, and the standard 

471 deviation across samples frames should be less than the given value. Assumes that the 

472 frame samples are 2D (no colour channels). 

473 

474 :param bounds: For each frame, check that: bounds[0] < M < bounds[1], 

475 where M = mean(frame). If less than 75% of sample frames outside of these bounds, the 

476 outcome is WARNING. If <75% of frames within twice the bounds, the outcome is FAIL. 

477 :param max_std: The standard deviation of the frame luminance means must be less than this 

478 :param roi: If True, check brightness on ROI of frame 

479 :param display: When True the mean frame luminance is plotted against sample frames. 

480 The sample frames with the lowest and highest mean luminance are shown. 

481 """ 

482 if self.data['frame_samples'] is None: 5 ctx1jacbe

483 return spec.QC.NOT_SET 1 ctx1j

484 if roi is True: 5 ctx1jacbe

485 _, h, w = self.data['frame_samples'].shape 5 ctx1jacbe

486 if self.label == 'body': # Latter half 5 ctx1jacbe

487 roi = (slice(None), slice(None), slice(int(w / 2), None, None)) 1 ctx1e

488 elif self.label == 'left': # Top left quadrant (~2/3, 1/2 height) 5 ctx1jacbe

489 roi = (slice(None), slice(None, int(h / 2), None), slice(None, int(w * .66), None)) 5 ctx1jacbe

490 else: # Top right quadrant (~2/3 width, 1/2 height) 

491 roi = (slice(None), slice(None, int(h / 2), None), slice(int(w * .66), None, None)) 1 ctx1e

492 else: 

493 roi = (slice(None), slice(None), slice(None)) 

494 brightness = self.data['frame_samples'][roi].mean(axis=(1, 2)) 5 ctx1jacbe

495 # dims = self.data['frame_samples'].shape 

496 # brightness /= np.array((*dims[1:], 255)).prod() # Normalize 

497 

498 if display: 5 ctx1jacbe

499 f = plt.figure() 1 ctx1j

500 gs = f.add_gridspec(2, 3) 1 ctx1j

501 indices = self.frame_samples_idx 1 ctx1j

502 # Plot mean frame luminance 

503 ax = f.add_subplot(gs[:2, :2]) 1 ctx1j

504 plt.plot(indices, brightness, label='brightness') 1 ctx1j

505 ax.set( 1 ctx1j

506 xlabel='frame #', 

507 ylabel='brightness (mean pixel)', 

508 title='Brightness') 

509 ax.hlines(bounds, 0, indices[-1], 1 ctx1j

510 colors='tab:orange', linestyles=':', label='warning bounds') 

511 ax.hlines((bounds[0] / 2, bounds[1] * 2), 0, indices[-1], 1 ctx1j

512 colors='r', linestyles=':', label='failure bounds') 

513 ax.legend() 1 ctx1j

514 # Plot min-max frames 

515 for i, idx in enumerate((np.argmax(brightness), np.argmin(brightness))): 1 ctx1j

516 a = f.add_subplot(gs[i, 2]) 1 ctx1j

517 ax.annotate('*', (indices[idx], brightness[idx]), # this is the point to label 1 ctx1j

518 textcoords='offset points', xytext=(0, 1), ha='center') 

519 frame = self.data['frame_samples'][idx][roi[1:]] 1 ctx1j

520 title = ('min' if i else 'max') + ' mean luminance = %.2f' % brightness[idx] 1 ctx1j

521 self.imshow(frame, ax=a, title=title) 1 ctx1j

522 

523 PCT_PASS = .75 # Proportion of sample frames that must pass 5 ctx1jacbe

524 # Warning if brightness not within range (3/4 of frames must be between bounds) 

525 warn_range = np.logical_and(brightness > bounds[0], brightness < bounds[1]) 5 ctx1jacbe

526 warn_range = 'PASS' if sum(warn_range) / self.n_samples > PCT_PASS else 'WARNING' 5 ctx1jacbe

527 # Fail if brightness not within twice the range or std less than threshold 

528 fail_range = np.logical_and(brightness > bounds[0] / 2, brightness < bounds[1] * 2) 5 ctx1jacbe

529 within_range = sum(fail_range) / self.n_samples > PCT_PASS 5 ctx1jacbe

530 fail_range = 'PASS' if within_range and np.std(brightness) < max_std else 'FAIL' 5 ctx1jacbe

531 return self.overall_outcome([warn_range, fail_range]) 5 ctx1jacbe

532 

533 def check_file_headers(self): (empty)

534 """Check reported frame rate matches FPGA frame rate.""" 

535 if None in (self.data['video'], self.video_meta): 5 ctx1tacbe

536 return spec.QC.NOT_SET 1 ctx1t

537 expected = self.video_meta[self.type][self.label] 5 ctx1tacbe

538 return spec.QC.PASS if self.data['video']['fps'] == expected['fps'] else spec.QC.FAIL 5 ctx1tacbe

539 

540 def check_framerate(self, threshold=1.): (empty)

541 """Check camera times match specified frame rate for camera. 

542 

543 :param threshold: The maximum absolute difference between timestamp sample rate and video 

544 frame rate. NB: Does not take into account dropped frames. 

545 """ 

546 if any(x is None for x in (self.data['timestamps'], self.video_meta)): 4 ctx1racb

547 return spec.QC.NOT_SET 1 ctx1r

548 fps = self.video_meta[self.type][self.label]['fps'] 4 ctx1racb

549 Fs = 1 / np.median(np.diff(self.data['timestamps'])) # Approx. frequency of camera 4 ctx1racb

550 return spec.QC.PASS if abs(Fs - fps) < threshold else spec.QC.FAIL, float(round(Fs, 3)) 4 ctx1racb

551 

552 def check_pin_state(self, display=False): (empty)

553 """Check the pin state reflects Bpod TTLs.""" 

554 if not data_for_keys(('video', 'pin_state', 'audio'), self.data): 4 ctx1kacb

555 return spec.QC.NOT_SET 1 ctx1k

556 size_diff = int(self.data['pin_state'].shape[0] - self.data['video']['length']) 4 ctx1kacb

557 # NB: The pin state can be high for 2 consecutive frames 

558 low2high = np.insert(np.diff(self.data['pin_state'][:, -1].astype(int)) == 1, 0, False) 4 ctx1kacb

559 # NB: Time between two consecutive TTLs can be sub-frame, so this will fail 

560 ndiff_low2high = int(self.data['audio'][::2].size - sum(low2high)) 4 ctx1kacb

561 # state_ttl_matches = ndiff_low2high == 0 

562 # Check within ms of audio times 

563 if display: 4 ctx1kacb

564 plt.Figure() 1 ctx1k

565 plt.plot(self.data['timestamps'][low2high], np.zeros(sum(low2high)), 'o', 1 ctx1k

566 label='GPIO Low -> High') 

567 plt.plot(self.data['audio'], np.zeros(self.data['audio'].size), 'rx', 1 ctx1k

568 label='Audio TTL High') 

569 plt.xlabel('FPGA frame times / s') 1 ctx1k

570 plt.gca().set(yticklabels=[]) 1 ctx1k

571 plt.gca().tick_params(left=False) 1 ctx1k

572 plt.legend() 1 ctx1k

573 

574 outcome = self.overall_outcome( 4 ctx1kacb

575 ('PASS' if size_diff == 0 else 'WARNING' if np.abs(size_diff) < 5 else 'FAIL', 

576 'PASS' if np.abs(ndiff_low2high) < 5 else 'WARNING') 

577 ) 

578 return outcome, ndiff_low2high, size_diff 4 ctx1kacb

579 

580 def check_dropped_frames(self, threshold=.1): (empty)

581 """Check how many frames were reported missing. 

582 

583 :param threshold: The maximum allowable percentage of dropped frames 

584 """ 

585 if not data_for_keys(('video', 'count'), self.data): 4 ctx1lacb

586 return spec.QC.NOT_SET 1 ctx1l

587 size_diff = int(self.data['count'].size - self.data['video']['length']) 4 ctx1lacb

588 strict_increase = np.all(np.diff(self.data['count']) > 0) 4 ctx1lacb

589 if not strict_increase: 4 ctx1lacb

590 n_affected = np.sum(np.invert(strict_increase)) 1 ctx1l

591 _log.info(f'frame count not strictly increasing: ' 1 ctx1l

592 f'{n_affected} frames affected ({n_affected / strict_increase.size:.2%})') 

593 return spec.QC.CRITICAL 1 ctx1l

594 dropped = np.diff(self.data['count']).astype(int) - 1 4 ctx1lacb

595 pct_dropped = (sum(dropped) / len(dropped) * 100) 4 ctx1lacb

596 # Calculate overall outcome for this check 

597 outcome = self.overall_outcome( 4 ctx1lacb

598 ('PASS' if size_diff == 0 else 'WARNING' if np.abs(size_diff) < 5 else 'FAIL', 

599 'PASS' if pct_dropped < threshold else 'FAIL') 

600 ) 

601 return outcome, int(sum(dropped)), size_diff 4 ctx1lacb

602 

603 def check_timestamps(self): (empty)

604 """Check that the camera.times array is reasonable.""" 

605 if not data_for_keys(('timestamps', 'video'), self.data): 4 ctx1sacb

606 return spec.QC.NOT_SET 1 ctx1s

607 # Check number of timestamps matches video 

608 length_matches = self.data['timestamps'].size == self.data['video'].length 4 ctx1sacb

609 # Check times are strictly increasing 

610 increasing = all(np.diff(self.data['timestamps']) > 0) 4 ctx1sacb

611 # Check times do not contain nans 

612 nanless = not np.isnan(self.data['timestamps']).any() 4 ctx1sacb

613 return spec.QC.PASS if increasing and length_matches and nanless else spec.QC.FAIL 4 ctx1sacb

614 

615 def check_camera_times(self): (empty)

616 """Check that the number of raw camera timestamps matches the number of video frames.""" 

617 if not data_for_keys(('bonsai_times', 'video'), self.data): 4 ctx1uacb

618 return spec.QC.NOT_SET 1 ctx1u

619 length_match = len(self.data['camera_times']) == self.data['video'].length 4 ctx1uacb

620 outcome = spec.QC.PASS if length_match else spec.QC.WARNING 4 ctx1uacb

621 # 1 / np.median(np.diff(self.data.camera_times)) 

622 return outcome, len(self.data['camera_times']) - self.data['video'].length 4 ctx1uacb

623 

624 def check_resolution(self): (empty)

625 """Check that the timestamps and video file resolution match what we expect.""" 

626 if self.data['video'] is None: 5 ctx1oacbe

627 return spec.QC.NOT_SET 1 ctx1o

628 actual = self.data['video'] 5 ctx1oacbe

629 expected = self.video_meta[self.type][self.label] 5 ctx1oacbe

630 match = actual['width'] == expected['width'] and actual['height'] == expected['height'] 5 ctx1oacbe

631 return spec.QC.PASS if match else spec.QC.FAIL 5 ctx1oacbe

632 

633 def check_wheel_alignment(self, tolerance=(1, 2), display=False): (empty)

634 """Check wheel motion in video correlates with the rotary encoder signal. 

635 

636 Check is skipped for body camera videos as the wheel is often obstructed 

637 

638 Parameters 

639 ---------- 

640 tolerance : int, (int, int) 

641 Maximum absolute offset in frames. If two values, the maximum value is taken as the 

642 warning threshold. 

643 display : bool 

644 If true, the wheel motion energy is plotted against the rotary encoder. 

645 

646 Returns 

647 ------- 

648 one.alf.spec.QC 

649 The outcome, one of {'NOT_SET', 'FAIL', 'WARNING', 'PASS'}. 

650 int 

651 Frame offset, i.e. by how many frames the video was shifted to match the rotary encoder 

652 signal. Negative values mean the video was shifted backwards with respect to the wheel 

653 timestamps. 

654 

655 Notes 

656 ----- 

657 - A negative frame offset typically means that there were frame TTLs at the beginning that 

658 do not correspond to any video frames (sometimes the first few frames aren't saved to 

659 disk). Since 2021-09-15 the extractor should compensate for this. 

660 """