Coverage for ibllib/io/extractors/widefield.py: 94%

1"""Data extraction from widefield binary file""" 

2 

3import logging (empty)

4import numpy as np (empty)

5import shutil (empty)

6from pathlib import Path (empty)

7import pandas as pd (empty)

8 

9import ibllib.exceptions as err (empty)

10import ibllib.io.extractors.base as extractors_base (empty)

11from ibllib.io.extractors.ephys_fpga import get_sync_fronts, get_sync_and_chn_map (empty)

12from ibllib.io.video import get_video_meta (empty)

13 

14import wfield.cli as wfield_cli (empty)

15from labcams.io import parse_cam_log (empty)

16 

17_logger = logging.getLogger('ibllib') (empty)

18 

19"""Available LEDs for Widefield Imaging""" (empty)

20LIGHT_SOURCE_MAP = { (empty)

21 0: 'None', 

22 405: 'Violet', 

23 470: 'Blue', 

24} 

25 

26DEFAULT_WIRING_MAP = { (empty)

27 5: 470, 

28 6: 405 

29} 

30 

31 

32class Widefield(extractors_base.BaseExtractor): (empty)

33 save_names = (None, None, None, 'widefieldChannels.frameAverage.npy', 'widefieldU.images.npy', 'widefieldSVT.uncorrected.npy', (empty)

34 None, None, 'widefieldSVT.haemoCorrected.npy', 'imaging.times.npy', 'imaging.imagingLightSource.npy', 

35 'imagingLightSource.properties.htsv') 

36 raw_names = ('motioncorrect_2_540_640_uint16.bin', 'motion_correction_shifts.npy', 'motion_correction_rotation.npy', (empty)

37 'frames_average.npy', 'U.npy', 'SVT.npy', 'rcoeffs.npy', 'T.npy', 'SVTcorr.npy', 'timestamps.npy', 'led.npy', 

38 'led_properties.htsv') 

39 var_names = () (empty)

40 

41 def __init__(self, *args, **kwargs): (empty)

42 """An extractor for all widefield data""" 

43 super().__init__(*args, **kwargs) 7 ctx1chfbaed

44 self.data_path = self.session_path.joinpath('raw_widefield_data') 7 ctx1chfbaed

45 self.default_path = 'alf/widefield' 7 ctx1chfbaed

46 

47 def _channel_meta(self, light_source_map=None): (empty)

48 """ 

49 Return table of light source wavelengths and corresponding colour labels. 

50 

51 Parameters 

52 ---------- 

53 light_source_map : dict 

54 An optional map of light source wavelengths (nm) used and their corresponding colour name. 

55 

56 Returns 

57 ------- 

58 pandas.DataFrame 

59 A sorted table of wavelength and colour name. 

60 """ 

61 light_source_map = light_source_map or LIGHT_SOURCE_MAP 3 ctx1hfa

62 names = ('wavelength', 'color') 3 ctx1hfa

63 meta = pd.DataFrame(sorted(light_source_map.items()), columns=names) 3 ctx1hfa

64 meta.index.rename('channel_id', inplace=True) 3 ctx1hfa

65 return meta 3 ctx1hfa

66 

67 def _channel_wiring(self): (empty)

68 try: 3 ctx1hfa

69 wiring = pd.read_csv(self.data_path.joinpath('widefieldChannels.wiring.htsv'), sep='\t') 3 ctx1hfa

70 except FileNotFoundError: 1 ctx1f

71 _logger.warning('LED wiring map not found, using default') 1 ctx1f

72 wiring = pd.DataFrame(DEFAULT_WIRING_MAP.items(), columns=('LED', 'wavelength')) 1 ctx1f

73 

74 return wiring 3 ctx1hfa

75 

76 def _extract(self, extract_timestamps=True, save=False, **kwargs): (empty)

77 """ 

78 NB: kwargs should be loaded from meta file 

79 Parameters 

80 ---------- 

81 n_channels 

82 dtype 

83 shape 

84 kwargs 

85 

86 Returns 

87 ------- 

88 

89 """ 

90 self.preprocess(**kwargs) 1 ctx1b

91 if extract_timestamps: 1 ctx1b

92 _ = self.sync_timestamps(save=save) 

93 

94 return None 1 ctx1b

95 

96 def _save(self, data=None, path_out=None): (empty)

97 

98 if not path_out: 2 ctx1gb

99 path_out = self.session_path.joinpath(self.default_path) 2 ctx1gb

100 path_out.mkdir(exist_ok=True, parents=True) 2 ctx1gb

101 

102 new_files = [] 2 ctx1gb

103 if not self.data_path.exists(): 2 ctx1gb

104 _logger.warning(f'Path does not exist: {self.data_path}') 

105 return new_files 

106 

107 for before, after in zip(self.raw_names, self.save_names): 2 ctx1gb

108 if after is None: 2 ctx1gb

109 continue 2 ctx1gb

110 else: 

111 try: 2 ctx1gb

112 file_orig = next(self.data_path.glob(before)) 2 ctx1gb

113 file_new = path_out.joinpath(after) 2 ctx1gb

114 shutil.move(file_orig, file_new) 2 ctx1gb

115 new_files.append(file_new) 2 ctx1gb

116 except StopIteration: 1 ctx1b

117 _logger.warning(f'File not found: {before}') 1 ctx1b

118 

119 return new_files 2 ctx1gb

120 

121 def preprocess(self, fs=30, functional_channel=0, nbaseline_frames=30, k=200, nchannels=2): (empty)

122 

123 # MOTION CORRECTION 

124 wfield_cli._motion(str(self.data_path), nchannels=nchannels, plot_ext='.png') 1 ctx1b

125 # COMPUTE AVERAGE FOR BASELINE 

126 wfield_cli._baseline(str(self.data_path), nbaseline_frames, nchannels=nchannels) 1 ctx1b

127 # DATA REDUCTION 

128 wfield_cli._decompose(str(self.data_path), k=k, nchannels=nchannels) 1 ctx1b

129 # HAEMODYNAMIC CORRECTION 

130 # check if it is 2 channel 

131 dat = wfield_cli.load_stack(str(self.data_path), nchannels=nchannels) 1 ctx1b

132 if dat.shape[1] == 2: 1 ctx1b

133 del dat 1 ctx1b

134 wfield_cli._hemocorrect(str(self.data_path), fs=fs, functional_channel=functional_channel, plot_ext='.png') 1 ctx1b

135 

136 def remove_files(self, file_prefix='motion'): (empty)

137 motion_files = self.data_path.glob(f'{file_prefix}*') 2 ctx1ib

138 for file in motion_files: 2 ctx1ib

139 _logger.info(f'Removing {file}') 2 ctx1ib

140 file.unlink() 2 ctx1ib

141 

142 def sync_timestamps(self, bin_exists=False, save=False, save_paths=None, sync_collection='raw_sync_data', **kwargs): (empty)

143 

144 if save and save_paths: 4 ctx1jaed

145 assert len(save_paths) == 3, 'Must provide save_path as list with 3 paths' 4 ctx1jaed

146 for save_path in save_paths: 3 ctx1aed

147 Path(save_path).parent.mkdir(parents=True, exist_ok=True) 3 ctx1aed

148 

149 # Load in fpga sync 

150 fpga_sync, chmap = get_sync_and_chn_map(self.session_path, sync_collection) 3 ctx1aed

151 fpga_led = get_sync_fronts(fpga_sync, chmap['frame_trigger']) 3 ctx1aed

152 fpga_led_up = fpga_led['times'][fpga_led['polarities'] == 1] # only consider up pulse times 3 ctx1aed

153 

154 # Load in camlog sync 

155 logdata, led, sync, ncomm = parse_cam_log(next(self.data_path.glob('*.camlog')), readTeensy=True) 3 ctx1aed

156 assert led.frame.is_monotonic_increasing 3 ctx1aed

157 

158 if led.frame.size != fpga_led_up.size: 3 ctx1aed

159 _logger.warning(f'Sync mismatch by {np.abs(led.frame.size - fpga_led_up.size)} ' 

160 f'NIDQ sync times: {fpga_led_up.size}, LED frame times {led.frame.size}') 

161 raise ValueError('Sync mismatch') 

162 

163 # Get video meta data to check number of widefield frames 

164 video_path = next(self.data_path.glob('imaging.frames*.mov')) 3 ctx1aed

165 video_meta = get_video_meta(video_path) 3 ctx1aed

166 

167 # Check for differences between video and ttl (in some cases we expect there to be extra ttl than frame, this is okay) 

168 diff = len(led) - video_meta.length 3 ctx1aed

169 if diff < 0: 3 ctx1aed

170 raise ValueError('More video frames than led frames detected') 1 ctx1e

171 if diff > 2: 2 ctx1ad

172 raise ValueError('Led frames and video frames differ by more than 2') 1 ctx1d

173 

174 # take the timestamps as those recorded on fpga, no need to do any sycning 

175 widefield_times = fpga_led_up[0:video_meta.length] 1 ctx1a

176 

177 # Now extract the LED channels and meta data 

178 # Load channel meta and wiring map 

179 channel_meta_map = self._channel_meta(kwargs.get('light_source_map')) 1 ctx1a

180 channel_wiring = self._channel_wiring() 1 ctx1a

181 channel_id = np.empty_like(led.led.values) 1 ctx1a

182 

183 for _, d in channel_wiring.iterrows(): 1 ctx1a

184 mask = led.led.values == d['LED'] 1 ctx1a

185 if np.sum(mask) == 0: 1 ctx1a

186 raise err.WidefieldWiringException 

187 channel_id[mask] = channel_meta_map.get(channel_meta_map['wavelength'] == d['wavelength']).index[0] 1 ctx1a

188 

189 if save: 1 ctx1a

190 save_time = save_paths[0] if save_paths else self.data_path.joinpath('timestamps.npy') 1 ctx1a

191 save_led = save_paths[1] if save_paths else self.data_path.joinpath('led.npy') 1 ctx1a

192 save_meta = save_paths[2] if save_paths else self.data_path.joinpath('led_properties.htsv') 1 ctx1a

193 save_paths = [save_time, save_led, save_meta] 1 ctx1a

194 np.save(save_time, widefield_times) 1 ctx1a

195 np.save(save_led, channel_id) 1 ctx1a

196 channel_meta_map.to_csv(save_meta, sep='\t') 1 ctx1a

197 

198 return save_paths 1 ctx1a

199 else: 

200 return widefield_times, channel_id, channel_meta_map