Coverage for ibllib/io/extractors/widefield.py: 91%

1"""Data extraction from widefield binary file""" 

2 

3import logging (empty)

4import numpy as np (empty)

5import shutil (empty)

6from pathlib import Path (empty)

7import pandas as pd (empty)

8 

9import ibllib.exceptions as err (empty)

10import ibllib.io.extractors.base as extractors_base (empty)

11from ibllib.io.extractors.ephys_fpga import get_sync_fronts, get_sync_and_chn_map (empty)

12from ibllib.io.video import get_video_meta (empty)

13 

14_logger = logging.getLogger('ibllib') (empty)

15 

16try: (empty)

17 import wfield.cli as wfield_cli (empty)

18except ImportError: 

19 _logger.warning('wfield not installed') 

20 

21try: (empty)

22 from labcams.io import parse_cam_log (empty)

23except ImportError: 

24 _logger.warning('labcams not installed') 

25 

26_logger = logging.getLogger('ibllib') (empty)

27 

28"""Available LEDs for Widefield Imaging""" (empty)

29LIGHT_SOURCE_MAP = { (empty)

30 0: 'None', 

31 405: 'Violet', 

32 470: 'Blue', 

33} 

34 

35DEFAULT_WIRING_MAP = { (empty)

36 5: 470, 

37 6: 405 

38} 

39 

40 

41class Widefield(extractors_base.BaseExtractor): (empty)

42 save_names = (None, None, None, 'widefieldChannels.frameAverage.npy', 'widefieldU.images.npy', 'widefieldSVT.uncorrected.npy', (empty)

43 None, None, 'widefieldSVT.haemoCorrected.npy', 'imaging.times.npy', 'imaging.imagingLightSource.npy', 

44 'imagingLightSource.properties.htsv') 

45 raw_names = ('motioncorrect_2_540_640_uint16.bin', 'motion_correction_shifts.npy', 'motion_correction_rotation.npy', (empty)

46 'frames_average.npy', 'U.npy', 'SVT.npy', 'rcoeffs.npy', 'T.npy', 'SVTcorr.npy', 'timestamps.npy', 'led.npy', 

47 'led_properties.htsv') 

48 var_names = () (empty)

49 

50 def __init__(self, *args, **kwargs): (empty)

51 """An extractor for all widefield data""" 

52 super().__init__(*args, **kwargs) 7 ctx1bhfcaed

53 self.data_path = self.session_path.joinpath('raw_widefield_data') 7 ctx1bhfcaed

54 self.default_path = 'alf/widefield' 7 ctx1bhfcaed

55 

56 def _channel_meta(self, light_source_map=None): (empty)

57 """ 

58 Return table of light source wavelengths and corresponding colour labels. 

59 

60 Parameters 

61 ---------- 

62 light_source_map : dict 

63 An optional map of light source wavelengths (nm) used and their corresponding colour name. 

64 

65 Returns 

66 ------- 

67 pandas.DataFrame 

68 A sorted table of wavelength and colour name. 

69 """ 

70 light_source_map = light_source_map or LIGHT_SOURCE_MAP 3 ctx1hfa

71 names = ('wavelength', 'color') 3 ctx1hfa

72 meta = pd.DataFrame(sorted(light_source_map.items()), columns=names) 3 ctx1hfa

73 meta.index.rename('channel_id', inplace=True) 3 ctx1hfa

74 return meta 3 ctx1hfa

75 

76 def _channel_wiring(self): (empty)

77 try: 3 ctx1hfa

78 wiring = pd.read_csv(self.data_path.joinpath('widefieldChannels.wiring.htsv'), sep='\t') 3 ctx1hfa

79 except FileNotFoundError: 1 ctx1f

80 _logger.warning('LED wiring map not found, using default') 1 ctx1f

81 wiring = pd.DataFrame(DEFAULT_WIRING_MAP.items(), columns=('LED', 'wavelength')) 1 ctx1f

82 

83 return wiring 3 ctx1hfa

84 

85 def _extract(self, extract_timestamps=True, save=False, **kwargs): (empty)

86 """ 

87 NB: kwargs should be loaded from meta file 

88 Parameters 

89 ---------- 

90 n_channels 

91 dtype 

92 shape 

93 kwargs 

94 

95 Returns 

96 ------- 

97 

98 """ 

99 self.preprocess(**kwargs) 1 ctx1c

100 if extract_timestamps: 1 ctx1c

101 _ = self.sync_timestamps(save=save) 

102 

103 return None 1 ctx1c

104 

105 def _save(self, data=None, path_out=None): (empty)

106 

107 if not path_out: 2 ctx1gc

108 path_out = self.session_path.joinpath(self.default_path) 2 ctx1gc

109 path_out.mkdir(exist_ok=True, parents=True) 2 ctx1gc

110 

111 new_files = [] 2 ctx1gc

112 if not self.data_path.exists(): 2 ctx1gc

113 _logger.warning(f'Path does not exist: {self.data_path}') 

114 return new_files 

115 

116 for before, after in zip(self.raw_names, self.save_names): 2 ctx1gc

117 if after is None: 2 ctx1gc

118 continue 2 ctx1gc

119 else: 

120 try: 2 ctx1gc

121 file_orig = next(self.data_path.glob(before)) 2 ctx1gc

122 file_new = path_out.joinpath(after) 2 ctx1gc

123 shutil.move(file_orig, file_new) 2 ctx1gc

124 new_files.append(file_new) 2 ctx1gc

125 except StopIteration: 1 ctx1c

126 _logger.warning(f'File not found: {before}') 1 ctx1c

127 

128 return new_files 2 ctx1gc

129 

130 def preprocess(self, fs=30, functional_channel=0, nbaseline_frames=30, k=200, nchannels=2): (empty)

131 

132 # MOTION CORRECTION 

133 wfield_cli._motion(str(self.data_path), nchannels=nchannels, plot_ext='.png') 1 ctx1c

134 # COMPUTE AVERAGE FOR BASELINE 

135 wfield_cli._baseline(str(self.data_path), nbaseline_frames, nchannels=nchannels) 1 ctx1c

136 # DATA REDUCTION 

137 wfield_cli._decompose(str(self.data_path), k=k, nchannels=nchannels) 1 ctx1c

138 # HAEMODYNAMIC CORRECTION 

139 # check if it is 2 channel 

140 dat = wfield_cli.load_stack(str(self.data_path), nchannels=nchannels) 1 ctx1c

141 if dat.shape[1] == 2: 1 ctx1c

142 del dat 1 ctx1c

143 wfield_cli._hemocorrect(str(self.data_path), fs=fs, functional_channel=functional_channel, plot_ext='.png') 1 ctx1c

144 

145 def remove_files(self, file_prefix='motion'): (empty)

146 motion_files = self.data_path.glob(f'{file_prefix}*') 2 ctx1ic

147 for file in motion_files: 2 ctx1ic

148 _logger.info(f'Removing {file}') 2 ctx1ic

149 file.unlink() 2 ctx1ic

150 

151 def sync_timestamps(self, bin_exists=False, save=False, save_paths=None, sync_collection='raw_sync_data', **kwargs): (empty)

152 

153 if save and save_paths: 4 ctx1jaed

154 assert len(save_paths) == 3, 'Must provide save_path as list with 3 paths' 4 ctx1jaed

155 for save_path in save_paths: 3 ctx1aed

156 Path(save_path).parent.mkdir(parents=True, exist_ok=True) 3 ctx1aed

157 

158 # Load in fpga sync 

159 fpga_sync, chmap = get_sync_and_chn_map(self.session_path, sync_collection) 3 ctx1aed

160 fpga_led = get_sync_fronts(fpga_sync, chmap['frame_trigger']) 3 ctx1aed

161 fpga_led_up = fpga_led['times'][fpga_led['polarities'] == 1] # only consider up pulse times 3 ctx1aed

162 

163 # Load in camlog sync 

164 logdata, led, sync, ncomm = parse_cam_log(next(self.data_path.glob('*.camlog')), readTeensy=True) 3 ctx1aed

165 assert led.frame.is_monotonic_increasing 3 ctx1aed

166 

167 if led.frame.size != fpga_led_up.size: 3 ctx1aed

168 _logger.warning(f'Sync mismatch by {np.abs(led.frame.size - fpga_led_up.size)} ' 

169 f'NIDQ sync times: {fpga_led_up.size}, LED frame times {led.frame.size}') 

170 raise ValueError('Sync mismatch') 

171 

172 # Get video meta data to check number of widefield frames 

173 video_path = next(self.data_path.glob('imaging.frames*.mov')) 3 ctx1aed

174 video_meta = get_video_meta(video_path) 3 ctx1aed

175 

176 # Check for differences between video and ttl (in some cases we expect there to be extra ttl than frame, this is okay) 

177 diff = len(led) - video_meta.length 3 ctx1aed

178 if diff < 0: 3 ctx1aed

179 raise ValueError('More video frames than led frames detected') 1 ctx1e

180 if diff > 2: 2 ctx1ad

181 raise ValueError('Led frames and video frames differ by more than 2') 1 ctx1d

182 

183 # take the timestamps as those recorded on fpga, no need to do any sycning 

184 widefield_times = fpga_led_up[0:video_meta.length] 1 ctx1a

185 

186 # Now extract the LED channels and meta data 

187 # Load channel meta and wiring map 

188 channel_meta_map = self._channel_meta(kwargs.get('light_source_map')) 1 ctx1a

189 channel_wiring = self._channel_wiring() 1 ctx1a

190 channel_id = np.empty_like(led.led.values) 1 ctx1a

191 

192 for _, d in channel_wiring.iterrows(): 1 ctx1a

193 mask = led.led.values == d['LED'] 1 ctx1a

194 if np.sum(mask) == 0: 1 ctx1a

195 raise err.WidefieldWiringException 

196 channel_id[mask] = channel_meta_map.get(channel_meta_map['wavelength'] == d['wavelength']).index[0] 1 ctx1a

197 

198 if save: 1 ctx1a

199 save_time = save_paths[0] if save_paths else self.data_path.joinpath('timestamps.npy') 1 ctx1a

200 save_led = save_paths[1] if save_paths else self.data_path.joinpath('led.npy') 1 ctx1a

201 save_meta = save_paths[2] if save_paths else self.data_path.joinpath('led_properties.htsv') 1 ctx1a

202 save_paths = [save_time, save_led, save_meta] 1 ctx1a

203 np.save(save_time, widefield_times) 1 ctx1a

204 np.save(save_led, channel_id) 1 ctx1a

205 channel_meta_map.to_csv(save_meta, sep='\t') 1 ctx1a

206 

207 return save_paths 1 ctx1a

208 else: 

209 return widefield_times, channel_id, channel_meta_map