Coverage for ibllib/pipes/purge_rig_data.py: 0%

1""" 

2Purge data from acquisition PC. 

3 

4Steps: 

5 

6- Find all files by rglob 

7- Find all sessions of the found files 

8- Check Alyx if corresponding datasetTypes have been registered as existing 

9 sessions and files on Flatiron 

10- Delete local raw file if found on Flatiron 

11""" 

12import argparse 

13from pathlib import Path 

14 

15from one.api import ONE 

16from one.alf.files import get_session_path 

17 

18 

19def session_name(path) -> str: 

20 """Returns the session name (subject/date/number) string for any filepath 

21 using session_path""" 

22 return '/'.join(get_session_path(path).parts[-3:]) 

23 

24 

25def purge_local_data(local_folder, file_name, lab=None, dry=False): 

26 # Figure out datasetType from file_name or file path 

27 file_name = Path(file_name).name 

28 alf_parts = file_name.split('.') 

29 dstype = '.'.join(alf_parts[:2]) 

30 print(f'Looking for file <{file_name}> in folder <{local_folder}>') 

31 # Get all paths for file_name in local folder 

32 local_folder = Path(local_folder) 

33 files = list(local_folder.rglob(f'*{file_name}')) 

34 print(f'Found {len(files)} files') 

35 print(f'Checking on Flatiron for datsetType: {dstype}...') 

36 # Get all sessions and details from Alyx that have the dstype 

37 one = ONE(cache_rest=None) 

38 if lab is None: 

39 eid, det = one.search(dataset_types=[dstype], details=True) 

40 else: 

41 eid, det = one.search(dataset_types=[dstype], lab=lab, details=True) 

42 urls = [] 

43 for d in det: 

44 urls.extend([x['data_url'] for x in d['data_dataset_session_related'] 

45 if x['dataset_type'] == dstype]) 

46 # Remove None answers when session is registered but dstype not htere yet 

47 urls = [u for u in urls if u is not None] 

48 print(f'Found files on Flatiron: {len(urls)}') 

49 to_remove = [] 

50 for f in files: 

51 sess_name = session_name(f) 

52 for u in urls: 

53 if sess_name in u: 

54 to_remove.append(f) 

55 print(f'Local files to remove: {len(to_remove)}') 

56 for f in to_remove: 

57 print(f) 

58 if dry: 

59 continue 

60 else: 

61 f.unlink() 

62 return 

63 

64 

65if __name__ == "__main__": 

66 parser = argparse.ArgumentParser(description='Delete files from rig') 

67 parser.add_argument('folder', help='Local iblrig_data folder') 

68 parser.add_argument( 

69 'file', help='File name to search and destroy for every session') 

70 parser.add_argument('-lab', required=False, default=None, 

71 help='Lab name, search on Alyx faster. default: None') 

72 parser.add_argument('--dry', required=False, default=False, 

73 action='store_true', help='Dry run? default: False') 

74 args = parser.parse_args() 

75 purge_local_data(args.folder, args.file, lab=args.lab, dry=args.dry) 

76 print('Done\n')