Coverage for src/km3pipe/cmd.py: 32%

1# Filename: cmd.py 

2""" 

3KM3Pipe command line utility. 

4 

5Usage: 

6 km3pipe test 

7 km3pipe update [GIT_BRANCH] 

8 km3pipe createconf [--overwrite] [--dump] 

9 km3pipe retrieve DET_ID RUN [-i -o OUTFILE] 

10 km3pipe (-h | --help) 

11 km3pipe --version 

12 

13Options: 

14 -h --help Show this screen. 

15 -m Get the MC detector file (flips the sign of DET_ID). 

16 -c CALIBR_ID Geometrical calibration ID (eg. A01466417) 

17 -i Use iRODS instead of xrootd to retrieve files. 

18 -o OUT Output folder or filename. 

19 -t T0_SET Time calibration ID (eg. A01466431) 

20 -s REGEX Regular expression to filter the runsetup name/id. 

21 DET_ID Detector ID (eg. D_ARCA001). 

22 DETECTOR Detector (eg. ARCA). 

23 GIT_BRANCH Git branch to pull (eg. develop). 

24 RUN Run number. 

25 

26""" 

27import re 

28import sys 

29import subprocess 

30import os 

31from datetime import datetime 

32import time 

33 

34import km3db 

35 

36from . import version 

37from .tools import irods_path, xrootd_path 

38from .hardware import Detector 

39 

40from km3pipe.logger import get_logger 

41 

42log = get_logger(__name__) # pylint: disable=C0103 

43 

44__author__ = "Tamas Gal" 

45__copyright__ = "Copyright 2016, Tamas Gal and the KM3NeT collaboration." 

46__credits__ = [] 

47__license__ = "MIT" 

48__maintainer__ = "Tamas Gal and Moritz Lotze" 

49__email__ = "tgal@km3net.de" 

50__status__ = "Development" 

51 

52SPS_CACHE = "/sps/km3net/repo/data/cache" 

53KM3PIPE_GIT = "https://git.km3net.de/km3py/km3pipe.git" 

54 

55 

56def run_tests(): 

57 print( 

58 "Running the km3pipe test suite. Make sure you have all the test " 

59 "dependencies and extras installed with\n\n" 

60 ' pip install "km3pipe[dev]"\n' 

61 ' pip install "km3pipe[extras]"\n' 

62 ) 

63 

64 import pytest 

65 import km3pipe 

66 

67 pytest.main([os.path.dirname(km3pipe.__file__)]) 

68 

69 

70def update_km3pipe(git_branch=""): 

71 if git_branch == "" or git_branch is None: 

72 git_branch = "master" 

73 os.system(f"pip install -U git+{KM3PIPE_GIT}@{git_branch}") 

74 

75 

76def retrieve(run_id, det_id, use_irods=False, out=None): 

77 """Retrieve run from iRODS for a given detector (O)ID""" 

78 det_id = int(det_id) 

79 

80 if use_irods: 

81 rpath = irods_path(det_id, run_id) 

82 cmd = "iget -Pv" 

83 else: 

84 rpath = xrootd_path(det_id, run_id) 

85 cmd = "xrdcp" 

86 

87 filename = os.path.basename(rpath) 

88 

89 if out is not None and os.path.isdir(out): 

90 outfile = os.path.join(out, filename) 

91 elif out is None: 

92 outfile = filename 

93 else: 

94 outfile = out 

95 

96 if os.path.exists(outfile): 

97 print("Output file '{}' already exists.".format(outfile)) 

98 return 

99 

100 cmd += " {0} {1}".format(rpath, outfile) 

101 

102 if not km3db.core.on_whitelisted_host("lyon"): 

103 subprocess.call(cmd.split()) 

104 return 

105 

106 subfolder = os.path.join(*rpath.split("/")[-3:-1]) 

107 cached_subfolder = os.path.join(SPS_CACHE, subfolder) 

108 cached_filepath = os.path.join(cached_subfolder, filename) 

109 lock_file = cached_filepath + ".in_progress" 

110 

111 if os.path.exists(lock_file): 

112 print("File is already requested, waiting for the download to finish.") 

113 for _ in range(6 * 15): # 15 minute timeout 

114 time.sleep(10) 

115 if not os.path.exists(lock_file): 

116 print("Done.") 

117 break 

118 else: 

119 print( 

120 "Timeout reached. Deleting the lock file and initiating a " 

121 "new download." 

122 ) 

123 os.remove(lock_file) 

124 

125 if not os.path.exists(cached_filepath): 

126 print("Downloading file to local SPS cache ({}).".format(SPS_CACHE)) 

127 try: 

128 os.makedirs(cached_subfolder, exist_ok=True) 

129 subprocess.call(["touch", lock_file]) 

130 subprocess.call(["chmod", "g+w", lock_file]) 

131 subprocess.call(cmd.split()) 

132 subprocess.call(["chmod", "g+w", outfile]) 

133 subprocess.call(["cp", "-p", outfile, cached_filepath]) 

134 except (OSError, KeyboardInterrupt) as e: 

135 print("\nAborting... {}".format(e)) 

136 for f in [outfile, lock_file, cached_filepath]: 

137 subprocess.call(["rm", "-f", f]) 

138 return 

139 finally: 

140 for f in [outfile, lock_file]: 

141 subprocess.call(["rm", "-f", f]) 

142 

143 subprocess.call(["ln", "-s", cached_filepath, outfile]) 

144 

145 

146def main(): 

147 from docopt import docopt 

148 

149 args = docopt( 

150 __doc__, 

151 version="KM3Pipe {}".format( 

152 version, 

153 ), 

154 ) 

155 

156 if args["test"]: 

157 run_tests() 

158 

159 if args["update"]: 

160 update_km3pipe(args["GIT_BRANCH"]) 

161 

162 if args["retrieve"]: 

163 retrieve(int(args["RUN"]), args["DET_ID"], use_irods=args["-i"], out=args["-o"])