Script xalt_run_submission_in_py
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26 from __future__ import print_function
27 import os, re, sys
28
29 dirNm, execName = os.path.split(sys.argv[0])
30 sys.path.insert(1,os.path.abspath(os.path.join(dirNm, "../libexec")))
31 sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site")))
32
33 from xalt_util import capture, which, config_logger
34 from xalt_transmission_factory import XALT_transmission_factory
35 from xalt_global import *
36
37 import subprocess, time, socket, json, argparse, platform
38
39 logger = config_logger()
40
42 """
43 Find a default system host name. Return 2nd name after the dot, unless there is only one, then return first name.
44 """
45 hostA = platform.node().split('.')
46 idx = 1
47 if (len(hostA) < 2):
48 idx = 0
49 return hostA[idx]
50
52 """ Command line Options class """
53
55 """ Empty Ctor """
56 pass
57
59 """ Specify command line arguments and parse the command line"""
60 parser = argparse.ArgumentParser()
61 parser.add_argument("--start", dest='startTime', action="store", type=float, default="0.0", help="start time")
62 parser.add_argument("--end", dest='endTime', action="store", type=float, default="0.0", help="end time")
63 parser.add_argument("--fn", dest='resultFn', action="store", default = "/dev/null", help="resultFn")
64 parser.add_argument("--ntasks", dest='ntasks', action="store", default = "-1", help="number of mpi tasks")
65 parser.add_argument("--syshost", dest='syshost', action="store", default = syshost(), help="system host name")
66 parser.add_argument("--run_uuid",dest='run_uuid', action="store", default = None, help="run uuid")
67 parser.add_argument("exec_prog", nargs='+', help="user program")
68
69 args = parser.parse_args()
70
71 return args
72
73 keyPat = re.compile(r'.*<(.*)>.*')
74 shFuncPat = re.compile(r'^\(\) *{')
75
77 """
78 This class extracts the XALT scribe placed in the program or shared library.
79 """
81 """
82 Parse the input path for the xalt record. If it exists extract the
83 key value pairs and store in a table.
84
85 @param cmd: the path to the program or shared library that has (or could have) an XALT record.
86 """
87 outStr = capture(["objdump", "-s", "-j", ".xalt", cmd])
88 fieldT = {}
89 if (not outStr.find("Contents of section .xalt:") != -1):
90 return fieldT
91
92 outputA = outStr.split('\n')
93 outputA.pop(0)
94 outputA.pop(0)
95 outputA.pop(0)
96 outputA.pop(0)
97
98 sA = []
99 for line in outputA:
100 split = line.split()
101 if (len(split) > 0):
102 sA.append(split[-1])
103 s = "".join(sA)
104
105 xaltA = re.split('%%', s)
106
107 fieldT = {}
108
109 N = len(xaltA)
110 idx = -1
111 while (True):
112 idx = idx + 1
113 line = xaltA[idx]
114 if (line.find("XALT_Link_Info_End") != -1):
115 break
116 m = keyPat.search(line)
117 if (m):
118 key = m.group(1)
119 idx = idx + 1
120 value = xaltA[idx].replace("_%_%_"," ")
121 fieldT[key] = value
122
123 self.__fieldT = fieldT
124
126 """ Return the XALT values found in cmd. """
127 return self.__fieldT
128
129
131 """ Class to extract important values from the environment """
133 """
134 Ctor to construct the important user env values and store them in userT.
135
136 @param args: The parsed command line arguments.
137 @param userExec: the path to the user executable.
138 """
139 ltime = time.time()
140 userT = {}
141 userT['cwd'] = os.getcwd()
142 userT['syshost'] = args.syshost
143 userT['run_uuid'] = args.run_uuid
144 userT['num_threads'] = int(os.environ.get("OMP_NUM_THREADS","0"))
145 userT['user'] = os.environ.get("USER","unknown")
146 userT['num_tasks'] = args.ntasks
147 userT['start_date'] = time.strftime("%c",time.localtime(args.startTime))
148 userT['start_time'] = args.startTime
149 userT['currentEpoch'] = ltime
150 userT['end_time'] = args.endTime
151 userT['run_time'] = max(0, args.endTime - args.startTime)
152 userT['exec_path'] = userExec.execName()
153 userT['exec_type'] = userExec.execType()
154 userT['exec_epoch'] = userExec.execEpoch()
155 userT['execModify'] = userExec.execModify()
156
157 self.__userT = userT
158
160 """ return the constructed user table. """
161 return self.__userT
162
164 """
165 Find all about the user's executable.
166 """
168 """
169 Find the user's executable by walking the command line
170 and skipping the executables name in ignoreT. Then find
171 the full path to the executable. Finally find the shared
172 libraries if there and get the hash time.
173
174 @param exec_progA: the command line after the mpirun type
175 arguments have been removed.
176 """
177 ignoreT = {
178 'env' : True,
179 'time' : True,
180 }
181 cmd = None
182 for prog in exec_progA:
183 bare = os.path.basename(prog)
184 if (not (bare in ignoreT)):
185 cmd = prog
186 break
187
188 self.__execType = None
189 self.__execName = which(cmd)
190 self.__libA = []
191 if (self.__execName):
192 outStr = capture(["file", self.__execName])
193 if (outStr.find("script") > 0 or outStr.find("text") > 0):
194 self.__execType = "script"
195 else:
196 self.__execType = "binary"
197 ldd = capture(["ldd", self.__execName])
198 self.__libA = self.__parseLDD(ldd)
199
200 info = os.stat(self.__execName)
201 self.__modify = info.st_mtime
202 self.__hash = self.__computeHash(self.__execName)
203
204
206 """ Return the name of the executable """
207 return self.__execName
208
210 """ Return the executable type: binary or script. """
211 return self.__execType
212
214 """ Return the executables modify time in epoch time."""
215 return self.__modify
216
218 """ Return the modify date time string. """
219 return time.strftime("%c",time.localtime(self.__modify))
220
222 """ Return the array of shared libraries for this executable. """
223 return self.__libA
224
226 """ Return the sha1sum of the executable. """
227 return self.__hash
228
230 """ Compute the sha1sum of the executable. """
231 fieldA = capture(["sha1sum", cmd]).split()
232 return fieldA[0]
233
235 """ Return the list of shared libraries with their sha1sum. """
236 if (not ldd or ldd.find("not a dynamic executable") > 0):
237 return []
238
239 lineA = ldd.split('\n')
240
241 libA = []
242 d = {}
243 for line in lineA:
244 fieldA = line.split()
245 N = len(fieldA)
246
247 if (N < 1):
248 break
249 elif (N == 4):
250 lib = fieldA[2]
251 else:
252 lib = fieldA[0]
253 lib = os.path.realpath(lib)
254 d[lib] = True
255
256 libA = d.keys()
257 libA = sorted(libA)
258
259 libB = []
260 for lib in libA:
261 hash_line = capture(['sha1sum', lib])
262 if (hash_line.find("No such file or directory") != -1):
263 v = "unknown"
264 else:
265 v = hash_line.split()[0]
266
267 libB.append([lib, v])
268
269
270 return libB
271
273 """ Capture the user's environment. Remove some variables."""
275 """ Save the users environment. """
276 self.__envT = self.__reportUserEnv()
278 """ Return the user environment. """
279 return self.__envT
281 """
282 Walk the users environment and save every thing except for a few env. vars.
283 Also remove any exported shell functions.
284 """
285
286
287 ignoreKeyA = [
288 re.compile(r'^HIST.*$'),
289 re.compile(r'^LESS_TERMCAP_.*$'),
290 re.compile(r'^COLORTERM$'),
291 re.compile(r'^LS_.*$'),
292 re.compile(r'^LANG$'),
293 re.compile(r'^LESS.*$'),
294 re.compile(r'^MORE$'),
295 re.compile(r'^PAGER$'),
296 re.compile(r'^NNTPSERVER$'),
297 re.compile(r'^NLSPATH$'),
298 re.compile(r'^MAIL$'),
299 re.compile(r'^_ModuleTable.*$'),
300 re.compile(r'^_SettargTable.*$'),
301 re.compile(r'^SSH_.*$'),
302 re.compile(r'^XDG_.*$'),
303 re.compile(r'^PS1$'),
304 ]
305
306 envT = {}
307 for k in os.environ:
308 v = os.environ[k]
309 keep = True
310 for pat in ignoreKeyA:
311 m = pat.search(k)
312 if (m):
313 keep = False
314 break
315 m = shFuncPat.search(v)
316 if (m):
317 keep = False
318 if (keep):
319 envT[k] = v
320 return envT
321
322
324 """
325 Write the environment, XALT info from executable and use the
326 transmission factory to save it.
327 """
328
329 try:
330
331 args = CmdLineOptions().execute()
332
333 userExec = UserExec(args.exec_prog)
334 if (not userExec.execName()):
335 return
336
337 userT = UserEnvT(args, userExec).userT()
338
339 submitT = {}
340 submitT['userT'] = userT
341 submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT()
342 submitT['libA'] = userExec.libA()
343 submitT['envT'] = EnvT().envT()
344 submitT['hash_id'] = userExec.hash()
345
346 xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE,
347 args.syshost, "run", args.resultFn)
348 xfer.save(submitT)
349 except Exception as e:
350 print("XALT_EXCEPTION(xalt_run_submission.py): ",e)
351 logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
352
353
354 if ( __name__ == '__main__'): main()
355