Script xalt_run_submission_in_py
[hide private]
[frames] | no frames]

Source Code for Script script-xalt_run_submission_in_py

  1  # -*- python -*- 
  2  # 
  3  # Git Version: @git@ 
  4   
  5  #----------------------------------------------------------------------- 
  6  # XALT: A tool that tracks users jobs and environments on a cluster. 
  7  # Copyright (C) 2013-2014 University of Texas at Austin 
  8  # Copyright (C) 2013-2014 University of Tennessee 
  9  #  
 10  # This library is free software; you can redistribute it and/or modify 
 11  # it under the terms of the GNU Lesser General Public License as 
 12  # published by the Free Software Foundation; either version 2.1 of  
 13  # the License, or (at your option) any later version.  
 14  # 
 15  # This library is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 18  # Lesser  General Public License for more details.  
 19  # 
 20  # You should have received a copy of the GNU Lesser General Public 
 21  # License along with this library; if not, write to the Free 
 22  # Software Foundation, Inc., 59 Temple Place, Suite 330, 
 23  # Boston, MA 02111-1307 USA 
 24  #----------------------------------------------------------------------- 
 25   
 26  from __future__ import print_function 
 27  import os, re, sys 
 28   
 29  dirNm, execName = os.path.split(sys.argv[0]) 
 30  sys.path.insert(1,os.path.abspath(os.path.join(dirNm, "../libexec"))) 
 31  sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site"))) 
 32   
 33  from xalt_util                 import capture, which, config_logger 
 34  from xalt_transmission_factory import XALT_transmission_factory 
 35  from xalt_global               import * 
 36   
 37  import subprocess, time, socket, json, argparse, platform 
 38   
 39  logger = config_logger() 
 40   
41 -def syshost():
42 """ 43 Find a default system host name. Return 2nd name after the dot, unless there is only one, then return first name. 44 """ 45 hostA = platform.node().split('.') 46 idx = 1 47 if (len(hostA) < 2): 48 idx = 0 49 return hostA[idx]
50
51 -class CmdLineOptions(object):
52 """ Command line Options class """ 53
54 - def __init__(self):
55 """ Empty Ctor """ 56 pass
57
58 - def execute(self):
59 """ Specify command line arguments and parse the command line""" 60 parser = argparse.ArgumentParser() 61 parser.add_argument("--start", dest='startTime', action="store", type=float, default="0.0", help="start time") 62 parser.add_argument("--end", dest='endTime', action="store", type=float, default="0.0", help="end time") 63 parser.add_argument("--fn", dest='resultFn', action="store", default = "/dev/null", help="resultFn") 64 parser.add_argument("--ntasks", dest='ntasks', action="store", default = "-1", help="number of mpi tasks") 65 parser.add_argument("--syshost", dest='syshost', action="store", default = syshost(), help="system host name") 66 parser.add_argument("--run_uuid",dest='run_uuid', action="store", default = None, help="run uuid") 67 parser.add_argument("exec_prog", nargs='+', help="user program") 68 69 args = parser.parse_args() 70 71 return args
72 73 keyPat = re.compile(r'.*<(.*)>.*') 74 shFuncPat = re.compile(r'^\(\) *{') 75
76 -class ExtractXALT(object):
77 """ 78 This class extracts the XALT scribe placed in the program or shared library. 79 """
80 - def __init__(self, cmd):
81 """ 82 Parse the input path for the xalt record. If it exists extract the 83 key value pairs and store in a table. 84 85 @param cmd: the path to the program or shared library that has (or could have) an XALT record. 86 """ 87 outStr = capture(["objdump", "-s", "-j", ".xalt", cmd]) 88 fieldT = {} 89 if (not outStr.find("Contents of section .xalt:") != -1): 90 return fieldT 91 92 outputA = outStr.split('\n') 93 outputA.pop(0) 94 outputA.pop(0) 95 outputA.pop(0) 96 outputA.pop(0) 97 98 sA = [] 99 for line in outputA: 100 split = line.split() 101 if (len(split) > 0): 102 sA.append(split[-1]) 103 s = "".join(sA) 104 105 xaltA = re.split('%%', s) 106 107 fieldT = {} 108 109 N = len(xaltA) 110 idx = -1 111 while (True): 112 idx = idx + 1 113 line = xaltA[idx] 114 if (line.find("XALT_Link_Info_End") != -1): 115 break 116 m = keyPat.search(line) 117 if (m): 118 key = m.group(1) 119 idx = idx + 1 120 value = xaltA[idx].replace("_%_%_"," ") 121 fieldT[key] = value 122 123 self.__fieldT = fieldT
124
125 - def xaltRecordT(self):
126 """ Return the XALT values found in cmd. """ 127 return self.__fieldT
128 129
130 -class UserEnvT(object):
131 """ Class to extract important values from the environment """
132 - def __init__(self, args, userExec):
133 """ 134 Ctor to construct the important user env values and store them in userT. 135 136 @param args: The parsed command line arguments. 137 @param userExec: the path to the user executable. 138 """ 139 ltime = time.time() 140 userT = {} 141 userT['cwd'] = os.getcwd() 142 userT['syshost'] = args.syshost 143 userT['run_uuid'] = args.run_uuid 144 userT['num_threads'] = int(os.environ.get("OMP_NUM_THREADS","0")) 145 userT['user'] = os.environ.get("USER","unknown") 146 userT['num_tasks'] = args.ntasks 147 userT['start_date'] = time.strftime("%c",time.localtime(args.startTime)) 148 userT['start_time'] = args.startTime 149 userT['currentEpoch'] = ltime 150 userT['end_time'] = args.endTime 151 userT['run_time'] = max(0, args.endTime - args.startTime) 152 userT['exec_path'] = userExec.execName() 153 userT['exec_type'] = userExec.execType() 154 userT['exec_epoch'] = userExec.execEpoch() 155 userT['execModify'] = userExec.execModify() 156 157 self.__userT = userT
158
159 - def userT(self):
160 """ return the constructed user table. """ 161 return self.__userT
162
163 -class UserExec(object):
164 """ 165 Find all about the user's executable. 166 """
167 - def __init__(self, exec_progA):
168 """ 169 Find the user's executable by walking the command line 170 and skipping the executables name in ignoreT. Then find 171 the full path to the executable. Finally find the shared 172 libraries if there and get the hash time. 173 174 @param exec_progA: the command line after the mpirun type 175 arguments have been removed. 176 """ 177 ignoreT = { 178 'env' : True, 179 'time' : True, 180 } 181 cmd = None 182 for prog in exec_progA: 183 bare = os.path.basename(prog) 184 if (not (bare in ignoreT)): 185 cmd = prog 186 break 187 188 self.__execType = None 189 self.__execName = which(cmd) 190 self.__libA = [] 191 if (self.__execName): 192 outStr = capture(["file", self.__execName]) 193 if (outStr.find("script") > 0 or outStr.find("text") > 0): 194 self.__execType = "script" 195 else: 196 self.__execType = "binary" 197 ldd = capture(["ldd", self.__execName]) 198 self.__libA = self.__parseLDD(ldd) 199 200 info = os.stat(self.__execName) 201 self.__modify = info.st_mtime 202 self.__hash = self.__computeHash(self.__execName)
203 204
205 - def execName(self):
206 """ Return the name of the executable """ 207 return self.__execName
208
209 - def execType(self):
210 """ Return the executable type: binary or script. """ 211 return self.__execType
212
213 - def execEpoch(self):
214 """ Return the executables modify time in epoch time.""" 215 return self.__modify
216
217 - def execModify(self):
218 """ Return the modify date time string. """ 219 return time.strftime("%c",time.localtime(self.__modify))
220
221 - def libA(self):
222 """ Return the array of shared libraries for this executable. """ 223 return self.__libA
224
225 - def hash(self):
226 """ Return the sha1sum of the executable. """ 227 return self.__hash
228
229 - def __computeHash(self, cmd):
230 """ Compute the sha1sum of the executable. """ 231 fieldA = capture(["sha1sum", cmd]).split() 232 return fieldA[0]
233
234 - def __parseLDD(self,ldd):
235 """ Return the list of shared libraries with their sha1sum. """ 236 if (not ldd or ldd.find("not a dynamic executable") > 0): 237 return [] 238 239 lineA = ldd.split('\n') 240 241 libA = [] 242 d = {} 243 for line in lineA: 244 fieldA = line.split() 245 N = len(fieldA) 246 247 if (N < 1): 248 break 249 elif (N == 4): 250 lib = fieldA[2] 251 else: 252 lib = fieldA[0] 253 lib = os.path.realpath(lib) 254 d[lib] = True 255 256 libA = d.keys() 257 libA = sorted(libA) 258 259 libB = [] 260 for lib in libA: 261 hash_line = capture(['sha1sum', lib]) 262 if (hash_line.find("No such file or directory") != -1): 263 v = "unknown" 264 else: 265 v = hash_line.split()[0] 266 267 libB.append([lib, v]) 268 269 270 return libB
271
272 -class EnvT(object):
273 """ Capture the user's environment. Remove some variables."""
274 - def __init__(self):
275 """ Save the users environment. """ 276 self.__envT = self.__reportUserEnv()
277 - def envT(self):
278 """ Return the user environment. """ 279 return self.__envT
280 - def __reportUserEnv(self):
281 """ 282 Walk the users environment and save every thing except for a few env. vars. 283 Also remove any exported shell functions. 284 """ 285 286 # blacklist of env vars not to track 287 ignoreKeyA = [ 288 re.compile(r'^HIST.*$'), 289 re.compile(r'^LESS_TERMCAP_.*$'), 290 re.compile(r'^COLORTERM$'), 291 re.compile(r'^LS_.*$'), 292 re.compile(r'^LANG$'), 293 re.compile(r'^LESS.*$'), 294 re.compile(r'^MORE$'), 295 re.compile(r'^PAGER$'), 296 re.compile(r'^NNTPSERVER$'), 297 re.compile(r'^NLSPATH$'), 298 re.compile(r'^MAIL$'), 299 re.compile(r'^_ModuleTable.*$'), 300 re.compile(r'^_SettargTable.*$'), 301 re.compile(r'^SSH_.*$'), 302 re.compile(r'^XDG_.*$'), 303 re.compile(r'^PS1$'), 304 ] 305 306 envT = {} 307 for k in os.environ: 308 v = os.environ[k] 309 keep = True 310 for pat in ignoreKeyA: 311 m = pat.search(k) 312 if (m): 313 keep = False 314 break 315 m = shFuncPat.search(v) 316 if (m): 317 keep = False 318 if (keep): 319 envT[k] = v 320 return envT
321 322
323 -def main():
324 """ 325 Write the environment, XALT info from executable and use the 326 transmission factory to save it. 327 """ 328 329 try: 330 # parse command line options: 331 args = CmdLineOptions().execute() 332 333 userExec = UserExec(args.exec_prog) 334 if (not userExec.execName()): 335 return 336 337 userT = UserEnvT(args, userExec).userT() 338 339 submitT = {} 340 submitT['userT'] = userT 341 submitT['xaltLinkT'] = ExtractXALT(userExec.execName()).xaltRecordT() 342 submitT['libA'] = userExec.libA() 343 submitT['envT'] = EnvT().envT() 344 submitT['hash_id'] = userExec.hash() 345 346 xfer = XALT_transmission_factory.build(XALT_TRANSMISSION_STYLE, 347 args.syshost, "run", args.resultFn) 348 xfer.save(submitT) 349 except Exception as e: 350 print("XALT_EXCEPTION(xalt_run_submission.py): ",e) 351 logger.exception("XALT_EXCEPTION:xalt_run_submission.py")
352 353 354 if ( __name__ == '__main__'): main() 355