Script xalt_file_to_db_in_py
[hide private]
[frames] | no frames]

Source Code for Script script-xalt_file_to_db_in_py

  1  #!/usr/bin/env python 
  2  # -*- python -*- 
  3  # 
  4  # Git Version: @git@ 
  5   
  6  #----------------------------------------------------------------------- 
  7  # XALT: A tool that tracks users jobs and environments on a cluster. 
  8  # Copyright (C) 2013-2014 University of Texas at Austin 
  9  # Copyright (C) 2013-2014 University of Tennessee 
 10  #  
 11  # This library is free software; you can redistribute it and/or modify 
 12  # it under the terms of the GNU Lesser General Public License as 
 13  # published by the Free Software Foundation; either version 2.1 of  
 14  # the License, or (at your option) any later version.  
 15  # 
 16  # This library is distributed in the hope that it will be useful, 
 17  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 18  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 19  # Lesser  General Public License for more details.  
 20  # 
 21  # You should have received a copy of the GNU Lesser General Public 
 22  # License along with this library; if not, write to the Free 
 23  # Software Foundation, Inc., 59 Temple Place, Suite 330, 
 24  # Boston, MA 02111-1307 USA 
 25  #----------------------------------------------------------------------- 
 26   
 27  #  xalt_json_to_db takes the output found in the ~/.xalt.d/[link,run]* 
 28  #  output files and puts it into the database 
 29  # 
 30  #  optional input: 
 31  #    XALT_USERS:  colon separated list of users; only these users are  
 32  #       considered instead of all 
 33  # 
 34   
 35  from __future__  import print_function 
 36  import os, sys, re, MySQLdb, json, time, argparse 
 37   
 38  dirNm, execName = os.path.split(os.path.realpath(sys.argv[0])) 
 39  sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../libexec"))) 
 40  sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site"))) 
 41   
 42  from XALTdb        import XALTdb 
 43  from xalt_site_pkg import translate 
 44  from xalt_util     import * 
 45  from xalt_global   import * 
 46  from progressBar   import ProgressBar 
 47  from XALT_Rmap     import Rmap 
 48  import warnings, getent 
 49  warnings.filterwarnings("ignore", "Unknown table.*") 
 50   
 51  logger       = config_logger() 
 52   
53 -class CmdLineOptions(object):
54 """ Command line Options class """ 55
56 - def __init__(self):
57 """ Empty Ctor """ 58 pass
59
60 - def execute(self):
61 """ Specify command line arguments and parse the command line""" 62 parser = argparse.ArgumentParser() 63 parser.add_argument("--delete", dest='delete', action="store_true", help="delete files after reading") 64 parser.add_argument("--timer", dest='timer', action="store_true", help="Time runtime") 65 parser.add_argument("--report_file", dest='listFn', action="store_true", help="list file") 66 parser.add_argument("--reverseMapD", dest='rmapD', action="store", help="Path to the directory containing the json reverseMap") 67 parser.add_argument("--dbname", dest='dbname', action="store", default="xalt", help="Name of the database") 68 args = parser.parse_args() 69 return args
70 71 111 112
113 -def run_json_to_db(xalt, listFn, reverseMapT, runFnA):
114 """ 115 Reads in each run file name and converts json to python table and sends it to be written to DB. 116 117 @param xalt: An XALTdb object. 118 @param listFn: A flag that causes the name of the file to be written to stderr. 119 @param reverseMapT: The Reverse Map Table. 120 @param runFnA: An array of run file names 121 122 """ 123 num = 0 124 query = "" 125 try: 126 for fn in runFnA: 127 if (listFn): 128 sys.stderr.write(fn+"\n") 129 XALT_Stack.push("fn: "+fn) 130 num += 1 131 f = open(fn,"r") 132 133 try: 134 runT = json.loads(f.read()) 135 except: 136 f.close() 137 v = XALT_Stack.pop() 138 carp("fn",v) 139 continue 140 f.close() 141 142 xalt.run_to_db(reverseMapT, runT) 143 v = XALT_Stack.pop() 144 carp("fn",v) 145 146 except Exception as e: 147 print(XALT_Stack.contents()) 148 print(query.encode("ascii","ignore")) 149 print ("run_json_to_db(): Error:",e) 150 sys.exit (1) 151 return num
152
153 -def passwd_generator():
154 """ 155 This generator walks the /etc/passwd file and returns the next 156 user and home directory. If XALT_USERS is set then it used that 157 instead. It is a colon separated list. 158 159 Super hack: if the colon separated list has a ";" in it then the 160 first part is the user the second is the home directory. This is 161 use in testing. 162 """ 163 164 xaltUserA = os.environ.get("XALT_USERS") 165 if (xaltUserA): 166 for user in xaltUserA.split(":"): 167 idx = user.find(";") 168 if (idx != -1): 169 hdir = user[idx+1:] 170 user = user[:idx] 171 else: 172 hdir = os.path.expanduser("~" + user) 173 yield user, hdir 174 175 else: 176 for entry in getent.passwd(): 177 yield entry.name, entry.dir
178 179 180
181 -def main():
182 """ 183 Walks the list of users via the passwd_generator and load the 184 link and run files. 185 """ 186 187 # Push command line on to XALT_Stack 188 sA = [] 189 sA.append("CommandLine:") 190 for v in sys.argv: 191 sA.append('"'+v+'"') 192 XALT_Stack.push(" ".join(sA)) 193 194 args = CmdLineOptions().execute() 195 xalt = XALTdb(dbConfigFn(args.dbname)) 196 197 num = int(capture("getent passwd | wc -l")) 198 pbar = ProgressBar(maxVal=num) 199 icnt = 0 200 201 t1 = time.time() 202 203 rmapT = Rmap(args.rmapD).reverseMapT() 204 205 iuser = 0 206 lnkCnt = 0 207 runCnt = 0 208 209 for user, hdir in passwd_generator(): 210 XALT_Stack.push("User: " + user) 211 xaltDir = os.path.join(hdir,".xalt.d") 212 if (os.path.isdir(xaltDir)): 213 iuser += 1 214 linkFnA = files_in_tree(xaltDir, "*/link.*.json") 215 XALT_Stack.push("link_json_to_db()") 216 lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA) 217 XALT_Stack.pop() 218 if (args.delete): 219 remove_files(linkFnA) 220 #remove_files(files_in_tree(xaltDir, "*/.link.*.json")) 221 222 runFnA = files_in_tree(xaltDir, "*/run.*.json") 223 XALT_Stack.push("run_json_to_db()") 224 runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA) 225 XALT_Stack.pop() 226 if (args.delete): 227 remove_files(runFnA) 228 #remove_files(files_in_tree(xaltDir, "*/.run.*.json")) 229 icnt += 1 230 v = XALT_Stack.pop() 231 carp("User",v) 232 pbar.update(icnt) 233 234 xalt.connect().close() 235 pbar.fini() 236 t2 = time.time() 237 rt = t2 - t1 238 if (args.timer): 239 print("Time: ", time.strftime("%T", time.gmtime(rt))) 240 241 print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
242 243 if ( __name__ == '__main__'): main() 244