Script xalt_syslog_to_db_in_py
[hide private]
[frames] | no frames]

Source Code for Script script-xalt_syslog_to_db_in_py

  1  #!/usr/bin/env python 
  2  # -*- python -*- 
  3  # 
  4  # Git Version: @git@ 
  5   
  6  #----------------------------------------------------------------------- 
  7  # XALT: A tool to track the programs on a cluster. 
  8  # Copyright (C) 2013-2014 Robert McLay and Mark Fahey 
  9  #  
 10  # This library is free software; you can redistribute it and/or modify 
 11  # it under the terms of the GNU Lesser General Public License as 
 12  # published by the Free Software Foundation; either version 2.1 of  
 13  # the License, or (at your option) any later version.  
 14  # 
 15  # This library is distributed in the hope that it will be useful, 
 16  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 17  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 18  # Lesser  General Public License for more details.  
 19  # 
 20  # You should have received a copy of the GNU Lesser General Public 
 21  # License along with this library; if not, write to the Free 
 22  # Software Foundation, Inc., 59 Temple Place, Suite 330, 
 23  # Boston, MA 02111-1307 USA 
 24  #----------------------------------------------------------------------- 
 25   
 26  #  xalt_syslog takes the output found in xalt syslog file 
 27  #  provided with the --syslog_file=ans argument and puts it  
 28  #  into the database 
 29  # 
 30  #  One has to set up syslog to capture the XALT output 
 31  #  Example is provided in the xalt_syslog.conf file that can  
 32  #  be put in the /etc/rsyslog.d directory. 
 33  # 
 34  # 
 35   
 36  from __future__  import print_function 
 37  import os, sys, re, MySQLdb, json, time, argparse, base64 
 38   
 39  dirNm, execName = os.path.split(os.path.realpath(sys.argv[0])) 
 40  sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../libexec"))) 
 41  sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site"))) 
 42   
 43  from XALTdb        import XALTdb 
 44  from xalt_site_pkg import translate 
 45  from xalt_util     import * 
 46  from xalt_global   import * 
 47  from progressBar   import ProgressBar 
 48  from XALT_Rmap     import Rmap 
 49   
 50  ConfigBaseNm = "xalt_db" 
 51  ConfigFn     = ConfigBaseNm + ".conf" 
 52  logger       = config_logger() 
 53   
54 -class CmdLineOptions(object):
55 """ Command line Options class """ 56
57 - def __init__(self):
58 """ Empty Ctor """ 59 pass
60
61 - def execute(self):
62 """ Specify command line arguments and parse the command line""" 63 parser = argparse.ArgumentParser() 64 parser.add_argument("--syslog_file", dest='syslog', action="store", help="location and name of syslog file") 65 parser.add_argument("--timer", dest='timer', action="store_true", help="Time runtime") 66 parser.add_argument("--reverseMapD", dest='rmapD', action="store", help="Path to the directory containing the json reverseMap") 67 args = parser.parse_args() 68 return args
69 70
71 -def main():
72 """ 73 read from syslog file into XALT db. 74 """ 75 76 sA = [] 77 sA.append("CommandLine:") 78 for v in sys.argv: 79 sA.append('"'+v+'"') 80 XALT_Stack.push(" ".join(sA)) 81 82 args = CmdLineOptions().execute() 83 xalt = XALTdb(ConfigFn) 84 85 syslogFile = args.syslog 86 87 num = int(capture("cat "+syslogFile+" | wc -l")) 88 pbar = ProgressBar(maxVal=num) 89 icnt = 0 90 91 t1 = time.time() 92 93 rmapT = Rmap(args.rmapD).reverseMapT() 94 95 lnkCnt = 0 96 runCnt = 0 97 count=0 98 99 if (os.path.isfile(syslogFile)): 100 f=open(syslogFile, 'r') 101 for line in f: 102 if "XALT_LOGGING" in line: 103 array=line.split(":") 104 date = array[0] + ":" + array[1] 105 type = array[3].strip() 106 syshost = array[4].strip() 107 resultT=json.loads(base64.b64decode(array[5])) 108 XALT_Stack.push("XALT_LOGGING: " + date + " " + type + " " + syshost) 109 # XALT_Stack.push("XALT_LOGGING resultT: " + resultT) 110 111 if ( type == "link" ): 112 XALT_Stack.push("link_to_db()") 113 xalt.link_to_db(rmapT, resultT) 114 XALT_Stack.pop() 115 lnkCnt += 1 116 elif ( kind == "run" ): 117 XALT_Stack.push("run_to_db()") 118 xalt.run_to_db(rmapT, resultT) 119 XALT_Stack.pop() 120 runCnt += 1 121 else: 122 print("Error in xalt_syslog_to_db") 123 XALT_Stack.pop() 124 count += 1 125 pbar.update(count) 126 127 # what should be done if there are errors? 128 # what went wrong? 129 # how do we restart? 130 # 131 # xalt.connect().close() 132 pbar.fini() 133 t2 = time.time() 134 rt = t2 - t1 135 if (args.timer): 136 print("Time: ", time.strftime("%T", time.gmtime(rt))) 137 # 138 print("total processed : ", count, ", num links: ", lnkCnt, ", num runs: ", runCnt)
139 140 if ( __name__ == '__main__'): main() 141