Module XALTdb
[hide private]
[frames] | no frames]

Source Code for Module XALTdb

  1  #----------------------------------------------------------------------- 
  2  # XALT: A tool that tracks users jobs and environments on a cluster. 
  3  # Copyright (C) 2013-2014 University of Texas at Austin 
  4  # Copyright (C) 2013-2014 University of Tennessee 
  5  #  
  6  # This library is free software; you can redistribute it and/or modify 
  7  # it under the terms of the GNU Lesser General Public License as 
  8  # published by the Free Software Foundation; either version 2.1 of  
  9  # the License, or (at your option) any later version.  
 10  # 
 11  # This library is distributed in the hope that it will be useful, 
 12  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
 13  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 
 14  # Lesser  General Public License for more details.  
 15  # 
 16  # You should have received a copy of the GNU Lesser General Public 
 17  # License along with this library; if not, write to the Free 
 18  # Software Foundation, Inc., 59 Temple Place, Suite 330, 
 19  # Boston, MA 02111-1307 USA 
 20  #----------------------------------------------------------------------- 
 21   
 22  from __future__ import print_function 
 23  import os, sys, re, base64 
 24  dirNm, execName = os.path.split(os.path.realpath(sys.argv[0])) 
 25  sys.path.append(os.path.realpath(os.path.join(dirNm, "../libexec"))) 
 26  sys.path.append(os.path.realpath(os.path.join(dirNm, "../site"))) 
 27   
 28  import MySQLdb, ConfigParser, getpass, time 
 29  import warnings 
 30  from   xalt_util     import * 
 31  from   xalt_global   import * 
 32  from   xalt_site_pkg import translate 
 33  warnings.filterwarnings("ignore", "Unknown table.*") 
 34   
35 -def convertToInt(s):
36 """ 37 Convert to string to int. Protect against bad input. 38 @param s: Input string 39 @return: integer value. If bad return 0. 40 """ 41 try: 42 value = int(s) 43 except ValueError: 44 value = 0 45 return value
46 47 patSQ = re.compile("'")
48 -class XALTdb(object):
49 """ 50 This XALTdb class opens the XALT database and is responsible for 51 all the database interactions. 52 """
53 - def __init__(self, confFn):
54 """ Initialize the class and save the db config file. """ 55 self.__host = None 56 self.__user = None 57 self.__passwd = None 58 self.__db = None 59 self.__conn = None 60 self.__confFn = confFn
61
62 - def __readFromUser(self):
63 """ Ask user for database access info. (private) """ 64 65 self.__host = raw_input("Database host:") 66 self.__user = raw_input("Database user:") 67 self.__passwd = getpass.getpass("Database pass:") 68 self.__db = raw_input("Database name:")
69
70 - def __readConfig(self):
71 """ Read database access info from config file. (private)""" 72 confFn = self.__confFn 73 try: 74 config=ConfigParser.ConfigParser() 75 config.read(confFn) 76 self.__host = config.get("MYSQL","HOST") 77 self.__user = config.get("MYSQL","USER") 78 self.__passwd = base64.b64decode(config.get("MYSQL","PASSWD")) 79 self.__db = config.get("MYSQL","DB") 80 except ConfigParser.NoOptionError, err: 81 sys.stderr.write("\nCannot parse the config file\n") 82 sys.stderr.write("Switch to user input mode...\n\n") 83 self.__readFromUser()
84
85 - def connect(self, db = None):
86 """ 87 Public interface to connect to DB. 88 @param db: If this exists it will be used. 89 90 """ 91 if(os.path.exists(self.__confFn)): 92 self.__readConfig() 93 else: 94 self.__readFromUser() 95 96 try: 97 self.__conn = MySQLdb.connect (self.__host,self.__user,self.__passwd) 98 if (db): 99 cursor = self.__conn.cursor() 100 101 # If MySQL version < 4.1, comment out the line below 102 cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"") 103 cursor.execute("USE "+xalt.db()) 104 105 106 except MySQLdb.Error, e: 107 print ("XALTdb: Error %d: %s" % (e.args[0], e.args[1]), file=sys.stderr) 108 raise 109 110 return self.__conn
111 112
113 - def db(self):
114 """ Return name of db""" 115 return self.__db
116 170
171 - def load_objects(self, conn, objA, reverseMapT, syshost, tableName, index):
172 """ 173 Stores the objects that make an executable into the XALT DB. 174 @param conn: The db connection object 175 @param objA: The array of objects that are stored. 176 @param reverseMapT: The map between directories and modules 177 @param syshost: The system host name (stampede, darter), not login1.stampede.tacc.utexas.edu 178 @param tableName: Name of the object table. 179 @param index: The db index for the join table. 180 """ 181 182 try: 183 for entryA in objA: 184 object_path = entryA[0] 185 hash_id = entryA[1] 186 if (hash_id == "unknown"): 187 continue 188 189 query = "SELECT obj_id FROM xalt_object WHERE hash_id='%s' AND object_path='%s' AND syshost='%s'" % ( 190 hash_id, object_path, syshost) 191 192 conn.query(query) 193 result = conn.store_result() 194 if (result.num_rows() > 0): 195 row = result.fetch_row() 196 obj_id = int(row[0][0]) 197 else: 198 moduleName = obj2module(object_path, reverseMapT) 199 obj_kind = obj_type(object_path) 200 201 query = "INSERT into xalt_object VALUES (NULL,'%s','%s','%s',%s,NOW(),'%s') " % ( 202 object_path, syshost, hash_id, moduleName, obj_kind) 203 conn.query(query) 204 obj_id = conn.insert_id() 205 #print("obj_id: ",obj_id, ", obj_kind: ", obj_kind,", path: ", object_path, "moduleName: ", moduleName) 206 207 # Now link libraries to xalt_link record: 208 query = "INSERT into %s VALUES (NULL,'%d','%d') " % (tableName, obj_id, index) 209 conn.query(query) 210 211 except Exception as e: 212 print(XALT_Stack.contents()) 213 print(query) 214 print ("load_xalt_objects(): Error %d: %s" % (e.args[0], e.args[1])) 215 sys.exit (1)
216
217 - def run_to_db(self, reverseMapT, runT):
218 """ 219 Store the "run" data into the database. 220 @param: reverseMapT: The map between directories and modules 221 @param: runT: The run data stored in a table 222 """ 223 224 nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host'] 225 try: 226 conn = self.connect() 227 query = "USE "+self.db() 228 conn.query(query) 229 query = "START TRANSACTION" 230 conn.query(query) 231 232 233 translate(nameA, runT['envT'], runT['userT']); 234 XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host']) 235 236 dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S", 237 time.localtime(float(runT['userT']['start_time']))) 238 uuid = runT['xaltLinkT'].get('Build.UUID') 239 if (uuid): 240 uuid = "'" + uuid + "'" 241 else: 242 uuid = "NULL" 243 244 #print( "Looking for run_uuid: ",runT['userT']['run_uuid']) 245 246 query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT['userT']['run_uuid'] 247 conn.query(query) 248 249 result = conn.store_result() 250 if (result.num_rows() > 0): 251 #print("found") 252 row = result.fetch_row() 253 run_id = int(row[0][0]) 254 query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % ( 255 runT['userT']['run_time'], runT['userT']['end_time'], run_id) 256 conn.query(query) 257 v = XALT_Stack.pop() 258 carp("SUBMIT_HOST",v) 259 return 260 else: 261 #print("not found") 262 moduleName = obj2module(runT['userT']['exec_path'], reverseMapT) 263 query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%s','%s',%s,'%s') " % ( 264 runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr, 265 runT['userT']['syshost'], uuid, runT['hash_id'], 266 runT['userT']['account'], runT['userT']['exec_type'], runT['userT']['start_time'], 267 runT['userT']['end_time'], runT['userT']['run_time'], runT['userT']['num_cores'], 268 runT['userT']['num_nodes'], runT['userT']['num_threads'], runT['userT']['queue'], 269 runT['userT']['user'], runT['userT']['exec_path'], moduleName, 270 runT['userT']['cwd']) 271 conn.query(query) 272 run_id = conn.insert_id() 273 274 self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'], 275 "join_run_object", run_id) 276 277 # loop over env. vars. 278 for key in runT['envT']: 279 # use the single quote pattern to protect all the single quotes in env vars. 280 value = patSQ.sub(r"\\'", runT['envT'][key]) 281 query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key 282 conn.query(query) 283 result = conn.store_result() 284 if (result.num_rows() > 0): 285 row = result.fetch_row() 286 env_id = int(row[0][0]) 287 found = True 288 else: 289 query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key 290 conn.query(query) 291 env_id = conn.insert_id() 292 found = False 293 #print("env_id: ", env_id, ", found: ",found) 294 295 296 query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % ( 297 env_id, run_id, value.encode("ascii","ignore")) 298 conn.query(query) 299 v = XALT_Stack.pop() 300 carp("SUBMIT_HOST",v) 301 query = "COMMIT" 302 conn.query(query) 303 conn.close() 304 305 except Exception as e: 306 print(XALT_Stack.contents()) 307 print(query.encode("ascii","ignore")) 308 print ("run_to_db(): ",e) 309 sys.exit (1)
310