1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22 from __future__ import print_function
23 import os, sys, re, base64
24 dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
25 sys.path.append(os.path.realpath(os.path.join(dirNm, "../libexec")))
26 sys.path.append(os.path.realpath(os.path.join(dirNm, "../site")))
27
28 import MySQLdb, ConfigParser, getpass, time
29 import warnings
30 from xalt_util import *
31 from xalt_global import *
32 from xalt_site_pkg import translate
33 warnings.filterwarnings("ignore", "Unknown table.*")
34
36 """
37 Convert to string to int. Protect against bad input.
38 @param s: Input string
39 @return: integer value. If bad return 0.
40 """
41 try:
42 value = int(s)
43 except ValueError:
44 value = 0
45 return value
46
47 patSQ = re.compile("'")
49 """
50 This XALTdb class opens the XALT database and is responsible for
51 all the database interactions.
52 """
54 """ Initialize the class and save the db config file. """
55 self.__host = None
56 self.__user = None
57 self.__passwd = None
58 self.__db = None
59 self.__conn = None
60 self.__confFn = confFn
61
63 """ Ask user for database access info. (private) """
64
65 self.__host = raw_input("Database host:")
66 self.__user = raw_input("Database user:")
67 self.__passwd = getpass.getpass("Database pass:")
68 self.__db = raw_input("Database name:")
69
71 """ Read database access info from config file. (private)"""
72 confFn = self.__confFn
73 try:
74 config=ConfigParser.ConfigParser()
75 config.read(confFn)
76 self.__host = config.get("MYSQL","HOST")
77 self.__user = config.get("MYSQL","USER")
78 self.__passwd = base64.b64decode(config.get("MYSQL","PASSWD"))
79 self.__db = config.get("MYSQL","DB")
80 except ConfigParser.NoOptionError, err:
81 sys.stderr.write("\nCannot parse the config file\n")
82 sys.stderr.write("Switch to user input mode...\n\n")
83 self.__readFromUser()
84
86 """
87 Public interface to connect to DB.
88 @param db: If this exists it will be used.
89
90 """
91 if(os.path.exists(self.__confFn)):
92 self.__readConfig()
93 else:
94 self.__readFromUser()
95
96 try:
97 self.__conn = MySQLdb.connect (self.__host,self.__user,self.__passwd)
98 if (db):
99 cursor = self.__conn.cursor()
100
101
102 cursor.execute("SET SQL_MODE=\"NO_AUTO_VALUE_ON_ZERO\"")
103 cursor.execute("USE "+xalt.db())
104
105
106 except MySQLdb.Error, e:
107 print ("XALTdb: Error %d: %s" % (e.args[0], e.args[1]), file=sys.stderr)
108 raise
109
110 return self.__conn
111
112
114 """ Return name of db"""
115 return self.__db
116
118 """
119 Stores the link table data into the XALT db
120 @param reverseMapT: The reverse map table that maps directories to modules
121 @param linkT: The table that contains the link data.
122 """
123 query = ""
124
125 try:
126 conn = self.connect()
127 query = "USE "+self.db()
128 conn.query(query)
129 query = "START TRANSACTION"
130 conn.query(query)
131
132
133 query = "SELECT uuid FROM xalt_link WHERE uuid='%s'" % linkT['uuid']
134 conn.query(query)
135 result = conn.store_result()
136 if (result.num_rows() > 0):
137 return
138
139 build_epoch = float(linkT['build_epoch'])
140 dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
141 time.localtime(float(linkT['build_epoch'])))
142
143
144 exit_code = convertToInt(linkT['exit_code'])
145 exec_path = patSQ.sub(r"\\'", linkT['exec_path'])
146
147
148
149 query = "INSERT into xalt_link VALUES (NULL,'%s','%s','%s','%s','%s','%s','%.2f','%d','%s') " % (
150 linkT['uuid'], linkT['hash_id'], dateTimeStr,
151 linkT['link_program'], linkT['build_user'], linkT['build_syshost'],
152 build_epoch, exit_code, exec_path)
153 conn.query(query)
154 link_id = conn.insert_id()
155
156 XALT_Stack.push("load_xalt_objects():"+linkT['exec_path'])
157 self.load_objects(conn, linkT['linkA'], reverseMapT, linkT['build_syshost'],
158 "join_link_object", link_id)
159 v = XALT_Stack.pop()
160 carp("load_xalt_objects()",v)
161 query = "COMMIT"
162 conn.query(query)
163 conn.close()
164
165 except Exception as e:
166 print(XALT_Stack.contents())
167 print(query)
168 print ("link_to_db(): Error ",e)
169 sys.exit (1)
170
171 - def load_objects(self, conn, objA, reverseMapT, syshost, tableName, index):
172 """
173 Stores the objects that make an executable into the XALT DB.
174 @param conn: The db connection object
175 @param objA: The array of objects that are stored.
176 @param reverseMapT: The map between directories and modules
177 @param syshost: The system host name (stampede, darter), not login1.stampede.tacc.utexas.edu
178 @param tableName: Name of the object table.
179 @param index: The db index for the join table.
180 """
181
182 try:
183 for entryA in objA:
184 object_path = entryA[0]
185 hash_id = entryA[1]
186 if (hash_id == "unknown"):
187 continue
188
189 query = "SELECT obj_id FROM xalt_object WHERE hash_id='%s' AND object_path='%s' AND syshost='%s'" % (
190 hash_id, object_path, syshost)
191
192 conn.query(query)
193 result = conn.store_result()
194 if (result.num_rows() > 0):
195 row = result.fetch_row()
196 obj_id = int(row[0][0])
197 else:
198 moduleName = obj2module(object_path, reverseMapT)
199 obj_kind = obj_type(object_path)
200
201 query = "INSERT into xalt_object VALUES (NULL,'%s','%s','%s',%s,NOW(),'%s') " % (
202 object_path, syshost, hash_id, moduleName, obj_kind)
203 conn.query(query)
204 obj_id = conn.insert_id()
205
206
207
208 query = "INSERT into %s VALUES (NULL,'%d','%d') " % (tableName, obj_id, index)
209 conn.query(query)
210
211 except Exception as e:
212 print(XALT_Stack.contents())
213 print(query)
214 print ("load_xalt_objects(): Error %d: %s" % (e.args[0], e.args[1]))
215 sys.exit (1)
216
218 """
219 Store the "run" data into the database.
220 @param: reverseMapT: The map between directories and modules
221 @param: runT: The run data stored in a table
222 """
223
224 nameA = [ 'num_cores', 'num_nodes', 'account', 'job_id', 'queue' , 'submit_host']
225 try:
226 conn = self.connect()
227 query = "USE "+self.db()
228 conn.query(query)
229 query = "START TRANSACTION"
230 conn.query(query)
231
232
233 translate(nameA, runT['envT'], runT['userT']);
234 XALT_Stack.push("SUBMIT_HOST: "+ runT['userT']['submit_host'])
235
236 dateTimeStr = time.strftime("%Y-%m-%d %H:%M:%S",
237 time.localtime(float(runT['userT']['start_time'])))
238 uuid = runT['xaltLinkT'].get('Build.UUID')
239 if (uuid):
240 uuid = "'" + uuid + "'"
241 else:
242 uuid = "NULL"
243
244
245
246 query = "SELECT run_id FROM xalt_run WHERE run_uuid='%s'" % runT['userT']['run_uuid']
247 conn.query(query)
248
249 result = conn.store_result()
250 if (result.num_rows() > 0):
251
252 row = result.fetch_row()
253 run_id = int(row[0][0])
254 query = "UPDATE xalt_run SET run_time='%.2f', end_time='%.2f' WHERE run_id='%d'" % (
255 runT['userT']['run_time'], runT['userT']['end_time'], run_id)
256 conn.query(query)
257 v = XALT_Stack.pop()
258 carp("SUBMIT_HOST",v)
259 return
260 else:
261
262 moduleName = obj2module(runT['userT']['exec_path'], reverseMapT)
263 query = "INSERT INTO xalt_run VALUES (NULL,'%s','%s','%s', '%s',%s,'%s', '%s','%s','%.2f', '%.2f','%.2f','%d', '%d','%d','%s', '%s','%s',%s,'%s') " % (
264 runT['userT']['job_id'], runT['userT']['run_uuid'], dateTimeStr,
265 runT['userT']['syshost'], uuid, runT['hash_id'],
266 runT['userT']['account'], runT['userT']['exec_type'], runT['userT']['start_time'],
267 runT['userT']['end_time'], runT['userT']['run_time'], runT['userT']['num_cores'],
268 runT['userT']['num_nodes'], runT['userT']['num_threads'], runT['userT']['queue'],
269 runT['userT']['user'], runT['userT']['exec_path'], moduleName,
270 runT['userT']['cwd'])
271 conn.query(query)
272 run_id = conn.insert_id()
273
274 self.load_objects(conn, runT['libA'], reverseMapT, runT['userT']['syshost'],
275 "join_run_object", run_id)
276
277
278 for key in runT['envT']:
279
280 value = patSQ.sub(r"\\'", runT['envT'][key])
281 query = "SELECT env_id FROM xalt_env_name WHERE env_name='%s'" % key
282 conn.query(query)
283 result = conn.store_result()
284 if (result.num_rows() > 0):
285 row = result.fetch_row()
286 env_id = int(row[0][0])
287 found = True
288 else:
289 query = "INSERT INTO xalt_env_name VALUES(NULL, '%s')" % key
290 conn.query(query)
291 env_id = conn.insert_id()
292 found = False
293
294
295
296 query = "INSERT INTO join_run_env VALUES (NULL, '%d', '%d', '%s')" % (
297 env_id, run_id, value.encode("ascii","ignore"))
298 conn.query(query)
299 v = XALT_Stack.pop()
300 carp("SUBMIT_HOST",v)
301 query = "COMMIT"
302 conn.query(query)
303 conn.close()
304
305 except Exception as e:
306 print(XALT_Stack.contents())
307 print(query.encode("ascii","ignore"))
308 print ("run_to_db(): ",e)
309 sys.exit (1)
310