Script xalt_file_to_db_in_py
|
|
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35 from __future__ import print_function
36 import os, sys, re, MySQLdb, json, time, argparse
37
38 dirNm, execName = os.path.split(os.path.realpath(sys.argv[0]))
39 sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../libexec")))
40 sys.path.insert(1,os.path.realpath(os.path.join(dirNm, "../site")))
41
42 from XALTdb import XALTdb
43 from xalt_site_pkg import translate
44 from xalt_util import *
45 from xalt_global import *
46 from progressBar import ProgressBar
47 from XALT_Rmap import Rmap
48 import warnings, getent
49 warnings.filterwarnings("ignore", "Unknown table.*")
50
51 logger = config_logger()
52
54 """ Command line Options class """
55
57 """ Empty Ctor """
58 pass
59
61 """ Specify command line arguments and parse the command line"""
62 parser = argparse.ArgumentParser()
63 parser.add_argument("--delete", dest='delete', action="store_true", help="delete files after reading")
64 parser.add_argument("--timer", dest='timer', action="store_true", help="Time runtime")
65 parser.add_argument("--report_file", dest='listFn', action="store_true", help="list file")
66 parser.add_argument("--reverseMapD", dest='rmapD', action="store", help="Path to the directory containing the json reverseMap")
67 parser.add_argument("--dbname", dest='dbname', action="store", default="xalt", help="Name of the database")
68 args = parser.parse_args()
69 return args
70
71
73 """
74 Reads in each link file name and converts json to python table and sends it to be written to DB.
75
76 @param xalt: An XALTdb object.
77 @param listFn: A flag that causes the name of the file to be written to stderr.
78 @param reverseMapT: The Reverse Map Table.
79 @param linkFnA: An array of link file names
80
81 """
82 num = 0
83 query = ""
84
85 try:
86 for fn in linkFnA:
87 if (listFn):
88 sys.stderr.write(fn+"\n")
89 XALT_Stack.push("fn: "+fn)
90 f = open(fn,"r")
91 try:
92 linkT = json.loads(f.read())
93 except:
94 f.close()
95 v = XALT_Stack.pop()
96 carp("fn",v)
97 continue
98
99 f.close()
100 xalt.link_to_db(reverseMapT, linkT)
101 num += 1
102 v = XALT_Stack.pop()
103 carp("fn",v)
104
105 except Exception as e:
106 print(XALT_Stack.contents())
107 print(query)
108 print ("link_json_to_db(): Error: ",e)
109 sys.exit (1)
110 return num
111
112
114 """
115 Reads in each run file name and converts json to python table and sends it to be written to DB.
116
117 @param xalt: An XALTdb object.
118 @param listFn: A flag that causes the name of the file to be written to stderr.
119 @param reverseMapT: The Reverse Map Table.
120 @param runFnA: An array of run file names
121
122 """
123 num = 0
124 query = ""
125 try:
126 for fn in runFnA:
127 if (listFn):
128 sys.stderr.write(fn+"\n")
129 XALT_Stack.push("fn: "+fn)
130 num += 1
131 f = open(fn,"r")
132
133 try:
134 runT = json.loads(f.read())
135 except:
136 f.close()
137 v = XALT_Stack.pop()
138 carp("fn",v)
139 continue
140 f.close()
141
142 xalt.run_to_db(reverseMapT, runT)
143 v = XALT_Stack.pop()
144 carp("fn",v)
145
146 except Exception as e:
147 print(XALT_Stack.contents())
148 print(query.encode("ascii","ignore"))
149 print ("run_json_to_db(): Error:",e)
150 sys.exit (1)
151 return num
152
154 """
155 This generator walks the /etc/passwd file and returns the next
156 user and home directory. If XALT_USERS is set then it used that
157 instead. It is a colon separated list.
158
159 Super hack: if the colon separated list has a ";" in it then the
160 first part is the user the second is the home directory. This is
161 use in testing.
162 """
163
164 xaltUserA = os.environ.get("XALT_USERS")
165 if (xaltUserA):
166 for user in xaltUserA.split(":"):
167 idx = user.find(";")
168 if (idx != -1):
169 hdir = user[idx+1:]
170 user = user[:idx]
171 else:
172 hdir = os.path.expanduser("~" + user)
173 yield user, hdir
174
175 else:
176 for entry in getent.passwd():
177 yield entry.name, entry.dir
178
179
180
182 """
183 Walks the list of users via the passwd_generator and load the
184 link and run files.
185 """
186
187
188 sA = []
189 sA.append("CommandLine:")
190 for v in sys.argv:
191 sA.append('"'+v+'"')
192 XALT_Stack.push(" ".join(sA))
193
194 args = CmdLineOptions().execute()
195 xalt = XALTdb(dbConfigFn(args.dbname))
196
197 num = int(capture("getent passwd | wc -l"))
198 pbar = ProgressBar(maxVal=num)
199 icnt = 0
200
201 t1 = time.time()
202
203 rmapT = Rmap(args.rmapD).reverseMapT()
204
205 iuser = 0
206 lnkCnt = 0
207 runCnt = 0
208
209 for user, hdir in passwd_generator():
210 XALT_Stack.push("User: " + user)
211 xaltDir = os.path.join(hdir,".xalt.d")
212 if (os.path.isdir(xaltDir)):
213 iuser += 1
214 linkFnA = files_in_tree(xaltDir, "*/link.*.json")
215 XALT_Stack.push("link_json_to_db()")
216 lnkCnt += link_json_to_db(xalt, args.listFn, rmapT, linkFnA)
217 XALT_Stack.pop()
218 if (args.delete):
219 remove_files(linkFnA)
220
221
222 runFnA = files_in_tree(xaltDir, "*/run.*.json")
223 XALT_Stack.push("run_json_to_db()")
224 runCnt += run_json_to_db(xalt, args.listFn, rmapT, runFnA)
225 XALT_Stack.pop()
226 if (args.delete):
227 remove_files(runFnA)
228
229 icnt += 1
230 v = XALT_Stack.pop()
231 carp("User",v)
232 pbar.update(icnt)
233
234 xalt.connect().close()
235 pbar.fini()
236 t2 = time.time()
237 rt = t2 - t1
238 if (args.timer):
239 print("Time: ", time.strftime("%T", time.gmtime(rt)))
240
241 print("num users: ", iuser, ", num links: ", lnkCnt, ", num runs: ", runCnt)
242
243 if ( __name__ == '__main__'): main()
244