Coverage for wrainfo/reader.py: 94%

1"""Reader module.""" 

2 

3# WRaINfo, Is a software to process FURUNO weather radar data. 

4# 

5# Copyright (c) 2022, FernLab (GFZ Potsdam, fernlab@gfz-potsdam.de) 

6# 

7# This software was developed within the context of the RaINfo ("Potential use of 

8# high resolution weather data in agriculture") project of FernLab funded by 

9# the Impulse and Networking Fund of the Helmholtz Association. 

10# 

11# Licensed under the Apache License, Version 2.0 (the "License"); 

12# you may not use this file except in compliance with the License. 

13# 

14# You may obtain a copy of the License at 

15# http://www.apache.org/licenses/LICENSE-2.0 

16# 

17# Unless required by applicable law or agreed to in writing, software 

18# distributed under the License is distributed on an "AS IS" BASIS, 

19# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

20# See the License for the specific language governing permissions and 

21# limitations under the License. 

22 

23# imports 

24# ------- 

25 

26import datetime as dt 

27import os 

28import glob 

29import re 

30import xarray as xr 

31import numpy as np 

32import pickle 

33import json 

34from pathlib import Path 

35 

36 

37# read configuration file 

38# ----------------------- 

39 

40def read_config_file(path=".../example_settings_path_dependencies_wr_furuno.json", selection=None): 

41 """Read wr_furuno config file in json format. 

42 

43 Parameter 

44 --------- 

45 path : str 

46 Path to config file. 

47 selection : str 

48 select setting parameter 

49 

50 Return 

51 ------ 

52 : dict 

53 configured directories in the configuration file. 

54 """ 

55 path = path.replace("~", str(Path.home())) 

56 

57 if not Path(path).is_file(): 

58 raise Exception("wrainfo.reader.read_config_file(): Config file'", path, "' not found.") 

59 

60 file = open(path) 

61 conf = json.load(file) 

62 file.close() 

63 

64 if selection is not None: 

65 if selection in conf.keys(): 

66 return conf[selection] 

67 else: 

68 raise Exception("wrainfo.reader.read_config_file(): Setting parameter'", 

69 selection, 

70 "' not in settings file '", 

71 path, 

72 "'.") 

73 else: 

74 return conf 

75 

76 

77# Functions to read the Furuno data 

78# ---------------------------------- 

79 

80def get_furuno_path(path, start_time=dt.datetime.today()): 

81 """Create path of Furuno radar data files. 

82 

83 Parameters 

84 ---------- 

85 path : str 

86 Path to configuration file 

87 start_time : datetime.datetime 

88 datetime object to select correct folder 

89 

90 Returns 

91 ------- 

92 radar_path : str 

93 Path to radar data 

94 """ 

95 raw_path = read_config_file(path=path, selection="raw_data_directory") 

96 subfolder_struct_raw_path = read_config_file(path=path, selection="subfolder_structure_raw_data") 

97 

98 radar_path = os.path.join(raw_path, subfolder_struct_raw_path) 

99 return radar_path.format(start_time.year, start_time.month, start_time.day) 

100 

101 

102def get_file_date_regex(filename): 

103 """Get regex from filename.""" 

104 # regex for ""%Y-%m-%d--%H:%M:%S" 

105 reg0 = r"\d{4}.\d{2}.\d{2}..\d{2}.\d{2}.\d{2}" 

106 # regex for "%Y%m%d%H%M%S" 

107 reg1 = r"\d{14}" 

108 # regex for 20220216_085000 

109 reg2 = r"\d{4}\d{2}\d{2}.\d{2}\d{2}\d{2}" 

110 for reg in [reg0, reg1, reg2]: 

111 match = re.search(reg, os.path.basename(filename)) 

112 if match is not None: 

113 return reg 

114 return None 

115 

116 

117def get_datetime_from_filename(filename, regex): 

118 """Get datetime from filename.""" 

119 fmt = "%Y%m%d%H%M%S" 

120 match = re.search(regex, os.path.basename(filename)) 

121 match = "".join(re.findall(r"[0-9]+", match.group())) 

122 return dt.datetime.strptime(match, fmt) 

123 

124 

125def load_error_flist(path): 

126 """Load error file list. 

127 

128 Parameter 

129 --------- 

130 path : str 

131 Path to configuration file 

132 

133 Returns 

134 ------ 

135 : list 

136 list of error files. 

137 """ 

138 path_error_flist = read_config_file(path=path, selection="error_flist_directory") 

139 

140 with open(path_error_flist, 'rb') as fp: 

141 error_files = pickle.load(fp) 

142 

143 return error_files 

144 

145 

146def create_filelist(starttime, endtime, path, pattern="_000.scnx.gz"): 

147 """Create filelist from path_glob and filename dates. 

148 

149 Parameters 

150 ---------- 

151 starttime : dt.datetime 

152 start time 

153 endtime : dt.datetime 

154 end time 

155 path : str 

156 Path to configuration file 

157 pattern : str 

158 extension of the scnx/netcdf file 

159 (scnx file: elevation angle 0.5° = "_000.scnx.gz") 

160 (netcdf file: = ".nc" , elevation angle is included in the filename) 

161 

162 Returns 

163 ------- 

164 : list 

165 list of files 

166 """ 

167 flist = [] 

168 flist_scnx = [] 

169 flist_h5 = [] 

170 flist_nc = [] 

171 datetime_list = [] 

172 date = starttime 

173 

174 error_flist = load_error_flist(path=path) 

175 

176 while date <= endtime: 

177 

178 raw_path = get_furuno_path(path=path, start_time=date) 

179 

180 file_names = sorted(glob.glob(os.path.join(raw_path, "*"))) 

181 

182 # fixed empty list of file_names 

183 if len(file_names) > 0: 

184 regex = get_file_date_regex(file_names[0]) 

185 

186 for fname in file_names: 

187 time = get_datetime_from_filename(fname, regex) 

188 if time >= date: 

189 if time < endtime: 

190 if fname.endswith(pattern): 

191 if fname not in error_flist: 

192 flist_scnx.append(fname) 

193 if fname.endswith(".h5"): 

194 if fname not in error_flist: 

195 flist_h5.append(fname) 

196 if fname.endswith(".nc"): 

197 if fname not in error_flist: 

198 flist_nc.append(fname) 

199 

200 if len(flist_nc) > 0: 

201 for file in flist_nc: 

202 flist.append(file) 

203 

204 if len(flist_scnx) == 0: 

205 if len(flist_h5) > 0: 

206 for file in flist_h5: 

207 flist.append(file) 

208 

209 if len(flist_scnx) > 0: 

210 for file in flist_scnx: 

211 filetime_scnx = get_datetime_from_filename(file, regex) 

212 datetime_list.append(filetime_scnx) 

213 flist.append(file) 

214 if len(flist_h5) > 0: 

215 for file in flist_h5: 

216 filetime_h5 = get_datetime_from_filename(file, regex) 

217 if filetime_h5 not in datetime_list: 

218 flist.append(file) 

219 

220 date = date + dt.timedelta(1) 

221 

222 flist = sorted(set(flist)) 

223 

224 return flist 

225 

226 

227# read single file of Furuno data 

228# (important for create the clutter map) 

229# --------------------------------------- 

230 

231def read_single_file(file, grp="dataset1"): 

232 """Read a single file, reindexes and returns a dataset object. 

233 

234 Parameters 

235 ---------- 

236 file : str 

237 path to file as string 

238 grp : str 

239 hdf5 group (elevation angle) 

240 

241 Returns 

242 ------- 

243 : xarray Dataset 

244 xarray Dataset with variables. 

245 """ 

246 extension = os.path.splitext(file)[1] 

247 

248 if extension == ".gz": 

249 # the elevation angle the elevation angle is already selected in the flist! 

250 tmp = xr.open_mfdataset((file,), engine="furuno", group=1, 

251 concat_dim="time", combine="nested", 

252 backend_kwargs=dict(reindex_angle=1.0)) 

253 

254 else: 

255 tmp = xr.open_mfdataset((file,), engine="odim", group=grp, 

256 concat_dim="time", combine="nested", 

257 backend_kwargs=dict(reindex_angle=1.0)) 

258 

259 # TODO add case sensitive decision on these parameters 

260 re_index_parameters = np.arange(0.25, 360, 0.5) 

261 

262 ri = tmp.reindex(azimuth=re_index_parameters, method="nearest") 

263 return ri 

264 

265 

266# read processed cluttermap from Furuno 

267# ------------------------------------- 

268 

269def get_cmap(starttime, 

270 endtime, 

271 path, 

272 elev="0.5", 

273 timestr="%Y%m%d"): 

274 """Read processed cluttermap from Furuno. 

275 

276 Parameters 

277 --------- 

278 starttime : datetime.datetime 

279 datetime - object to select correct files for list 

280 endtime : datetime.datetime 

281 datetime - object to select correct files for list 

282 path : str 

283 Path to configuration file 

284 elev : str 

285 selected elevation angle of cmap 

286 

287 Returns 

288 ------- 

289 : xarray.Dataset 

290 clutter map as xarray.Dataset. 

291 """ 

292 # read settings from configuration file 

293 clutter_dir = read_config_file(path=path, selection="monthly_clutter_directory") 

294 subfolderstruct = read_config_file(path=path, selection="subfolder_structure_clutter_directory") 

295 

296 # create file structure 

297 file_struct = os.path.join(f"{clutter_dir}", f"{subfolderstruct}", "*") 

298 

299 path = file_struct.format(year=starttime.year) 

300 

301 # list all cmaps in the folder 

302 cmaps = sorted(glob.glob(path)) 

303 

304 selected_map = None 

305 last_date = dt.datetime(1900, 1, 1) 

306 

307 for cmap in cmaps: 

308 # choose the stop time from filename 

309 filetime_end = dt.datetime.strptime((os.path.basename(cmap)[19:27]), timestr) 

310 elev_cmap = os.path.basename(cmap)[33:36] 

311 

312 if filetime_end > (endtime - dt.timedelta(45)): 

313 if filetime_end > last_date: 

314 if elev_cmap == elev: # noqa E712 

315 selected_map = cmap 

316 last_date = filetime_end 

317 

318 if selected_map is None: 

319 print("wrainfo.reader.get_cmap(): No cmap found for these dates.") 

320 

321 # read cmap 

322 cmap = xr.open_dataset(selected_map, engine="h5netcdf") 

323 

324 return cmap