Coverage for wrainfo/compression.py: 84%

1"""Compression module.""" 

2 

3# WRaINfo, Is a software to process FURUNO weather radar data. 

4# 

5# Copyright (c) 2022, FernLab (GFZ Potsdam, fernlab@gfz-potsdam.de) 

6# 

7# This software was developed within the context of the RaINfo ("Potential use of 

8# high resolution weather data in agriculture") project of FernLab funded by 

9# the Impulse and Networking Fund of the Helmholtz Association. 

10# 

11# Licensed under the Apache License, Version 2.0 (the "License"); 

12# you may not use this file except in compliance with the License. 

13# 

14# You may obtain a copy of the License at 

15# http://www.apache.org/licenses/LICENSE-2.0 

16# 

17# Unless required by applicable law or agreed to in writing, software 

18# distributed under the License is distributed on an "AS IS" BASIS, 

19# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 

20# See the License for the specific language governing permissions and 

21# limitations under the License. 

22 

23# imports 

24# ------- 

25 

26import datetime as dt 

27import pathlib 

28import tarfile 

29import os 

30import glob 

31from wrainfo.reader import read_config_file 

32from wrainfo.reader import get_furuno_path 

33 

34 

35# create tar.gz files 

36# ------------------- 

37 

38def compress_furuno_raw_data(start_time, 

39 end_time, 

40 path, 

41 delta=dt.timedelta(days=1), 

42 delete_files=False): 

43 """Create daily tar.gz files of FURUNO raw data. 

44 

45 Parameters 

46 ---------- 

47 start_time : datetime.datetime 

48 end_time : datetime.datetime 

49 path : str 

50 Path to configuration file 

51 delta: datetime.delta 

52 timestamp in which sequence the data will be processed 

53 delete_files : bool 

54 if True than files which are packed into tar.gz file were delete 

55 

56 Returns 

57 ------- 

58 : files 

59 daily tar.gz files in output directory. 

60 """ 

61 date = start_time 

62 stop_time = end_time 

63 file_names = [] 

64 

65 outputdir = read_config_file(path=path, selection="output_directory_tar_gz_files") 

66 radar_location_identifier = read_config_file(path=path, selection="radar_location_identifier") 

67 

68 while date < stop_time: 

69 

70 path_day = get_furuno_path(path=path, start_time=date) 

71 

72 file_names = sorted(glob.glob(os.path.join(path_day, "*"))) 

73 

74 if len(file_names) == 0: 

75 

76 date += delta 

77 

78 else: 

79 file = file_names[1] 

80 basename = os.path.basename(file) 

81 year = basename[5:9] 

82 month = basename[9:11] 

83 day = basename[11:13] 

84 

85 outfilename = f"{radar_location_identifier}_{year}{month}{day}.tar.gz" 

86 

87 path1 = outputdir + "/" 

88 

89 # output path will create if not exists 

90 if not os.path.exists(path1): 

91 os.makedirs(path1, exist_ok=True) 

92 # overwrite/remove if exist 

93 f = pathlib.Path(outfilename) 

94 f.unlink(missing_ok=True) 

95 

96 # absolute output path 

97 outfilename = os.path.join(path1, outfilename) 

98 if not os.path.exists(outputdir): 

99 os.makedirs(outputdir) 

100 # overwrite/remove if exist 

101 f = pathlib.Path(outfilename) 

102 f.unlink(missing_ok=True) 

103 

104 with tarfile.open(outfilename, "w:gz") as tar_handle: 

105 for file in file_names: 

106 tar_handle.add(file, arcname=file[35:]) 

107 

108 if delete_files is True: 

109 

110 for file in file_names: 

111 os.remove(file) 

112 print("delete file:", file) 

113 

114 date += delta 

115 

116 return True 

117 

118 

119# extract tar.gz files 

120# --------------------- 

121 

122def extract_files(path, 

123 out_dir): 

124 """Extract tar.gz - files. 

125 

126 Parameters 

127 --------- 

128 path : path to configuration file 

129 out_dir : dir where the extracted tar.gz files are saved 

130 

131 Returns 

132 ------- 

133 : files 

134 Extracted tar.gz files in output directory. 

135 """ 

136 dir_compressed_files = read_config_file(path=path, selection="output_directory_tar_gz_files") 

137 

138 flist = sorted(glob.glob(os.path.join(dir_compressed_files, "*"))) 

139 

140 for gz_file in flist: 

141 file = tarfile.open(gz_file) 

142 file.extractall(out_dir) 

143 file.close() 

144 

145 return True