import pandas as pd
import xarray as xr
import numpy as np
import geopandas as gpd
import json
import warnings
import pyreadr
import sparse
import rasterio.features
from pathlib import Path
from tqdm import tqdm

warnings.filterwarnings('ignore')

# ==============================================================================
# 1. CONFIGURATION
# ==============================================================================

BASE_DIR = Path("./")  
CATCHMENT_INFO_PATH = BASE_DIR / 'Catchments_info_SSA' / 'Catchments.xlsx'

RDS_FOLDER = Path("D:/Global hydrological data") / 'GRDC&CAMELS'
GLDAS_FOLDER = Path("D:/Global hydrological data") / 'GLDAS&ERA5' / 'GLDAS'
ERA5_FOLDER = Path("D:/Global hydrological data") / 'GLDAS&ERA5' / 'ERA5'

SHP_FOLDER = BASE_DIR / 'Catchments_info_SSA'
CACHE_DIR = BASE_DIR / 'cache_SSA'

OUTPUT_FOLDER = BASE_DIR / 'Initial_output_SSA'

START_YEAR = 1980
END_YEAR = 2025

GLDAS_CONFIG = {
    "Rainf_f_tavg":             ("P",    "flux"),
    "Qs_acc":                   ("R_s",  "accum"),
    "Qsb_acc":                  ("R_sb", "accum"),
    "Evap_tavg":                ("ET",   "flux"),
    "Tair_f_inst":              ("T",    "state"),
    "PotEvap_tavg":             ("PET",  "energy_flux"),
    "SoilMoi0_10cm_inst":       ("SM1",  "depth"),
    "SoilMoi10_40cm_inst":      ("SM2",  "depth"),
    "SoilMoi40_100cm_inst":     ("SM3",  "depth"),
    "SoilMoi100_200cm_inst":    ("SM4",  "depth")
}

ERA5_CONFIG = {
    "tp":     ("P",     "rate_m"),
    "ro":     ("R",     "rate_m"),
    "e":      ("ET",    "rate_et"), 
    "pev":    ("PET",   "rate_pet"),
    "t2m":    ("T",     "state_k"),
    "swvl1":  ("SM1",   "pct"),
    "swvl2":  ("SM2",   "pct"),
    "swvl3":  ("SM3",   "pct"),
    "swvl4":  ("SM4",   "pct")
}

# ==============================================================================
# 2. CORE FUNCTIONS
# ==============================================================================
def load_all_shapefiles(shp_folder):
    """Loads shapefiles and normalizes the ID and geometry columns."""
    shp_files = list(Path(shp_folder).glob("*.shp"))
    if not shp_files:
        raise FileNotFoundError(f"No shapefiles found in {shp_folder}")

    gdfs = []
    print(f"  > Loading {len(shp_files)} shapefiles...")
    
    for shp in shp_files:
        try:
            gdf = gpd.read_file(shp)
            gdf.columns = [c.upper() for c in gdf.columns]
            
            candidates = ['REFID', 'FILENAME']
            id_col = next((c for c in candidates if c in gdf.columns), None)

            if id_col:
                if id_col == 'FILENAME':
                    gdf['Catchment_id'] = gdf[id_col].astype(str).str.replace(r'\.shp$', '', regex=True).str.strip()
                else:
                    gdf['Catchment_id'] = gdf[id_col].astype(str).str.strip()
                
                geom_col = next((c for c in gdf.columns if c == 'GEOMETRY'), None)
                if geom_col:
                    gdf = gdf.rename(columns={geom_col: 'geometry'})
                    gdf = gdf.set_geometry('geometry')
                
                gdf = gdf[['Catchment_id', 'geometry']]
                gdfs.append(gdf)
        except Exception as e:
            print(f"  [Warning] Could not read {shp.name}: {e}")

    if not gdfs:
        raise ValueError("No valid shapefiles loaded.")

    full_gdf = pd.concat(gdfs, ignore_index=True)
    full_gdf = gpd.GeoDataFrame(full_gdf, geometry='geometry', crs=gdfs[0].crs)
    full_gdf = full_gdf[full_gdf['Catchment_id'].notna() & (full_gdf['Catchment_id'] != 'nan')]
    
    print(f"  > Final merged shapefile count: {len(full_gdf)}")
    return full_gdf

def preprocess_gldas_ds(ds):
    """Standardizes coordinates (lat/lon) and unifies time dimensions."""
    if 'latitude' in ds.coords: ds = ds.rename({'latitude': 'lat'})
    if 'longitude' in ds.coords: ds = ds.rename({'longitude': 'lon'})
    
    # Fix 0-360 to -180/180
    if ds['lon'].max() > 180:
        ds.coords['lon'] = (ds.coords['lon'] + 180) % 360 - 180
        ds = ds.sortby(['lat', 'lon'])
    else:
        ds = ds.sortby(['lat', 'lon'])
        
    ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=True)
    ds.rio.write_crs("EPSG:4326", inplace=True)
    return ds

def get_weights(sample_file, gdf, cache_dir, prefix):
    weight_path = cache_dir / f"{prefix}_weights.npz"
    id_path = cache_dir / f"{prefix}_ids.json"

    if weight_path.exists() and id_path.exists():
        print(f"  > Loading spatial weights ({prefix})...")
        W = sparse.load_npz(str(weight_path))
        with open(id_path, 'r') as f: ids = json.load(f)
        return W, ids

    print(f"  > Computing new spatial weights for {prefix}...")
    ds = xr.open_dataset(sample_file)
    
    # Select Preprocessor based on prefix
    if prefix == "era5":
        ds = preprocess_era5_ds(ds)
    else:
        ds = preprocess_gldas_ds(ds)
        
    gdf = gdf.to_crs("EPSG:4326")
    
    shape = (ds.sizes['lat'], ds.sizes['lon'])
    transform = ds.rio.transform()
    lat_rad = np.deg2rad(ds.lat.values)
    area_weight = np.cos(lat_rad)[:, None] * np.ones((1, ds.sizes['lon']))
    area_weight_flat = area_weight.flatten()

    rows_idx, cols_idx, data_vals = [], [], []
    for i in tqdm(range(len(gdf)), desc=f"Mapping {prefix}"):
        geom = gdf.geometry.iloc[i]
        if geom is None or geom.is_empty: continue
        minx, miny, maxx, maxy = geom.bounds
        if (minx > ds.lon.max() or maxx < ds.lon.min() or miny > ds.lat.max() or maxy < ds.lat.min()): continue
        
        mask = rasterio.features.rasterize([(geom, 1)], out_shape=shape, transform=transform, fill=0, all_touched=True, dtype='float32').flatten()
        pix_indices = np.where(mask > 0)[0]
        
        if len(pix_indices) == 0:
            centroid = geom.centroid
            lat_idx = np.abs(ds.lat.values - centroid.y).argmin()
            lon_idx = np.abs(ds.lon.values - centroid.x).argmin()
            pix_id = lat_idx * ds.sizes['lon'] + lon_idx
            rows_idx.append(pix_id); cols_idx.append(i); data_vals.append(1.0)
        else:
            w = mask[pix_indices] * area_weight_flat[pix_indices]
            if w.sum() > 0:
                w /= w.sum()
                rows_idx.extend(pix_indices); cols_idx.extend([i] * len(pix_indices)); data_vals.extend(w)

    W = sparse.COO(np.array([rows_idx, cols_idx]), np.array(data_vals), shape=(np.prod(shape), len(gdf)))
    cache_dir.mkdir(parents=True, exist_ok=True)
    sparse.save_npz(str(weight_path), W)
    final_ids = gdf['Catchment_id'].tolist()
    with open(id_path, 'w') as f: json.dump(final_ids, f)
    return W, final_ids

def quantile_map_series(ref, bias, values):
    """
    Empirical Quantile Mapping:
    Maps 'values' (from biased distribution) onto the reference distribution.

    ref   : GLDAS-2.0 overlap values (reference)
    bias  : GLDAS-2.1 overlap values (biased)
    values: Full GLDAS-2.1 series to correct
    """

    ref = ref[np.isfinite(ref)]
    bias = bias[np.isfinite(bias)]

    if len(ref) < 20 or len(bias) < 20:
        return values  # insufficient data → no correction

    # Sort for empirical CDF construction
    ref_sorted = np.sort(ref)
    bias_sorted = np.sort(bias)

    # Build cumulative probabilities
    p_bias = np.linspace(0, 1, len(bias_sorted))
    p_ref  = np.linspace(0, 1, len(ref_sorted))

    # Step 1 — locate percentile of each value in biased CDF
    value_percentiles = np.interp(values, bias_sorted, p_bias, left=0.0, right=1.0)

    # Step 2 — map that percentile into reference distribution
    corrected = np.interp(value_percentiles, p_ref, ref_sorted)

    return corrected

def quantile_map_inverse(ref_v21, hist_v20, series_v20_to_convert):
    """
    Build mapping T21 = f(T20) using overlap distributions,
    then apply to historical v20 temperatures.
    """

    # remove NaNs
    mask = np.isfinite(ref_v21) & np.isfinite(hist_v20)
    ref_v21 = ref_v21[mask]
    hist_v20 = hist_v20[mask]

    # empirical CDF of v20 (predictor space)
    q = np.linspace(0, 1, len(hist_v20))
    v20_sorted = np.sort(hist_v20)

    # corresponding quantiles in v21 (target space)
    v21_sorted = np.sort(ref_v21)

    # interpolate: T20 → probability → T21
    prob = np.interp(series_v20_to_convert, v20_sorted, q,
                     left=0.0, right=1.0)

    reconstructed = np.interp(prob, q, v21_sorted)

    return reconstructed

def harmonize_versions(df, overlap_start="2000-01-01", overlap_end="2014-12-31"):

    df = df.copy()
    df['Date'] = pd.to_datetime(df['Date'])

    df20 = df[df["Version"] == "v20"].set_index("Date")
    df21 = df[df["Version"] == "v21"].set_index("Date")

    overlap20 = df20.loc[overlap_start:overlap_end]
    overlap21 = df21.loc[overlap_start:overlap_end]

    common_dates = overlap20.index.intersection(overlap21.index)

    diagnostics = {}

    corrected21 = df21.copy()

    for var in ["P", "R", "T", "ET", "PET", "SM1", "SM2", "SM3", "SM4"]:
        if var not in df20.columns or var not in df21.columns:
            continue

        ref = overlap20.loc[common_dates, var].values
        bias = overlap21.loc[common_dates, var].values

        mask_future = df21.index > overlap_end
        future_series = df21.loc[mask_future, var].values

        corrected = quantile_map_series(ref, bias, future_series)
        corrected21.loc[mask_future, var] = corrected

    merged = pd.concat([
        df20[df20.index <= overlap_end],
        corrected21[corrected21.index > overlap_end]
    ])

    merged = merged.sort_index().reset_index()
    merged = merged.rename(columns={"index": "Date"})

    return merged, diagnostics


def process_gldas_series(nc_files, W, mapped_ids):
    """
    Process GLDAS NetCDF files using the Robust NaN Handling strategy from InteData script.
    
    KEY IMPROVEMENT:
    - Uses np.nan_to_num() to force grid NaNs to 0.0 BEFORE matrix multiplication.
    - Prevents catchments near coastlines (NaN pixels) from returning NaN results.
    """
    
    results = {cid: [] for cid in mapped_ids}

    for f in tqdm(sorted(nc_files), desc="Processing GLDAS"):
        if ".021." in f.name:
            version = "v21"
        elif ".020." in f.name:
            version = "v20"
        else:
            raise ValueError(f"Cannot determine GLDAS version from {f.name}")

        try:
            with xr.open_dataset(f) as ds:
                ds = preprocess_gldas_ds(ds)

                # Time setup
                if 'time' in ds.coords:
                    current_dt = pd.to_datetime(ds.time.values[0])
                else:
                    # Fallback for some GLDAS versions lacking standard time dim
                    fname = f.name
                    date_part = next(p for p in fname.split('.') if p.startswith('A') and p[1:].isdigit())
                    current_dt = pd.to_datetime(date_part[1:], format='%Y%m%d')

                current_date = current_dt.normalize()
                days_in_month = current_dt.days_in_month
                seconds_in_month = days_in_month * 86400

                for var_nc, (var_out, var_type) in GLDAS_CONFIG.items():
                    if var_nc not in ds.data_vars: continue

                    raw_vals = ds[var_nc].values.flatten()

                    # 1. Identify valid pixels (not NaN)
                    valid_mask = np.isfinite(raw_vals)

                    # 2. Treat NaNs as 0.0 ONLY strictly for the matrix dot product math
                    vals_filled = np.nan_to_num(raw_vals, nan=0.0)
    
                    # 3. Calculate raw weighted sum
                    raw_weighted_sum = vals_filled @ W

                    # 4. Calculate the sum of the weights for ONLY the valid pixels
                    valid_weights_sum = valid_mask.astype(float) @ W
                    
                    # 5. Normalize the result (avoiding division by zero). 
                    # If a catchment has no valid pixels, it becomes NaN instead of 0.
                    with np.errstate(divide='ignore', invalid='ignore'):
                        catchment_vals = np.where(valid_weights_sum > 0, 
                                                raw_weighted_sum / valid_weights_sum, 
                                                np.nan)

                    # Unit Conversion
                    if var_type == "flux":
                        catchment_vals = catchment_vals * seconds_in_month # kg/m2/s -> mm/month
                    elif var_type == "accum":
                        catchment_vals = catchment_vals * 8 * days_in_month # mm/3h -> mm/month 
                    elif var_type == "state":
                        catchment_vals = catchment_vals - 273.15 # K -> C
                    elif var_type == "energy_flux":
                        catchment_vals = catchment_vals * 0.0353 * days_in_month # W/m2 -> mm/month
                    else:
                        catchment_vals = catchment_vals

                    # Store results
                    for i, cid in enumerate(mapped_ids):
                        val = catchment_vals[i]

                        results[cid].append({
                            "Date": current_date,
                            "Variable": var_out,
                            "Version": version,
                            "Value": val
                        })

        except Exception as e:
            print(f"[Warning] Error processing {f.name}: {e}")

    # Pivot and finalize
    final_dfs = {}
    for cid, data_list in results.items():
        if not data_list: continue

        df = pd.DataFrame(data_list)
        # Average if multiple entries per day/month
        df = df.groupby(["Date", "Variable", "Version"])["Value"].mean().reset_index()

        df_pivot = df.pivot_table(
            index=["Date", "Version"],
            columns="Variable",
            values="Value"
        ).reset_index()

        # Runoff Summation
        r_cols = [c for c in ["R_s", "R_sb"] if c in df_pivot.columns]
        if r_cols:
            df_pivot["R"] = df_pivot[r_cols].sum(axis=1)
        else:
            df_pivot["R"] = 0.0

        # Soil Moisture Summation
        # --- Soil Moisture (0–1 m total storage, GLDAS already in mm) ---
        sm_cols = [c for c in ["SM1", "SM2", "SM3"] if c in df_pivot.columns]

        if sm_cols:
            df_pivot["SM"] = df_pivot[sm_cols].sum(axis=1) / 1000.0  # Convert mm to m
        else:
            df_pivot["SM"] = np.nan

        cols_ordered = ["Date", "Version", "P", "R", "ET", "PET", "T", "SM1","SM2","SM3","SM4","SM"]
        final_cols = [c for c in cols_ordered if c in df_pivot.columns]
        final_dfs[cid] = df_pivot[final_cols].sort_values(["Date", "Version"])

    return final_dfs

def preprocess_era5_ds(ds):
    """
    Handles ERA5 specific quirks:
    1. Combines 'expver' (ERA5 vs ERA5T) if present.
    2. Renames 'valid_time' -> 'time'.
    3. Standardizes lat/lon.
    4. Ensures 'time' is a dimension (not just a scalar coordinate).
    """

    # 1. Standardize Time Coordinate Name
    # ERA5 often uses 'valid_time' instead of 'time'
    if 'valid_time' in ds.dims or 'valid_time' in ds.coords:
        ds = ds.rename({'valid_time': 'time'})
    
    # 2. Rename Spatial Coordinates
    rename_map = {}
    if 'latitude' in ds.coords: rename_map['latitude'] = 'lat'
    if 'longitude' in ds.coords: rename_map['longitude'] = 'lon'
    ds = ds.rename(rename_map)

    # 3. Standardize Lon (0-360 -> -180-180) and Sort
    if ds['lon'].max() > 180:
        ds.coords['lon'] = (ds.coords['lon'] + 180) % 360 - 180
    
    ds = ds.sortby(['lat', 'lon'])

    # 4. Ensure Time is a Dimension (Fix for scalar time error)
    if 'time' in ds.coords and 'time' not in ds.dims:
        ds = ds.expand_dims('time')
        
    ds.rio.set_spatial_dims(x_dim="lon", y_dim="lat", inplace=True)
    ds.rio.write_crs("EPSG:4326", inplace=True)
    
    return ds

def process_era5_series(nc_files, W, mapped_ids):
    """
    Process ERA5 files where data is Monthly Mean of Daily Rates.
    Formula: Total = Rate (m/day) * 1000 (mm/m) * Days_in_Month
    
    KEY FIX: Forces all dates to the 1st of the month to prevent 
    misalignment between Temperature (00:00) and Fluxes (06:00).
    """
    results = {cid: [] for cid in mapped_ids}

    for f in tqdm(sorted(nc_files), desc="Processing ERA5"):
        try:
            with xr.open_dataset(f) as ds:
                # 1. Standardize Coordinates
                if 'latitude' in ds.coords: ds = ds.rename({'latitude': 'lat'})
                if 'longitude' in ds.coords: ds = ds.rename({'longitude': 'lon'})
                if 'valid_time' in ds.coords: ds = ds.rename({'valid_time': 'time'})
                
                # Handle Expver (ERA5 vs ERA5T)
                if 'expver' in ds.dims:
                    try:
                        ds = ds.sel(expver=1).combine_first(ds.sel(expver=5))
                    except:
                        ds = ds.mean(dim='expver')
                
                # Ensure 0-360 to -180-180
                if ds['lon'].max() > 180:
                    ds.coords['lon'] = (ds.coords['lon'] + 180) % 360 - 180
                ds = ds.sortby(['lat', 'lon'])

                # 2. Iterate Variables
                for var_nc, (var_out, var_type) in ERA5_CONFIG.items():
                    if var_nc not in ds.data_vars: continue

                    data = ds[var_nc].values
                    
                    data = np.nan_to_num(data, nan=0.0)

                    # 3. Spatial Aggregation
                    if data.ndim == 3:
                        data_flat = data.reshape(data.shape[0], -1)
                    elif data.ndim == 2:
                        data_flat = data.reshape(1, -1)
                    
                    catchment_vals = data_flat @ W

                    # 4. Temporal Loop
                    times = pd.to_datetime(ds.time.values)
                    if not isinstance(times, (pd.DatetimeIndex, np.ndarray, list)):
                         times = [times]

                    for t_idx, current_date in enumerate(times):
                        # --- FIX: NORMALIZE DATE ---
                        # Force date to YYYY-MM-01 00:00:00 regardless of input hour
                        norm_date = pd.Timestamp(current_date).normalize().replace(day=1)
                        
                        days_in_month = norm_date.days_in_month
                        raw_val = catchment_vals[t_idx, :]

                        # --- UNIT & TIME INTEGRATION ---
                        if var_type == "rate_m":
                            # m/day -> mm/month
                            final_val = raw_val * 1000.0 * days_in_month
                        elif var_type == "rate_pet":
                            # negative m/day -> positive mm/month
                            final_val = raw_val * -1000.0 * days_in_month
                        elif var_type == "rate_et":
                            # negative m/day -> positive mm/month
                            final_val = raw_val * -1000.0 * days_in_month * 1.112
                        elif var_type == "state_k":
                            # Kelvin -> Celsius
                            final_val = raw_val - 273.15
                        else:
                            final_val = raw_val

                        # Append to results with NORMALIZED date
                        for i, cid in enumerate(mapped_ids):
                            results[cid].append({
                                "Date": norm_date,
                                "Variable": var_out,
                                "Value": final_val[i]
                            })

        except Exception as e:
            print(f"  [Error] {f.name}: {e}")

    # Finalize DataFrame
    final_dfs = {}
    for cid, data_list in results.items():
        if not data_list: continue
        
        df = pd.DataFrame(data_list)
        df = df.groupby(["Date", "Variable"])["Value"].mean().reset_index()
        
        df_pivot = df.pivot_table(index="Date", columns="Variable", values="Value").reset_index()

        for c in ["P","R","ET","PET","T","SM1","SM2","SM3","SM4","SM"]:
            if c not in df_pivot.columns:
                df_pivot[c] = np.nan

        # --- Soil Moisture (0–1 m storage in mm) ---
        if all(col in df_pivot.columns for col in ["SM1","SM2","SM3"]):

            # ERA5 soil moisture is volumetric (m3/m3)
            df_pivot["SM"] = (df_pivot["SM1"] * 0.07 + df_pivot["SM2"] * 0.21 + df_pivot["SM3"] * 0.72)

        else:
            df_pivot["SM"] = np.nan
            
        # --- Apply non-negativity constraint ---
        for col in ["ET", "PET"]:
            if col in df_pivot.columns:
                df_pivot[col] = df_pivot[col].clip(lower=0.0)
        # -----------------------------------------------------

        final_dfs[cid] = df_pivot.sort_values("Date")

    return final_dfs

def process_rds_file(path, cid):

    try:
        result = pyreadr.read_r(path)
        df = result[None]

        if 'Date' not in df.columns and 'date' in df.columns:
            df = df.rename(columns={'date': 'Date'})

        df['Date'] = pd.to_datetime(df['Date'])
        df = df.set_index('Date').sort_index()

        mappings = {'MSWEP': 'P', 'runoff_mm': 'R', 'NASA_Tavg': 'T'}
        df = df.rename(columns=mappings)

        # ---- Derive T if needed ----
        if 'T' not in df.columns:
            tmax = next((c for c in df.columns if 'tmax' in c.lower()), None)
            tmin = next((c for c in df.columns if 'tmin' in c.lower()), None)
            if tmax and tmin:
                df['T'] = (df[tmax] + df[tmin]) / 2

        # ==========================================================
        # MONTHLY AGGREGATION
        # ==========================================================
        monthly_data = {}
        full_monthly_idx = pd.date_range(df.index.min(), df.index.max(), freq='MS')

        agg_map = {'P': 'sum', 'R': 'sum', 'T': 'mean'}

        for col, method in agg_map.items():
            if col in df.columns:
                monthly_data[col] = df[col].resample('MS').agg(method).reindex(full_monthly_idx)

        df_monthly = pd.DataFrame(monthly_data)
        df_monthly.index.name = 'Date'
        df_monthly = df_monthly.reset_index()

        return df_monthly

    except Exception as e:
        print(f"[Error] RDS Process {cid}: {e}")
        return pd.DataFrame()

    
def filter_gldas_files(file_list, avg_start, avg_end):
    """
    Filters GLDAS files with an averaging logic for the overlap period.
    
    Logic:
    1. Date < avg_start: PREFER Version 2.0 (020).
    2. avg_start <= Date <= avg_end: USE BOTH (020 and 021) -> allows averaging downstream.
    3. Date > avg_end: PREFER Version 2.1 (021).
    """
    files_by_date = {}
    
    # 1. Catalog all files by Date and Version
    for f in file_list:
        try:
            parts = f.name.split('.')
            date_part = next(p for p in parts if p.startswith('A') and p[1:5].isdigit())
            version_part = next(p for p in parts if p in ['020', '021'])
            
            # Key: 'YYYY-MM'
            dt_key = pd.to_datetime(date_part[1:7], format='%Y%m').strftime('%Y-%m')
            
            if dt_key not in files_by_date:
                files_by_date[dt_key] = {}
            files_by_date[dt_key][version_part] = f
            
        except Exception:
            print(f"  [Skipping] Could not parse filename: {f.name}")

    # 2. Select files based on the window
    final_files = []
    
    # Convert bounds to YYYY-MM for string comparison
    start_ym = pd.to_datetime(avg_start).strftime('%Y-%m')
    end_ym = pd.to_datetime(avg_end).strftime('%Y-%m')
    
    sorted_dates = sorted(files_by_date.keys())
    
    for dt in sorted_dates:
        versions = files_by_date[dt]
        has_v20 = '020' in versions
        has_v21 = '021' in versions
        
        # CASE A: Inside the Averaging Window (2000-2014)
        if start_ym <= dt <= end_ym:
            # Append BOTH if they exist
            if has_v20: final_files.append(versions['020'])
            if has_v21: final_files.append(versions['021'])
            
        # CASE B: Historic Period (before 2000) -> Prefer V2.0
        elif dt < start_ym:
            if has_v20:
                final_files.append(versions['020'])
            elif has_v21:
                final_files.append(versions['021'])
                
        # CASE C: Modern Period (after 2015) -> Prefer V2.1
        else:
            if has_v21:
                final_files.append(versions['021'])
            elif has_v20:
                final_files.append(versions['020'])

    print(f"  > Filtered GLDAS files: {len(file_list)} input -> {len(final_files)} files to process.")
    print(f"    (Averaging V2.0 and V2.1 between {avg_start} and {avg_end})")
    
    return sorted(final_files)

# ==============================================================================
# 3. MAIN EXECUTION
# ==============================================================================

def main():
    OUTPUT_FOLDER.mkdir(exist_ok=True)
    CACHE_DIR.mkdir(exist_ok=True)

    print("--- 1. Loading Metadata & Shapefiles ---")
    meta_df = pd.read_excel(CATCHMENT_INFO_PATH)
    all_shps = load_all_shapefiles(SHP_FOLDER)
    
    target_ids = meta_df['Catchment_id'].astype(str).unique()
    rel_gdf = all_shps[all_shps['Catchment_id'].isin(target_ids)]
    print(f"  > Processing {len(rel_gdf)} catchments.")

    # --- ERA5 PROCESSING ---
    era5_results = {}
    if "ERA5" in meta_df['Dataset'].values:
        print("\n--- 2a. Processing ERA5 Data ---")
        era5_files = sorted(list(ERA5_FOLDER.glob("*.nc")))
        
        if not rel_gdf.empty and era5_files:
            # Prefix 'era5'
            W_era, mapped_ids_era = get_weights(era5_files[0], rel_gdf, CACHE_DIR, prefix="era5")
            if W_era is not None:
                era5_results = process_era5_series(era5_files, W_era, mapped_ids_era)
        else:
            print("  [Warning] No ERA5 files or Shapefiles found.")

    # --- GLDAS PROCESSING ---
    gldas_results = {}
    if "GLDAS" in meta_df['Dataset'].values:
        print("\n--- 2b. Processing GLDAS Data ---")
        nc_files = sorted(list(GLDAS_FOLDER.glob("*.nc")) + list(GLDAS_FOLDER.glob("*.nc4")))
        if not rel_gdf.empty and nc_files:
            # Prefix 'gldas'
            W, mapped_ids = get_weights(nc_files[0], rel_gdf, CACHE_DIR, prefix="gldas") 
            if W is not None:
                gldas_results = process_gldas_series(nc_files, W, mapped_ids)

    print(f"\n--- 3. Finalizing & Saving ---")
    for idx, row in tqdm(meta_df.iterrows(), total=len(meta_df), desc="Saving CSVs"):
        cid = str(row['Catchment_id'])
        dataset = row['Dataset']
        
        # 1. Initialize cleanly to prevent cross-catchment data leakage
        df_raw = pd.DataFrame()
        
        # 2. Process and assign both raw and corrected states per dataset
        if dataset == 'GLDAS':
            df_unharmonized = gldas_results.get(cid, pd.DataFrame())

            if not df_unharmonized.empty:
                # Apply the empirical quantile mapping harmonization
                df_raw, diagnostics = harmonize_versions(
                    gldas_results.get(cid, pd.DataFrame()), 
                    overlap_start="2000-01-01", 
                    overlap_end="2014-12-31"
                )

        elif dataset == 'ERA5':
            df_raw = era5_results.get(cid, pd.DataFrame())

        elif dataset in ['CAMELS', 'GRDC']:
            candidates = list(RDS_FOLDER.glob(f"*{cid}*.rds"))
            if candidates:
                rds_path = candidates[0]
                df_raw = process_rds_file(rds_path, cid)

        # 3. Apply strict temporal filtering to BOTH dataframes
        if not df_raw.empty and 'Date' in df_raw.columns:
            df_raw = df_raw[(df_raw['Date'].dt.year >= START_YEAR) & (df_raw['Date'].dt.year <= END_YEAR)]

        df_raw.to_csv(OUTPUT_FOLDER / f"{dataset}_{cid}.csv", index=False)

    print("\nProcessing Complete.")

if __name__ == "__main__":
    main()