""" temporal_trend_normalized.py Two normalized views of the quarterly temporal trend figure (temporal_trend_analysis_quarterly.py), built on the exact same category selection (same Poisson interaction test, FDR correction, top-3 by effect size), to make the differing temporal *shapes* comparable despite the categories' very different absolute sizes. Option A - "own-total" normalization (temporal_trend_normalized_owntotal): Each quarter's count divided by that series' own total N (Overall corpus, or each category's total papers). All 4 lines (Overall + 3 categories) end up on a comparable 0-100% scale, showing each series' own adoption shape - useful for comparing timing (early vs. late growth) independent of scale, but it discards the "vs. the rest of the corpus" framing the categories were actually selected on: a category with the same shape as the overall corpus looks identical to it here even though that's not how these 3 were chosen. Option B - "share of corpus" normalization (temporal_trend_normalized_shareofcorpus): Each category's quarterly count divided by the overall corpus's count in that same quarter. This is the direct visual counterpart of the Poisson interaction test itself (which tests exactly this ratio's trend). "Overall corpus" is dropped as a line here (it would be a trivial flat 100% of itself); the 3 selected categories' share-of- corpus trends are shown instead. The partial-quarter (2025 Q2) extrapolation factor cancels out of this ratio algebraically (it's applied equally to numerator and denominator), so no extrapolation adjustment is needed for this option - the raw ratio is used directly, though it's still flagged as noisier since it's based on only 1 month of data. Output ------ - temporal_trend_normalized_owntotal.png / .pdf - temporal_trend_normalized_shareofcorpus.png / .pdf - temporal_trend_normalized.xlsx (both series' underlying values) """ from __future__ import annotations import argparse from pathlib import Path import matplotlib.pyplot as plt import numpy as np import pandas as pd from temporal_trend_analysis_quarterly import ( FIRST_QUARTER, LAST_COMPLETE_QUARTER, PARTIAL_QUARTER, MIN_CATEGORY_N, N_TOP_CATEGORIES, ROLLING_WINDOW_QUARTERS, load_merged, get_candidate_categories, run_all_tests, select_top_categories, quarterly_counts, build_series_with_extrapolation, rolling_mean_centered, all_quarters, quarter_label, ) FIGURE_SIZE = (13, 6.5) DPI = 300 # ============================================================================= # CORE LOGIC # ============================================================================= def make_owntotal_figure(overall_series: pd.Series, category_series: dict[str, pd.Series], overall_total: int, category_totals: dict[str, int], complete_quarters: list[tuple[int, int]]) -> plt.Figure: fig, ax = plt.subplots(figsize=FIGURE_SIZE) x_positions = {q: i for i, q in enumerate(overall_series.index)} colors = ["#4C72B0", "#DD8452", "#55A868", "#C44E52"] last_complete_x = x_positions[complete_quarters[-1]] all_series = {"Overall corpus": (overall_series, overall_total), **{ label: (series, category_totals[label]) for label, series in category_series.items() }} for (label, (series, total)), color in zip(all_series.items(), colors): pct = series / total * 100 xs = np.array([x_positions[q] for q in pct.index]) ys = pct.values raw_mask = xs <= last_complete_x ax.plot(xs[raw_mask], ys[raw_mask], marker="o", markersize=3, linewidth=1.0, color=color, alpha=0.5, label=f"{label} (raw)") tail_mask = xs >= last_complete_x ax.plot(xs[tail_mask], ys[tail_mask], marker="o", markersize=5, markerfacecolor="white", markeredgecolor=color, markeredgewidth=1.4, linewidth=1.0, linestyle="--", color=color) smoothed = rolling_mean_centered(pct, ROLLING_WINDOW_QUARTERS) sm_xs = np.array([x_positions[q] for q in smoothed.index]) sm_confirmed = sm_xs <= last_complete_x sm_tail = sm_xs >= last_complete_x ax.plot(sm_xs[sm_confirmed], smoothed.values[sm_confirmed], linewidth=2.2, color=color, label=f"{label} (4-qtr rolling mean)") ax.plot(sm_xs[sm_tail], smoothed.values[sm_tail], linewidth=2.2, linestyle="--", color=color) year_starts = [q for q in overall_series.index if q[1] == 1] ax.set_xticks([x_positions[q] for q in year_starts]) ax.set_xticklabels([str(q[0]) for q in year_starts], fontsize=10) ax.set_xlabel("Publication quarter (year labels mark Q1; all 4 quarters plotted)", fontsize=11) ax.set_ylabel("% of that series' own total papers", fontsize=11) ax.grid(axis="y", linestyle="--", alpha=0.4, zorder=0) ax.legend(fontsize=7.5, loc="upper left", ncol=1) fig.tight_layout() return fig def plot_shareofcorpus_panel(ax: plt.Axes, overall_raw: pd.Series, category_raw: dict[str, pd.Series], complete_quarters: list[tuple[int, int]], xlabel_fontsize: float = 11, ylabel_fontsize: float = 11, tick_fontsize: float = 10, legend_fontsize: float = 8, rotate_xticks: bool = False, draw_legend: bool = True, legend_loc: str = "upper right", legend_bbox_to_anchor=None, legend_ncol: int = 1) -> None: """Draw the share-of-corpus trend lines onto an existing Axes. Factored out of make_shareofcorpus_figure so the same panel can be embedded in other multi-panel figures (e.g. as panel F of facet_distributions_combined_A_E).""" x_positions = {q: i for i, q in enumerate(overall_raw.index)} colors = ["#DD8452", "#55A868", "#C44E52"] last_complete_x = x_positions[complete_quarters[-1]] for (label, series), color in zip(category_raw.items(), colors): # Extrapolation factor cancels in the ratio; use raw counts directly. share = (series / overall_raw).replace([np.inf, -np.inf], np.nan) * 100 xs = np.array([x_positions[q] for q in share.index]) ys = share.values raw_mask = xs <= last_complete_x ax.plot(xs[raw_mask], ys[raw_mask], marker="o", markersize=3, linewidth=1.0, color=color, alpha=0.5, label=f"{label} (raw)") tail_mask = xs >= last_complete_x ax.plot(xs[tail_mask], ys[tail_mask], marker="o", markersize=5, markerfacecolor="white", markeredgecolor=color, markeredgewidth=1.4, linewidth=1.0, linestyle="--", color=color) smoothed = rolling_mean_centered(share, ROLLING_WINDOW_QUARTERS) sm_xs = np.array([x_positions[q] for q in smoothed.index]) sm_confirmed = sm_xs <= last_complete_x sm_tail = sm_xs >= last_complete_x ax.plot(sm_xs[sm_confirmed], smoothed.values[sm_confirmed], linewidth=2.2, color=color, label=f"{label} (4-qtr rolling mean)") ax.plot(sm_xs[sm_tail], smoothed.values[sm_tail], linewidth=2.2, linestyle="--", color=color) year_starts = [q for q in overall_raw.index if q[1] == 1] ax.set_xticks([x_positions[q] for q in year_starts]) if rotate_xticks: ax.set_xticklabels([str(q[0]) for q in year_starts], fontsize=tick_fontsize, rotation=45, ha="right") else: ax.set_xticklabels([str(q[0]) for q in year_starts], fontsize=tick_fontsize) ax.set_xlabel("Publication quarter (year labels mark Q1; all 4 quarters plotted)", fontsize=xlabel_fontsize) ax.set_ylabel("Category's share of that quarter's total corpus (%)", fontsize=ylabel_fontsize) ax.grid(axis="y", linestyle="--", alpha=0.4, zorder=0) if draw_legend: ax.legend(fontsize=legend_fontsize, loc=legend_loc, bbox_to_anchor=legend_bbox_to_anchor, ncol=legend_ncol) def make_shareofcorpus_figure(overall_raw: pd.Series, category_raw: dict[str, pd.Series], complete_quarters: list[tuple[int, int]]) -> plt.Figure: fig, ax = plt.subplots(figsize=FIGURE_SIZE) plot_shareofcorpus_panel(ax, overall_raw, category_raw, complete_quarters) fig.tight_layout() return fig def load_shareofcorpus_panel_data(xlsx_path: Path): """Rebuild the (overall_raw, category_raw, complete_quarters) inputs plot_shareofcorpus_panel needs, from the values already saved in this script's own output workbook (temporal_trend_normalized.xlsx, "Share of corpus pct" sheet), rather than recomputing from the raw PubMed source file. Used by figures that embed this panel alongside others (e.g. facet_distributions_combined_A_E panel F, temporal_summary_figure.py panel B) as a presentation-layer step over an already-produced figure, not a new analysis. Setting overall_raw to a constant 1.0 series and category_raw to (saved_pct / 100) reproduces the exact same plotted values as the original (category_count / overall_count) * 100 computation, since only the final ratio - not the underlying counts - is needed by the panel.""" complete_quarters = all_quarters(FIRST_QUARTER, LAST_COMPLETE_QUARTER) all_q = complete_quarters + [PARTIAL_QUARTER] share_df = pd.read_excel(xlsx_path, sheet_name="Share of corpus pct") assert len(share_df) == len(all_q), (len(share_df), len(all_q)) idx = pd.MultiIndex.from_tuples(all_q) overall_raw = pd.Series(1.0, index=idx) category_raw = {} for col in share_df.columns: if col == "Quarter": continue label = col.replace(", share of corpus (%)", "") category_raw[label] = pd.Series(share_df[col].values / 100.0, index=idx) return overall_raw, category_raw, complete_quarters def main() -> None: parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--input-dir", type=Path, default=Path(".")) parser.add_argument("--output-dir", type=Path, default=Path(".")) args = parser.parse_args() merged = load_merged(args.input_dir) complete_quarters = all_quarters(FIRST_QUARTER, LAST_COMPLETE_QUARTER) all_q = complete_quarters + [PARTIAL_QUARTER] candidates = get_candidate_categories(merged, MIN_CATEGORY_N) results_df = run_all_tests(merged, candidates, complete_quarters) top3 = select_top_categories(results_df, N_TOP_CATEGORIES) overall_raw = quarterly_counts(merged, all_q) overall_series = build_series_with_extrapolation(overall_raw) overall_total = len(merged) category_raw = {} category_series = {} category_totals = {} for _, row in top3.iterrows(): label = f"{row['Category']} ({row['Facet']})" category_raw[label] = row["Quarterly counts"] category_series[label] = build_series_with_extrapolation(row["Quarterly counts"]) category_totals[label] = int(row["Total N"]) args.output_dir.mkdir(parents=True, exist_ok=True) fig_a = make_owntotal_figure(overall_series, category_series, overall_total, category_totals, complete_quarters) png_a = args.output_dir / "temporal_trend_normalized_owntotal.png" pdf_a = args.output_dir / "temporal_trend_normalized_owntotal.pdf" fig_a.savefig(png_a, dpi=DPI, bbox_inches="tight") fig_a.savefig(pdf_a, bbox_inches="tight") plt.close(fig_a) fig_b = make_shareofcorpus_figure(overall_raw, category_raw, complete_quarters) png_b = args.output_dir / "temporal_trend_normalized_shareofcorpus.png" pdf_b = args.output_dir / "temporal_trend_normalized_shareofcorpus.pdf" fig_b.savefig(png_b, dpi=DPI, bbox_inches="tight") fig_b.savefig(pdf_b, bbox_inches="tight") plt.close(fig_b) xlsx_path = args.output_dir / "temporal_trend_normalized.xlsx" with pd.ExcelWriter(xlsx_path, engine="openpyxl") as writer: labels = [quarter_label(y, q) for (y, q) in overall_series.index] owntotal_table = pd.DataFrame({ "Quarter": labels, "Overall corpus (%)": (overall_series / overall_total * 100).values, }) for label, series in category_series.items(): owntotal_table[f"{label} (%)"] = (series / category_totals[label] * 100).values owntotal_table.round(3).to_excel(writer, sheet_name="Own-total pct", index=False) share_table = pd.DataFrame({"Quarter": labels}) for label, series in category_raw.items(): share_table[f"{label}, share of corpus (%)"] = ((series / overall_raw) * 100).values share_table.round(3).to_excel(writer, sheet_name="Share of corpus pct", index=False) print(f"Saved {png_a.name}, {png_b.name}, {xlsx_path.name}") if __name__ == "__main__": main()