import pandas as pd
import numpy as np
import ast
import os
from sentence_transformers import SentenceTransformer
from sklearn.metrics.pairwise import cosine_similarity


BASE_PATH = "XX"
DATA_PATH = f"{BASE_PATH}/dataforarticle1/clean_data_with_bert.csv" 
MODEL_PATH = f"{BASE_PATH}/model"

OUTPUT_DATA_PATH = f"{BASE_PATH}/data"
os.makedirs(OUTPUT_DATA_PATH, exist_ok=True)


df = pd.read_csv(DATA_PATH)

EMB_MATRIX_PATH = f"{OUTPUT_DATA_PATH}/embeddings_bert_matrix.npy"
CENTER_VECTORS_PATH = f"{OUTPUT_DATA_PATH}/center_vectors_bert.npy"
SIM_MATRIX_PATH = f"{OUTPUT_DATA_PATH}/interdisc_similarity_matrix.csv"


if os.path.exists(EMB_MATRIX_PATH) and os.path.exists(CENTER_VECTORS_PATH):
    embeddings = np.load(EMB_MATRIX_PATH, allow_pickle=True)
    center_vectors_bert = np.load(CENTER_VECTORS_PATH, allow_pickle=True).item()
else:
    model = SentenceTransformer(MODEL_PATH)
    abstracts = df['abstract'].tolist()
    batch_size = 256  
    embeddings = []    
    for i in range(0, len(abstracts), batch_size):
        batch = abstracts[i:i+batch_size]
        emb = model.encode(batch, show_progress_bar=True)
        embeddings.append(emb)
    embeddings = np.vstack(embeddings)
    np.save(EMB_MATRIX_PATH, embeddings)


 
 
import pandas as pd
import numpy as np
import os
from sklearn.metrics.pairwise import cosine_similarity
from scipy import stats
from scipy.cluster.hierarchy import linkage, leaves_list
import matplotlib.pyplot as plt
import seaborn as sns

BASE_PATH = "/home/rongyu/project/project/BB_project"
DATA_PATH = f"{BASE_PATH}/dataforarticle1/clean_data_with_bert.csv"
OUTPUT_PATH = f"{BASE_PATH}/result"
os.makedirs(OUTPUT_PATH, exist_ok=True)



df = pd.read_csv(DATA_PATH)

EMB_PATH = f"{BASE_PATH}/data/embeddings_bert_matrix.npy"
embeddings = np.load(EMB_PATH, allow_pickle=True)



disciplines = sorted(df['concept_name'].unique())
center_vectors_bert = {}

for d in disciplines:
    mask = df['concept_name'] == d
    center_vectors_bert[d] = np.mean(embeddings[mask], axis=0)


CENTER_PATH = f"{BASE_PATH}/data/center_vectors_bert.npy"
np.save(CENTER_PATH, center_vectors_bert)



self_sims = []
for idx, row in df.iterrows():
    d = row['concept_name']
    sim = cosine_similarity([embeddings[idx]], [center_vectors_bert[d]])[0][0]
    self_sims.append(sim)


df['self_sim'] = self_sims


cross_sims = []
for idx, row in df.iterrows():
    d = row['concept_name']
    other_centers = [center_vectors_bert[dd] for dd in disciplines if dd != d]
    sims = cosine_similarity([embeddings[idx]], other_centers)[0]
    cross_sims.append(np.mean(sims))


df['cross_sim_mean'] = cross_sims


group1 = df['self_sim'].values
group2 = df['cross_sim_mean'].values
t_stat, p_val = stats.ttest_ind(group1, group2, equal_var=False)


table1_data = {
    'Group': ['Within-discipline', 'Between-disciplines'],
    'N': [len(df), len(df)],
    'Mean': [np.mean(group1).round(3), np.mean(group2).round(3)],
    'SD': [np.std(group1).round(3), np.std(group2).round(3)],
    'Min': [np.min(group1).round(3), np.min(group2).round(3)],
    'Max': [np.max(group1).round(3), np.max(group2).round(3)]
}


table1 = pd.DataFrame(table1_data)
table1.to_csv(f"{OUTPUT_PATH}/Table1_Similarity_Stats.csv", index=False)


n_disc = len(disciplines)
sim_matrix = np.zeros((n_disc, n_disc))
for i, d1 in enumerate(disciplines):
    for j, d2 in enumerate(disciplines):
        sim_matrix[i, j] = cosine_similarity([center_vectors_bert[d1]], [center_vectors_bert[d2]])[0][0]


df_sim = pd.DataFrame(sim_matrix, index=disciplines, columns=disciplines)


plt.rcParams['font.size'] = 10
g = sns.clustermap(
    df_sim, 
    annot=True, 
    fmt='.3f', 
    cmap='RdBu_r', 
    center=0,
    figsize=(14, 12), 
    linewidths=0.5, 
    cbar_kws={"shrink": 0.8, "label": "Cosine similarity"}
)
g.ax_heatmap.set_xticklabels(g.ax_heatmap.get_xticklabels(), rotation=45, ha='right')
g.ax_heatmap.set_yticklabels(g.ax_heatmap.get_yticklabels(), rotation=0)

plt.savefig(f"{OUTPUT_PATH}/Fig1_Hierarchical_Clustering.png", dpi=300, bbox_inches='tight')
plt.savefig(f"{OUTPUT_PATH}/Fig1_Hierarchical_Clustering.pdf", bbox_inches='tight')
plt.close()


FINAL_CSV = f"{BASE_PATH}/data/clean_data_with_validity_scores.csv"
df.to_csv(FINAL_CSV, index=False)



import pandas as pd
import numpy as np
import os
from sklearn.metrics.pairwise import cosine_similarity
from scipy import stats
from scipy.cluster.hierarchy import linkage, dendrogram
import matplotlib.pyplot as plt
import seaborn as sns
import warnings
warnings.filterwarnings('ignore')


BASE_DIR = "XX"
OUT_DIR  = "XXX"
os.makedirs(OUT_DIR, exist_ok=True)


df = pd.read_csv(os.path.join(BASE_DIR, "clean_data_with_validity_scores.csv"))
embeddings = np.load(os.path.join(BASE_DIR, "embeddings_bert_matrix.npy"), allow_pickle=True)
center_vectors = np.load(os.path.join(BASE_DIR, "center_vectors_bert.npy"), allow_pickle=True).item()
disciplines = sorted(df['concept_name'].unique())


table1_rows = []


for d in disciplines:
    mask = df['concept_name'] == d
    idx_sub = mask[mask].index.tolist()
    N = len(idx_sub)
    if N < 10:
        continue
    within_sim = []
    for i in idx_sub:
        sim = cosine_similarity([embeddings[i]], [center_vectors[d]])[0][0]
        within_sim.append(sim)
    within_mean = np.round(np.mean(within_sim), 3)
    within_sd = np.round(np.std(within_sim), 3)
    other_discs = [dd for dd in disciplines if dd != d]
    between_sim = []
    for i in idx_sub:
        sims = [cosine_similarity([embeddings[i]], [center_vectors[dd]])[0][0] for dd in other_discs]
        between_sim.append(np.mean(sims))
    between_mean = np.round(np.mean(between_sim), 3)
    between_sd = np.round(np.std(between_sim), 3)
    t_stat, p_val = stats.ttest_ind(within_sim, between_sim, equal_var=False)
    t_stat = np.round(t_stat, 2)
    sig = "***" if p_val < 0.001 else "**" if p_val < 0.01 else "*" if p_val < 0.05 else ""
    table1_rows.append({
        "Discipline": d,
        "N": N,
        "Within_Mean": within_mean,
        "Within_SD": within_sd,
        "Between_Mean": between_mean,
        "Between_SD": between_sd,
        "t_stat": t_stat,
        "Significance": sig
    })

within_all = []
between_all = []
for idx, row in df.iterrows():
    d = row['concept_name']
    within_all.append(cosine_similarity([embeddings[idx]], [center_vectors[d]])[0][0])
    other_discs = [dd for dd in disciplines if dd != d]
    sims = [cosine_similarity([embeddings[idx]], [center_vectors[dd]])[0][0] for dd in other_discs]
    between_all.append(np.mean(sims))

t_total, p_total = stats.ttest_ind(within_all, between_all, equal_var=False)
table1_rows.append({
    "Discipline": "Total (All Disciplines)",
    "N": len(df),
    "Within_Mean": np.round(np.mean(within_all), 3),
    "Within_SD": np.round(np.std(within_all), 3),
    "Between_Mean": np.round(np.mean(between_all), 3),
    "Between_SD": np.round(np.std(between_all), 3),
    "t_stat": np.round(t_total, 2),
    "Significance": "***" if p_total < 0.001 else ""
})

table1 = pd.DataFrame(table1_rows)
table1.to_csv(os.path.join(OUT_DIR, "Table1_Descriptive_statistics_of_semantic_similarity.csv"), index=False)
print(table1.to_string(index=False))


disc_list = sorted(disciplines)
sim_matrix = np.zeros((len(disc_list), len(disc_list)))
for i, d1 in enumerate(disc_list):
    for j, d2 in enumerate(disc_list):
        sim_matrix[i,j] = cosine_similarity([center_vectors[d1]], [center_vectors[d2]])[0][0]

distance_matrix = 1 - sim_matrix 


linkage_matrix = linkage(distance_matrix, method='ward')


plt.figure(figsize=(12, 8), dpi=300)
dendrogram(
    linkage_matrix,
    labels=disc_list,
    leaf_rotation=45,
    leaf_font_size=9,
    color_threshold=0.7,  
    above_threshold_color='black'
)


plt.title("Fig 1. Hierarchical clustering tree of 19 disciplinary prototype vectors", fontsize=12, pad=15)
plt.ylabel("Euclidean Distance (1 - Cosine Similarity)", fontsize=10)
plt.tight_layout()


plt.savefig(os.path.join(OUT_DIR, "Fig1_Hierarchical_Clustering_Tree.png"), dpi=300, bbox_inches='tight')
plt.savefig(os.path.join(OUT_DIR, "Fig1_Hierarchical_Clustering_Tree.pdf"), bbox_inches='tight')
plt.close()






import pandas as pd
import numpy as np


MAP_PATH = "xxxx/wos-OpenAlex.txt"
DATA_PATH = "xxx/clean_data_with_validity_scores.csv"
WOS_PATH = "xxx/wos-core_SCIE.csv"
OUT_DIR = "xxx"


mapping = pd.read_csv(MAP_PATH, sep="\t", engine="python", header=None, names=["WoS_Category", "OpenAlex_Discipline"])
WOS_TO_OA = dict(zip(mapping["WoS_Category"].str.strip(), mapping["OpenAlex_Discipline"].str.strip()))


df = pd.read_csv(DATA_PATH)


def extract_journal(json_text):
    try:
        data = eval(json_text)
        return data["source"]["display_name"]
    except:
        return np.nan

df["journal"] = df["primary_location"].apply(extract_journal)


wos = pd.read_csv(WOS_PATH)


def clean_name(x):
    if pd.isna(x):
        return np.nan
    return str(x).strip().lower()

df["journal_clean"] = df["journal"].apply(clean_name)
wos["journal_clean"] = wos["Journal title"].apply(clean_name)


df = df.merge(
    wos[["journal_clean", "Web of Science Categories"]],
    on="journal_clean",
    how="left"
)


def get_primary_category(cat_str):
    if pd.isna(cat_str):
        return np.nan
    return [x.strip() for x in str(cat_str).split("|")][0]

df["wos_primary"] = df["Web of Science Categories"].apply(get_primary_category)
df["wos_mapped"] = df["wos_primary"].map(WOS_TO_OA)


df_valid = df.dropna(subset=["wos_mapped"]).copy()
df_valid["is_correct"] = df_valid["concept_name"] == df_valid["wos_mapped"]


total = len(df_valid)
correct = df_valid["is_correct"].sum()
accuracy = correct / total * 100 if total > 0 else 0

summary.to_csv(f"{OUT_DIR}Table3_WoS_Consistency.csv", index=False, encoding="utf-8-sig")

dis_detail.to_csv(f"{OUT_DIR}Table3_Discipline_Detail.csv", index=False, encoding="utf-8-sig")






import pandas as pd
import numpy as np
import os
import pickle
from tqdm import tqdm
from collections import Counter
from sklearn.metrics.pairwise import cosine_similarity


BASE_DIR = "xxxx"
RAW_DIR = os.path.join(BASE_DIR, "rawdata")

df = pd.read_csv(os.path.join(BASE_DIR, "clean_data_with_classic_indicators.csv"))
with open(os.path.join(RAW_DIR, "ref_id_to_mylevel0.pkl"), "rb") as f:
    ref_to_level0 = pickle.load(f)

center_vectors = np.load(os.path.join(RAW_DIR, "center_vectors_bert.npy"), allow_pickle=True).item()
DISCIPLINES = sorted(df["concept_name"].unique())
N_DISC = len(DISCIPLINES)


vec_matrix = np.array([center_vectors[d] for d in DISCIPLINES])
sim_matrix = cosine_similarity(vec_matrix)
dist_matrix = 1.0 - sim_matrix


simpson_list = []
rao_list = []

for _, row in tqdm(df.iterrows(), total=len(df)):
    try:
        referenced_works = eval(row["referenced_works"])
    except:
        simpson_list.append(np.nan)
        rao_list.append(np.nan)
        continue

    ref_discs = []
    for url in referenced_works:
        ref_id = url.split("/")[-1]
        disc = ref_to_level0.get(ref_id)
        if disc is not None:
            ref_discs.append(disc)

    if len(ref_discs) == 0:
        simpson_list.append(0.0)
        rao_list.append(0.0)
        continue


    count = Counter(ref_discs)
    total_refs = len(ref_discs)
    p = np.array([count.get(d, 0) / total_refs for d in DISCIPLINES])

    s_val = 1.0 - np.sum(p ** 2)

    rs_val = np.sum(np.outer(p, p) * dist_matrix)

    simpson_list.append(s_val)
    rao_list.append(rs_val)


df["simpson_diversity"] = simpson_list
df["rao_stirling"] = rao_list
df.to_csv(os.path.join(BASE_DIR, "clean_data_with_FINAL_CORRECT.csv"), index=False, encoding="utf-8")


import pandas as pd
import numpy as np
import statsmodels.api as sm


df = pd.read_csv("/home/rongyu/project/project/BB_project/dataforarticle1/clean_data_with_FINAL_CORRECT.csv")


df_bert = pd.read_csv("/home/rongyu/project/project/BB_project/dataforarticle1/rawdata/clean_data_final.csv")
df = pd.merge(df_bert, df[["id", "simpson_diversity", "rao_stirling"]], on="id", how="inner")


corr_vars = ["cross_sim", "simpson_diversity", "rao_stirling"]
print(df[corr_vars].corr().round(3))


y = df["log_cite"]
controls = ["n_authors", "n_refs", "is_oa", "publication_year"]

X1 = df[["simpson_diversity", "rao_stirling"] + controls].copy()
X1 = sm.add_constant(X1)
model1 = sm.OLS(y, X1, missing="drop").fit(cov_type="cluster", cov_kwds={"groups": df["concept_name"]})

X2 = df[["simpson_diversity", "rao_stirling", "cross_sim"] + controls].copy()
X2 = sm.add_constant(X2)
model2 = sm.OLS(y, X2, missing="drop").fit(cov_type="cluster", cov_kwds={"groups": df["concept_name"]})