import pandas as pd
import spacy

# Load the Spacy large English model
nlp = spacy.load("en_core_web_lg")

# Load commit dataset and relevant commits dataset
df = pd.read_csv("commit_post_fix.csv")
relevant_commits = pd.read_csv("../sample_of_interest/relevant_commits.csv")

# Define the keywords of interest
kwords = ["vulnerability", "security"]

# Convert the 'is_fixing' column into boolean safely
df["is_fixing"] = df["is_fixing"].apply(lambda x: bool(x) if pd.notna(x) else False)

# Prepare variables
fixing_commit_hash = ''
fixing_message = ''
relevant_commit_hashes = set(relevant_commits["Commit"].astype(str).str.strip())
count = 0

# List to store the results before exporting to CSV
output_rows = []

# Iterate through each commit
for index, row in df.iterrows():

    is_fixing = row["is_fixing"]
    message = row["msg"]

    # Skip autogenerated or irrelevant commits (containing "* ")
    if "* " in message:
        continue

    # If the commit is a fixing commit, store its information
    if is_fixing:
        fixing_message = row["msg"]
        fixing_commit_hash = str(row["commit_hash"]).strip()

    else:
        # Process the commit message with Spacy NLP
        doc = nlp(message.lower())

        # Extract lemmata without stopwords
        lemmata_found = set([token.lemma_ for token in doc if not token.is_stop])

        # Check if message is relevant based on keywords or combination of 'fix' and 'security'
        if any(lemma in kwords for lemma in lemmata_found) or (("fix" in lemmata_found) and ("security" in lemmata_found)):
            later_commit_hash = str(row["commit_hash"]).strip()

            # Skip if it's already marked as relevant
            if later_commit_hash not in relevant_commit_hashes:

                # Display the information
                print(f"commit message: {message}")
                print(f"fixing commit message: {fixing_message}")
                print(f"fixing commit hash: {fixing_commit_hash}")
                print(f"later commit hash: {later_commit_hash}")
                print(f"Repo URL: {row['repo_url']}")

                # Save the data for later export
                output_rows.append({
                    "commit_message": message,
                    "fixing_commit_message": fixing_message,
                    "fixing_commit_hash": fixing_commit_hash,
                    "later_commit_hash": later_commit_hash,
                    "repo_url": row['repo_url']
                })
                count += 1

# Export the results to a CSV file
output_df = pd.DataFrame(output_rows)
output_df.to_csv("later_fixes.csv", index=False)

print("Total count:", count)
print("Saved to later_fixes.csv")
