{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from keybert import KeyBERT\n",
    "import pandas as pd\n",
    "from sentence_transformers import SentenceTransformer\n",
    "from nltk.tokenize import line_tokenize\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_model = SentenceTransformer('all-mpnet-base-v2')\n",
    "kw_model = KeyBERT(model=sentence_model)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# keybert on sexism_dataset1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_path = \"/.../sexism_dataset1/EXIST2021_test_labeled.tsv\"\n",
    "test_data = pd.read_csv(test_path, sep='\\t')\n",
    "\n",
    "train_path = \"/.../sexism_dataset1/EXIST2021_training.tsv\"\n",
    "train_data = pd.read_csv(train_path, sep='\\t')\n",
    "\n",
    "# print(test_data)\n",
    "# print(train_data)\n",
    "\n",
    "sexist_data2 = []\n",
    "for index, row in test_data.iterrows():\n",
    "    # print(row['task1'])\n",
    "    if row['task1'] == 'sexist':\n",
    "        sexist_data2.append(row['text'])\n",
    "# print(len(sexist_data))\n",
    "\n",
    "for index, row in train_data.iterrows():\n",
    "    if row['task1'] == 'sexist':\n",
    "        sexist_data2.append(row['text'])\n",
    "\n",
    "print(len(sexist_data2))\n",
    "\n",
    "nslice = 5\n",
    "cslice2 = len(sexist_data2)//nslice\n",
    "print(f\"Number of each slice is: {cslice2}\")\n",
    "split_lists2 = [sexist_data2[x:x+cslice2] for x in range(0, len(sexist_data2), cslice2)]\n",
    "\n",
    "\n",
    "slice_candidreview2 = []\n",
    "for slice in split_lists2:\n",
    "    slice_candidreview2.append(\" \".join(slice))\n",
    "\n",
    "print(len(slice_candidreview2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_keywords = [\"sexism\" , \"gender bias\", \"gender discrimination\" , \"sexual discrimination\" , \"male chauvinism\" , \"antifeminism\",\n",
    " \"favouritism\" , \"discrimination\" , \"gender disparity\" , \"gender difference\" , \"gender inequality\" , \"gender inequity\" , \"gender imbalance\" ,\n",
    "  \"gender\" , \"feminism\" , \"patriarchy\" , \"misogyny\", \"misandry\" , \"lgbtq\" , \"egalitarianism\" , \"masculine\" , \"manly\" ,\n",
    "   \"manful\" , \"mannish\" , \"manlike\" , \"Womanly\", \"womanlike\", \"womanish\", \"femalelike\", \"unfeminine\", \"paternal\" ,\n",
    "    \"maternal\" , \"lgb\" , \"lgbt\", \"transgender\" , \"gay\" , \"lesbian\" , \"bisexual\" , \"homosexual\" , \"genderfluid\",\n",
    "     \"no-binary\" , \"nonbinary\" , \"non-binary\", \"intersex\" , \"agender\", \"girl\", \"boy\", \"men\", \"man\", \"woman\", \"women\", \"male\", \"female\"]\n",
    "\n",
    "\n",
    "candidate_keywords2 = []\n",
    "for slice_doc in slice_candidreview2:\n",
    "     rel_keywords2 = kw_model.extract_keywords(docs=slice_doc, keyphrase_ngram_range=(3,3), top_n=50, use_mmr=True, diversity=0.1, seed_keywords = seed_keywords, stop_words=None)\n",
    "     candidate_keywords2.append(rel_keywords2)\n",
    "     print(rel_keywords2)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# keybert on sexism_dataset2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "input_path = \"/.../sexism_dataset2/sexism_data.csv\"\n",
    "input_data = pd.read_csv(input_path)\n",
    "\n",
    "# print(input_data)\n",
    "sexist_data3 = []\n",
    "for index, row in input_data.iterrows():\n",
    "    # temp = row['sexist']\n",
    "    # print(type(temp))\n",
    "    if row['sexist'] == True:\n",
    "        \n",
    "        sexist_data3.append(row['text'])\n",
    "\n",
    "print(len(sexist_data3))\n",
    "\n",
    "nslice = 3\n",
    "cslice3 = len(sexist_data3)//nslice\n",
    "print(f\"Number of each slice is: {cslice3}\")\n",
    "split_lists3 = [sexist_data3[x:x+cslice3] for x in range(0, len(sexist_data3), cslice3)]\n",
    "\n",
    "\n",
    "slice_candidreview3 = []\n",
    "for slice in split_lists3:\n",
    "    slice_candidreview3.append(\" \".join(slice))\n",
    "\n",
    "print(len(slice_candidreview3))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_keywords = [\"sexism\" , \"gender bias\", \"gender discrimination\" , \"sexual discrimination\" , \"male chauvinism\" , \"antifeminism\",\n",
    " \"favouritism\" , \"discrimination\" , \"gender disparity\" , \"gender difference\" , \"gender inequality\" , \"gender inequity\" , \"gender imbalance\" ,\n",
    "  \"gender\" , \"feminism\" , \"patriarchy\" , \"misogyny\", \"misandry\" , \"lgbtq\" , \"egalitarianism\" , \"masculine\" , \"manly\" ,\n",
    "   \"manful\" , \"mannish\" , \"manlike\" , \"Womanly\", \"womanlike\", \"womanish\", \"femalelike\", \"unfeminine\", \"paternal\" ,\n",
    "    \"maternal\" , \"lgb\" , \"lgbt\", \"transgender\" , \"gay\" , \"lesbian\" , \"bisexual\" , \"homosexual\" , \"genderfluid\",\n",
    "     \"no-binary\" , \"nonbinary\" , \"non-binary\", \"intersex\" , \"agender\", \"girl\", \"boy\", \"men\", \"man\", \"woman\", \"women\", \"male\", \"female\"]\n",
    "\n",
    "\n",
    "candidate_keywords3 = []\n",
    "for slice_doc in slice_candidreview3:\n",
    "     rel_keywords3 = kw_model.extract_keywords(docs=slice_doc, keyphrase_ngram_range=(3,3), top_n=50, use_mmr=True, diversity=0.1, seed_keywords = seed_keywords, stop_words=None)\n",
    "     candidate_keywords3.append(rel_keywords3)\n",
    "     print(rel_keywords3)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# keybert on sexism dataset3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from distutils.util import strtobool\n",
    "import pandas as pd\n",
    "\n",
    "ben_sexism_path = \"/.../benevolent_sexist_text.csv\"\n",
    "ben_sexism_data = pd.read_csv(ben_sexism_path)\n",
    "\n",
    "hos_sexism_path = \"/.../hostile_sexist_text.csv\"\n",
    "hos_sexism_data = pd.read_csv(hos_sexism_path)\n",
    "\n",
    "print(len(ben_sexism_data))\n",
    "print(len(hos_sexism_data))\n",
    "\n",
    "sexism_data4 = []\n",
    "for index, row in ben_sexism_data.iterrows():\n",
    "    sexism_data4.append(str(row['text']))\n",
    "\n",
    "for index, row in hos_sexism_data.iterrows():\n",
    "    sexism_data4.append(str(row['text']))\n",
    "\n",
    "print(len(sexism_data4))\n",
    "\n",
    "# str_sexism_data4 = []\n",
    "# for row in sexism_data4:\n",
    "#     str_sexism_data4.append(str(row))\n",
    "\n",
    "nslice = 5\n",
    "cslice4 = len(sexism_data4)//nslice\n",
    "print(f\"Number of each slice is: {cslice4}\")\n",
    "split_lists4 = [sexism_data4[x:x+cslice4] for x in range(0, len(sexism_data4), cslice4)]\n",
    "\n",
    "\n",
    "\n",
    "slice_candidreview4 = []\n",
    "for slice in split_lists4:\n",
    "    slice_candidreview4.append(\" \".join(slice))\n",
    "\n",
    "print(len(slice_candidreview4))\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed_keywords = [\"sexism\" , \"gender bias\", \"gender discrimination\" , \"sexual discrimination\" , \"male chauvinism\" , \"antifeminism\",\n",
    " \"favouritism\" , \"discrimination\" , \"gender disparity\" , \"gender difference\" , \"gender inequality\" , \"gender inequity\" , \"gender imbalance\" ,\n",
    "  \"gender\" , \"feminism\" , \"patriarchy\" , \"misogyny\", \"misandry\" , \"lgbtq\" , \"egalitarianism\" , \"masculine\" , \"manly\" ,\n",
    "   \"manful\" , \"mannish\" , \"manlike\" , \"Womanly\", \"womanlike\", \"womanish\", \"femalelike\", \"unfeminine\", \"paternal\" ,\n",
    "    \"maternal\" , \"lgb\" , \"lgbt\", \"transgender\" , \"gay\" , \"lesbian\" , \"bisexual\" , \"homosexual\" , \"genderfluid\",\n",
    "     \"no-binary\" , \"nonbinary\" , \"non-binary\", \"intersex\" , \"agender\", \"girl\", \"boy\", \"men\", \"man\", \"woman\", \"women\", \"male\", \"female\"]\n",
    "\n",
    "\n",
    "candidate_keywords4 = []\n",
    "for slice_doc in slice_candidreview4:\n",
    "     rel_keywords4 = kw_model.extract_keywords(docs=slice_doc, keyphrase_ngram_range=(3,3), top_n=50, use_mmr=True, diversity=0.1, seed_keywords = seed_keywords, stop_words=None)\n",
    "     candidate_keywords4.append(rel_keywords4)\n",
    "     print(rel_keywords4)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.10 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.5 (v3.10.5:f377153967, Jun  6 2022, 12:36:10) [Clang 13.0.0 (clang-1300.0.29.30)]"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "aee8b7b246df8f9039afb4144a1f6fd8d2ca17a180786b69acc140d282b71a49"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}