df = pd.read_excel("Tokopedia.xlsx",engine='openpyxl')
df.columns = df.iloc[0]
df = df[1:]

df.head()
df.info

data_tokped = df[['Ulasan' ,'Rating']]
data_tokped.head()

data = data_tokped
data['Ulasan'] = data['Ulasan'].astype(str)
data['Ulasan'] = data['Ulasan'].str.lower()

import string 
import re 
from nltk.tokenize import word_tokenize 
from nltk.probability import FreqDist

def remove_links(text):
    # menghapus tab, new line, ans back slice
    text = text.replace('\\t'," ").replace('\\n'," ").replace('\\u'," ").replace('\\',"")
    # menghapus non ASCII (emoticon, chinese word, .etc)
    text = text.encode('ascii', 'replace').decode('ascii')
    # menghapus mention, link, hashtag
    text = ' '.join(re.sub("([@#][A-Za-z0-9]+)|(\w+:\/\/\S+)"," ", text).split())
    # menghapus URL
    return text.replace("http://", " ").replace("https://", " ")
                
data['Ulasan'] = data['Ulasan'].apply(remove_links)


#menghapus number
def remove_number(text):
    return  re.sub(r"\d+", " ", text)

data['Ulasan'] = data['Ulasan'].apply(remove_number)


#menghapus punctuation
def remove_punctuation(text):
    return text.translate(str.maketrans("","",string.punctuation))

data['Ulasan'] = data['Ulasan'].apply(remove_punctuation)


# menghapus single char
def remove_singl_char(text):
    return re.sub(r"\b[a-zA-Z]\b", " ", text)

data['Ulasan'] = data['Ulasan'].apply(remove_singl_char)


# Tokenisasi 
def word_tokenize_wrapper(text):
    return word_tokenize(text)

data['Ulasan_tokenize'] = data['Ulasan'].apply(word_tokenize_wrapper)


# Menghitung Distibusi Persebaran Kata
def freqDist_wrapper(text):
    return FreqDist(text)

Ulasan_fqsist = data['Ulasan_tokenize'].apply(freqDist_wrapper)

print('Frequency Tokens : \n') 
print(Ulasan_fqsist.head().apply(lambda x : x.most_common()))



slank_word_dict = {
    "keduakali" : "kedua kali",
    "agak" : "sedikit",
    "pas" : "saat",
    "produkx" : "produknya",
    "sukamkasih" : "suka makasih",
    "gak" : "tidak",
    "cpt" : "cepat",
    "sdh" : "sudah",
    "recommend" : "rekomendasi",
    "bagusseller" : "bagus",
    "bagusssss" : "bagus",
    "bagussssss" : "bagus",
    "baguuussss" : "bagus",
    "cepatmakasih": "cepat",
    "cepatmantappp": "cepat",
    "cepatproduct": "cepat",
    "cepatrecomended": "cepat",
    "cepattoko": "cepat",
    "recomended": "rekomendasi",
    "recommended": "rekomendasi",
    "rekomendasi": "rekomendasi",
    "rekomended": "rekomendasi",
    }

def slank_normalized_term(document):
    return [slank_word_dict[term] if term in slank_word_dict else term for term in document]



normalizad_word = pd.read_csv("kamus_alay.csv")

normalizad_word_dict = {}

for index, row in normalizad_word.iterrows():
    if row[0] not in normalizad_word_dict:
        normalizad_word_dict[row[0]] = row[1] 

def normalized_term(document):
    return [normalizad_word_dict[term] if term in normalizad_word_dict else term for term in document]



data['Ulasan_normalized'] = data['Ulasan_tokenize'].apply(normalized_term).apply(slank_normalized_term)


from nltk.corpus import stopwords
list_stopwords = stopwords.words('indonesian')
#remove stopword pada list token
def stopwords_removal(words):
    return [word for word in words if word not in list_stopwords]

data['Ulasan_stop_removed'] = data['Ulasan_normalized'].apply(stopwords_removal) 

pip install Sastrawi
pip install swifter


# import Sastrawi package
from Sastrawi.Stemmer.StemmerFactory import StemmerFactory
import swifter


# create stemmer
factory = StemmerFactory()
stemmer = factory.create_stemmer()

# stemmed
def stemmed_wrapper(term):
    return stemmer.stem(term)

term_dict = {}

for document in data['Ulasan_stop_removed']:
    for term in document:
        if term not in term_dict:
            term_dict[term] = ' '
            
print(len(term_dict))
print("------------------------")

for term in term_dict:
    term_dict[term] = stemmed_wrapper(term)
    print(term,":" ,term_dict[term])
    
print(term_dict)
print("------------------------")


# apply stemmed term to dataframe
def get_stemmed_term(document):
    return [term_dict[term] for term in document]

data['Ulasan_Stemmed'] = data['Ulasan_stop_removed'].swifter.apply(get_stemmed_term)



data["Ulasan_clean"] = [' '.join(map(str, l)) for l in data['Ulasan_Stemmed']]

data.to_excel("tokped_text.xlsx")



ulasan = ' '.join(str(v) for v in data['Ulasan_clean'])
tokenize_ulasan = word_tokenize(ulasan)
fqdist = FreqDist(tokenize_ulasan)
fqdist.most_common(15)

import matplotlib.pyplot as plt

# plotting
fqdist.plot(10,cumulative=False)
plt.show()