import io
import os
import sys
import time
import string
import re
import nltk
import langid
from nltk.corpus import stopwords
from nltk.tokenize import word_tokenize 

# Stop words list
stop_words = stopwords.words('english')

# function to remove stop words
def remove_stopWords(issue):
    global stop_words

    s = ''
    issue_token = word_tokenize(issue)
    
    for i in issue_token:
        if(len(i) > 1):
            if not i in stop_words:
                s += i + ' '
    

    return s

# Function to call to detect language 
def detectIdiom(textcontent):
    trad = langid.classify(textcontent)
    time.sleep(1)

    return str(trad[0])

# Function to clean all the unnecessary elements in the issue 
def clean_issue_text(issue):

    if(issue is None):
        return issue

    issue = issue.lower()
    
    # code removal
    issue = re.sub(r'`.+`', " ", issue)

    # TAGS removal
    issue = re.sub(r'<.*?>', " ", issue)

    # HREF removal
    issue = re.sub(r' href=".*?"', " ", issue)

    # table removal
    issue = re.sub(r'(\|)([\|\s\-]|[\:arrow\_down\:]|[\:arrow\_up\:]|[❌]|[❓]|[✔️])+(\|)', " ", issue)	

    # url removal
    issue = re.sub(r"https?://[A-Za-z0-9./]+", " ", issue)

    # ref table removal
    issue = re.sub(r'[\!][[]([a-z]|[0-9]|[@]|[:]|[\/]|[/]|[\_])+[]]', ' ', issue)

    # code examples removal
    issue = re.sub(r'`.+`', ' ', issue)
    issue = re.sub(r'~~~.+ ~~~', ' ', issue)

    # reply comments removal
    issue = re.sub(r'([\r\n\r\n])*>\s([\w]|[\d]|[—])*(.)*[.!?]*([\r\n\r\n])', ' ', issue)

    # Warnings removal
    issue = re.sub(r'([a-zA-Z]+\s[0-9]+[,]\s[0-9]+\s[0-9]+[:][0-9]+[:][0-9]+\s[aApPmM]+([\w]|[\d]|[.]|[_]|[\s])+[\n])*([A-Z])+(:)([a-zA-Z]|[.]|[0-9]|[\s]|[(]|[)]|[\[]|[\]]|[{]|[}])+(:)([a-zA-Z]|[.]|[0-9]|[\s]|[(]|[)]|[\[]|[\]]|[{]|[}])+[.]', ' ', issue)

    # Exceptions removal
    issue = re.sub(r'([\w]|[\d]|[.]|[_])+[\s]?[:]([\w]|[\d]|[.]|[_]|[\s])+[\']([\w]|[\d]|[.]|[_]|[\s])+[\']([\w]|[\d]|[.]|[_]|[\s]|[\(]|[\)]|[\$]|[\:])+[\)\n]', ' ', issue)

    # Class names removal
    issue = re.sub(r'[...]?[a-zA-Z]+([\.]|[\/])([\w]|[\d]|[\_]|[\.]|[\/])+([a-zA-Z]|[0-9])+', ' ', issue)

    # Path removal 
    issue = re.sub(r'[\[]? ((([\w]|[...])([\:]|[\/])([\w]|[\d]|[\]|[\\]|[\/]|[/]|[\.]|[|]|[\-])+ )) [\]]?', ' ', issue)

    # Remove different chars
    issue = re.sub(r"@[A-Za-z0-9]+", " ", issue)    
    
    # Numbers removal
    issue = re.sub(r"[^a-zA-Z]+", " ", issue)

    # multiple blanks spaces removal
    issue = re.sub(r" +", " ", issue)

    # stop words removal call function
    issue = remove_stopWords(issue)

    return issue

# countTitleIssues
def cleanAndUpdateIssueContent(curr_collection, i):
    print(' ------ Start clean issue')
    issue_title = clean_issue_text(i['Title'])
    issue_body = clean_issue_text(i['Body'])
    issue_id = i['Id']

    curr_collection.update_one({"Id": int(issue_id)}, {"$set": {"Title": issue_title}})
    curr_collection.update_one({"Id": int(issue_id)}, {"$set": {"Body": issue_body}})

    issue_comments = i['Comments']
    
    cleaned_comments = []

    for i in issue_comments:
        i['Comments'] = clean_issue_text(i['Comments'])
        cleaned_comments.append(i)

    curr_collection.update_one({"Id": int(issue_id)}, {"$set": {"Comments": cleaned_comments}})    
        
    #print('Verifying language in repo: '+str(i['Repository_name'] + ' Issue: '+ str(issue_id)))
    if(issue_title is None):
        issue_title = ' '
    
    if(issue_body is None):
        issue_body = ' '
 
    idiom_title = detectIdiom(issue_title)
    idiom_body  = detectIdiom(issue_body)

    if(idiom_title == 'en' or idiom_body == 'en'):
        curr_collection.update_one({"Id": int(issue_id)}, {"$set": {"isInEnglish": 1}})
        return True
    else:
        curr_collection.update_one({"Id": int(issue_id)}, {"$set": {"isInEnglish": 0}})
        return False

    print('Updated Issue language: '+ str(issue_id))