# -*- coding: utf-8 -*- """ Created on Wed Jan 25 16:15:40 2023 @author: gsun SmartBuildSkip.py is a implemented according to "A Cost-efficient Approach to Building in Continuous Integration" by Xianhao Jin and Francisco Servant Please refer to the original paper! """ import numpy as np import pandas as pd from sklearn.metrics import matthews_corrcoef from collections import defaultdict from collections import Counter from sklearn.metrics import roc_auc_score,accuracy_score from sklearn.metrics import confusion_matrix import math,time from imblearn.combine import SMOTEENN from imblearn.under_sampling import RandomUnderSampler from imblearn.over_sampling import RandomOverSampler, SMOTE from sklearn.feature_selection import SelectFromModel,SelectKBest,chi2,f_classif,mutual_info_classif,SelectFdr,SelectFpr import os from sklearn import preprocessing from xgboost import XGBClassifier from sklearn.metrics import f1_score,recall_score,precision_score import random from sklearn.preprocessing import StandardScaler from sklearn.ensemble import RandomForestClassifier import warnings warnings.filterwarnings('ignore') #%% def SBS(model, sameflip): auc_roc=[] f10,f11,recall0, recall1, precision0, precision1=[],[],[],[],[],[] file_list=os.listdir(r"./20_projects/") file_list = list(filter(lambda x:x[-3:] =='csv',file_list)) pr_df =[] pr_shapes = [[],[]] pr_as_pred = [[],[],[],[]] auc_roc,auc_parrot = [],[] mcc = [] for i in range(0,len(file_list)): new_data=pd.read_csv("./20_projects/"+file_list[i],low_memory=False) pass_fail_total[0].append(new_data['now_label'].sum()) pass_fail_total[1].append(new_data['now_label'].shape[0]-new_data['now_label'].sum()) pass_fail_total[2].append(new_data['now_label'].shape[0]) pr_now = new_data.loc[new_data['last_label']!=new_data['now_label']].reset_index() pr_same = new_data.loc[new_data['last_label']==new_data['now_label']].reset_index() pr_shapes[0].append(pr_now.shape[0]) pr_shapes[1].append(new_data.shape[0]) new_features =['src_churn', 'test_churn', 'commits', 'time_interval', 'day_of_week','time_of_day', 'fail_ratio_pr', 'fail_ratio_re', 'last_fail_gap_avg','pr_src_files'] new_data=new_data[new_features+['now_label','last_label']] test_size=math.ceil(new_data.shape[0]/5) test_data=new_data.tail(test_size) train_data=new_data.drop(index=test_data.index) train_data.reset_index(inplace =True,drop=True) test_data.reset_index(inplace =True,drop=True) y_test_collect = list(test_data['now_label']) x_test=test_data.drop(['now_label'],axis=1) y_train=train_data['now_label'] X_train=train_data.drop(['now_label'],axis=1)[new_features] if sameflip=='sameflip': while True: rus=RandomUnderSampler(random_state=None) X_same_pass = train_data.loc[(train_data['last_label']==train_data['now_label']&(train_data['now_label']==0))].reset_index(drop=True) X_same_fail = train_data.loc[(train_data['last_label']==train_data['now_label']&(train_data['now_label']==1))].reset_index(drop=True) X_diff = train_data.loc[train_data['last_label']!=train_data['now_label']].reset_index(drop=True) y_same_pass = [0] * X_same_pass.shape[0] y_same_fail = [0] * X_same_fail.shape[0] y_diff = [1] * X_diff.shape[0] X_train = pd.concat([X_same_pass,X_same_fail,X_diff],axis=0).reset_index(drop=True)[new_features+['now_label']] y_train = y_same_pass+y_same_fail+y_diff X_train, y_train = rus.fit_sample(X_train, y_train) y_train = X_train['now_label'] X_train=X_train.drop(['now_label'],axis=1) if y_train.sum()!=len(y_train): #sanity check break elif sameflip=='under': rus=RandomUnderSampler(random_state=None) X_train, y_train = rus.fit_sample(X_train, y_train) if model == 'XGB': RF=XGBClassifier(random_state=0) else: RF=RandomForestClassifier(random_state=0) RF.fit(X_train,y_train) y_pred_collect = RF.predict(x_test[new_features]) predict_proba = RF.predict_proba(x_test[new_features]) y_pr = list(test_data['last_label']) y_pr = list(test_data['last_label']) # auc_roc.append(roc_auc_score(y_test_collect,predict_proba)) auc_parrot.append(roc_auc_score(y_test_collect,y_pr)) result_tosee.append(f1_score(y_test_collect,y_pred_collect,average='weighted')) auc_roc.append(roc_auc_score(y_test_collect,predict_proba[:,0],average='weighted')) print(file_list[i]) f10.append(f1_score(y_test_collect,y_pred_collect,average=None)[0]) f11.append(f1_score(y_test_collect,y_pred_collect,average=None)[1]) recall0.append(recall_score(y_test_collect,y_pred_collect,average=None)[0]) recall1.append(recall_score(y_test_collect,y_pred_collect,average=None)[1]) precision0.append(precision_score(y_test_collect,y_pred_collect,average=None)[0]) precision1.append(precision_score(y_test_collect,y_pred_collect,average=None)[1]) test_data['predict'] = y_pred_collect pr_df.append(test_data[['last_label','now_label','predict']]) pr_as_pred[0].append(precision_score(y_test_collect,y_pr,average=None)[0]) pr_as_pred[1].append(precision_score(y_test_collect,y_pr,average=None)[1]) pr_as_pred[2].append(recall_score(y_test_collect,y_pr,average=None)[0]) pr_as_pred[3].append(recall_score(y_test_collect,y_pr,average=None)[1]) mcc.append(matthews_corrcoef(y_test_collect,y_pred_collect)) return pr_df,pre_as_pred,mcc #%% model = "" sameflip = "" fsize = 10 from Plot import * pr_df,pr_as_pred,mcc = SBS(model,sameflip) shapes,overall,same,diff = compare(pr_df) pr_as_pred,overall,same,diff,pr_shapes = add_project_a_sbs(pr_as_pred,overall,same,diff,pr_shapes) plot_scatter_plot(same,diff,fsize,shapes,x_label = "Same",y_label = "Flip",title ="SBS-SameFlip") plot_scatter_plot(overall,pr_as_pred,fsize,shapes,x_label = "Parrot",y_label = "State-of-the-Art",title ="SBS-Parrot") bar_projects(pr_shapes) #%% print("SBS",sameflip) # print('overall') # print("precision f",format(np.mean(overall[0]),'.3f')) # print("precision p",format(np.mean(overall[1]),'.3f')) # print("recall f",format(np.mean(overall[2]),'.3f')) # print("recall p",format(np.mean(overall[3]),'.3f')) # print("mcc",format(np.mean(mcc),'.3f')) # print('same') # print("precision f",format(np.mean(same[0]),'.1f')) # print("precision p",format(np.mean(same[1]),'.3f')) # print("recall f",format(np.mean(same[2]),'.3f')) # print("recall p",format(np.mean(same[3]),'.3f')) # print('diff') # print("precision f",format(np.mean(diff[0]),'.3f')) # print("precision p",format(np.mean(diff[1]),'.3f')) # print("recall f",format(np.mean(diff[2]),'.3f')) # print("recall p",format(np.mean(diff[3]),'.3f')) print("substract") print("precision f",format(np.mean(same[0])-np.mean(diff[0]),'.3f')) print("precision p",format(np.mean(same[1])-np.mean(diff[1]),'.3f')) print("recall f",format(np.mean(same[2])-np.mean(diff[2]),'.3f')) print("recall p",format(np.mean(same[3])-np.mean(diff[3]),'.3f')) print("MCC",format(np.mean(mcc),'.3f')) #%% headers = ['Pass','Fail','Total'] print(tabulate(pass_fail_total, headers=headers))