Data analysis scripts for paper Classifying the Correctness of Generated White-Box Tests: An Exploratory Study
Dávid Honfi and Zoltán Micskei
27th February 2017
1 Introduction
This file contains the data analysis workflow of the our empirical investigation of generated oracles. The goal of this documentation is to provide a detailed description of how we analyze the data obtained from the experiment.
This scripts has the results folder as its input. The script traverses the results from both experiment dates and searches for CSV and JSON files with the given names.
2 Reading the results
The results are stored in two subfolders in the results folder: dec1 and dec8. In dec1, the participant identifiers are found from 1 to 40, while in dec8 from 41 to 80. Note that these are only participant identifiers, not all folders contain results, as there were only 54 participants in total.
2.1 Parsing participant answers
Each participant answer list is being put into a data frame. Then, all of the answers appended to a huge data frame containing all participant data.
answer_files <- list.files(params$results_folder, pattern="*.json", recursive=TRUE, full.names=TRUE)
result_answers <- data.frame(row.names = c("PID","Id","IsOK"), stringsAsFactors = FALSE)
for (json in answer_files) {
# Getting the answers of the participants from the JSON file
answers <- data.frame(fromJSON(readLines(json),simplifyDataFrame=TRUE),stringsAsFactors = FALSE)
# Getting the PID from the base directory
pid <- as.numeric(rep(basename(dirname(json)),nrow(answers)))
# Creating the column for the participant identifier
answers_with_pid <- cbind(pid,answers)
# Appending the new rows the previouses
result_answers <- rbind(result_answers,answers_with_pid)
}
rm(json,pid,answers,answers_with_pid,answer_files)2.2 Adding project and method information
result_answers$Project <- ifelse(result_answers$pid>40,"MathNet","NBitcoin")
nbitcoin_methods <- c(rep("CompareTo",3),rep("Constructor",3),rep("Equals",3),rep("Min",3),rep("Plus",3))
mathnet_methods <- c(rep("Combinations",3),rep("CombinationsWithRepetition",3),rep("Permutations",3),rep("Variations",3),rep("VariationsWithRepetition",3))
# Creating the mapping between test identifiers, projects and methods
test_method_mapping <- data.frame(Results.Id=rep(0:14,2),Method=c(nbitcoin_methods,mathnet_methods),Project=c(rep("NBitcoin",15),rep("MathNet",15)))
# Adding the method name to each row
result_answers <- plyr::join(result_answers,test_method_mapping,by=c("Results.Id","Project"))
# Adding test identifiers to each row
result_answers$TestId <- car::recode(as.factor(result_answers$Results.Id),"0='T0.1'; 1='T1.1'; 2='T2.1'; 3='T3.2'; 4='T4.2'; 5='T5.2'; 6='T6.3'; 7='T7.3'; 8='T8.3'; 9='T9.4'; 10='T10.4'; 11='T11.4'; 12='T12.5'; 13='T13.5'; 14='T14.5'")
rm(nbitcoin_methods,mathnet_methods)2.3 Parsing golden answers
The golden answers are stored in two files for the projects NBitcoin and MathNet: nbitcoin-gold.csv and mathnet-gold.csv. These are being parsed here.
nbitcoin_golden_answers <- read.csv(file.path(params$golden_folder,"nbitcoin-gold.csv"), header=TRUE, sep = ";",na.strings=c("",""),stringsAsFactors = FALSE)
mathnet_golden_answers <- read.csv(file.path(params$golden_folder,"mathnet-gold.csv"), header=TRUE, sep = ";",na.strings=c("",""),stringsAsFactors = FALSE)2.4 Parsing video logs
Each video annotation of the participants is put into a data frame. Then, all of the annotations form a huge data frame with participant identifiers. Note that some rows at the start of the CSV from Boris contains only irrelevant metadata, thus they are being skipped dynamically.
video_files <- list.files(params$results_folder, pattern="video.csv", recursive=TRUE, full.names=TRUE)
video_lengths <- data.frame()
result_videos <- data.frame()
for (video in video_files) {
# Getting the number of lines to skip at head (max. 20)
lines_to_skip <- grep("^Time,", readLines(video, n = 20))-1
# Reading annotations
annotations <- read.csv(video,header=TRUE,skip=lines_to_skip,sep=",",as.is = TRUE)
# Creating PID column based on the number of rows
pid <- rep(strsplit(readLines(video,n=1),split = ',')[[1]][2],nrow(annotations))
# Adding the PID column to the annotations
annotations_with_pid <- cbind(pid, annotations)
# Appending the new annotations to the previouses
result_videos <- rbind(result_videos,annotations_with_pid)
# Appending video length
video_lengths <- filter(result_videos %>% select(pid,Media.total.length) %>% distinct_(),pid != 55 & pid != 59)
}
print("Video lengths data NBitcoin")## [1] "Video lengths data NBitcoin"video_lengths_nbitcoin <- filter(video_lengths, as.numeric(pid) < 41)
print(paste("Min", min(video_lengths_nbitcoin$Media.total.length/60) , sep=" "))## [1] "Min 37.4133333333333"print(paste("Median",median(video_lengths_nbitcoin$Media.total.length/60), sep=" "))## [1] "Median 46.9433333333333"print(paste("Mean", mean(video_lengths_nbitcoin$Media.total.length/60), sep=" "))## [1] "Mean 46.2582905982906"print(paste("Max", max(video_lengths_nbitcoin$Media.total.length/60), sep=" "))## [1] "Max 54.9"print(paste("sd", sd(video_lengths_nbitcoin$Media.total.length/60), sep=" "))## [1] "sd 3.95760995213189"print("Video lengths data MathNet")## [1] "Video lengths data MathNet"video_lengths_mathnet <- filter(video_lengths, as.numeric(pid) > 40)
print(paste("Min", min(video_lengths_mathnet$Media.total.length/60) , sep=" "))## [1] "Min 34.8766666666667"print(paste("Median",median(video_lengths_mathnet$Media.total.length/60), sep=" "))## [1] "Median 44.5666666666667"print(paste("Mean", mean(video_lengths_mathnet$Media.total.length/60), sep=" "))## [1] "Mean 44.5192307692308"print(paste("Max", max(video_lengths_mathnet$Media.total.length/60), sep=" "))## [1] "Max 52.61"print(paste("sd", sd(video_lengths_mathnet$Media.total.length/60), sep=" "))## [1] "sd 5.24144155210754"#rm(video,lines_to_skip,video_files,annotations,annotations_with_pid,pid)2.5 Parsing background questionnaire
The answers of the background questionnaires for each participant are parsed here into a large data frame containing every participant.
bg_answers <- list.files(params$results_folder, pattern="background.csv", recursive=TRUE, full.names=TRUE)
result_bg_answers <- data.frame()
for (answer_file in bg_answers) {
answers <- read.csv(answer_file,header=TRUE,sep=";",as.is = TRUE)
colnames(answers)[1] <- "PID"
result_bg_answers <- rbind(result_bg_answers,answers)
}
rm(bg_answers,answers,answer_file)2.6 Parsing exit questionnaire
The answers of the exit questionnaires for each participant are parsed here into a large data frame containing every participant.
exit_answers <- list.files(params$results_folder, pattern="exit.csv", recursive=TRUE, full.names=TRUE)
result_exit_answers <- data.frame()
for (answer_file in exit_answers) {
answers <- read.csv(answer_file,header=TRUE,sep=";",as.is = TRUE)
colnames(answers)[1] <- "PID"
result_exit_answers <- rbind(result_exit_answers,answers)
}
rm(exit_answers,answer_file,answers)3 Data transformations, calculations
3.1 Participant answers compared with goldens
This section computes some basic results of the participants using the golden data. This includes classification of the participants’ answers as true positive (TP), false positive (FP), true negative (TN) and false negative (FN).
participant_results <- data.frame()
# Iterating through participants
for(pid in unique(result_answers$pid)) {
result_column <- c()
# Iterating through answers for checking correctness
for(test in result_answers[result_answers$pid == pid,]$Results.Id) {
if(as.numeric(pid) < 41) {
# Less than 41 is NBitcoin
golden_answers <- nbitcoin_golden_answers
} else {
# Otherwise is MathNet
golden_answers <- mathnet_golden_answers
}
participant_answer<- result_answers[result_answers$pid == pid & result_answers$Results.Id == test,]$Results.IsOK
if(golden_answers[golden_answers$id == test,]$isok == TRUE) {
# If the golden answer is OK
if(participant_answer == TRUE) {
result_column <- rbind(result_column,"TN")
} else {
result_column <- rbind(result_column,"FP")
}
} else {
# If the golden answer is Wrong
if(participant_answer == FALSE) {
result_column <- rbind(result_column,"TP")
} else {
result_column <- rbind(result_column,"FN")
}
}
}
participant_result_table <- cbind(result_answers[result_answers$pid == pid,],result_column)
colnames(participant_result_table)[7] <- "Check"
participant_results <- rbind(participant_results,participant_result_table)
}
rm(participant_result_table,result_column,golden_answers,pid,test,participant_answer)The next code summarizes the results of the participants in each row by enumerating their TPs, FPs, TNs and FNs.
participant_result_summary <- data.frame(row.names = c("PID","Project","TP","FP","TN","FN"),stringsAsFactors=FALSE)
# Iterating through participants
for(pid in unique(result_answers$pid)) {
table_with_checks <- participant_results[participant_results$pid == pid,]
tp_count <- length(which(table_with_checks$Check == "TP"))
fp_count <- length(which(table_with_checks$Check == "FP"))
tn_count <- length(which(table_with_checks$Check == "TN"))
fn_count <- length(which(table_with_checks$Check == "FN"))
if(as.numeric(pid) > 40) {
project = "MathNet"
} else {
project = "NBitcoin"
}
participant_summary_row <- data.frame(PID=pid, Project=project, TP=tp_count, FP=fp_count, TN=tn_count, FN=fn_count)
participant_result_summary <- rbind(participant_summary_row, participant_result_summary)
}
rm(participant_summary_row,project,tp_count,tn_count,fp_count,fn_count,table_with_checks,pid)3.2 Video log transformations and summaries
In this section, the video logs for each participant are analyzed and transformed. This is important for evaluating the time required for each activity. The code collects and summarizes the behaviors for each participant. This includes 1) the time spent with each case both on portal and Visual Studio (VS), 2) time spent with debugging at all and for each case as well, 3) the number of test runs per case, 4) the number of consistent markings per case, 5) the number of changing markings (Wrong to OK, OK to Wrong) per case.
extended_video_annotations <- data.frame(row.names = c("PID","Timestamp","Behavior","Modifier","Active","Page","Window","RunCase","DebugLength","DebugCase"),stringsAsFactors = FALSE)
vs_time_summary <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5","S","C","P","Total"), stringsAsFactors = FALSE)
portal_time_summary <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
marked_wrong_summary <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
marked_ok_summary <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
cut_time_for_test <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
sut_time_for_test <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
put_time_for_test <- data.frame(row.names = c("PID","T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), stringsAsFactors = FALSE)
# Iterating through participants
for(pid in unique(result_answers$pid)) {
annotations_for_participant <- result_videos[result_videos$pid == pid,]
vs_time_summary_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0, S = 0, C = 0, P = 0, Total = 0)
portal_time_summary_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
marked_wrong_summary_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
marked_ok_summary_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
cut_time_for_test_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
sut_time_for_test_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
put_time_for_test_row <- data.frame(PID = pid, T0.1 = 0, T1.1 = 0, T2.1 = 0, T3.2 = 0, T4.2 = 0, T5.2 = 0, T6.3 = 0, T7.3 = 0, T8.3 = 0, T9.4 = 0, T10.4 = 0, T11.4 = 0, T12.5 = 0, T13.5 = 0, T14.5 = 0)
# Iterating through the annotations of the participant
currently_active <- NA
vs_window <- NA
portal_page <- NA
debug_start_index <- NA
vs_start_index <- -1 # -1: VS has not been activated yet, otherwise the row index of activation point
window_start_index <- -1 # -1: A window in VS has not been activated yet, otherwise the row index of activation point
page_start_index <- -1 # -1: A page in the portal has not been activated yet, otherwise the row index of activation point
# Iterating through each row of a participant
for(i in 1:nrow(annotations_for_participant)) {
# Getting the corresponding row
row <- annotations_for_participant[i,]
# If the row is na somehow (due to some R bug)
if(is.na(row$pid)) {
break;
}
marked_case <- NA
run_case <- NA
debug_time <- NA
debug_case <- NA
# If the portal has been activated
if(row$Behavior == "Portal activated") {
currently_active <- "Portal" # Setting the currently active variable to Portal
if(is.na(portal_page)) {
# If the portal is activated and there were no previous pages, then the home page (H) will open
portal_page = "H"
}
page_start_index <- i
##### VS full timer summarization #####
# If there was a VS activation before, it must be stopped and added to the summary
if(vs_start_index != -1) {
# The elapsed time is the current row timestamp minus the previous VS activation timestamp
elapsed_vs_time <- row$Time - annotations_for_participant[annotations_for_participant$pid == pid,][vs_start_index,]$Time
# The elapsed time is added to the row
vs_time_summary_row$Total <- vs_time_summary_row$Total + elapsed_vs_time
}
vs_start_index <- NA # There is no start index for VS, waiting for the next
##### VS window timer summarization #####
# If there was a VS window activation before, it must stopped and added to the summary
if(window_start_index != -1) {
# Getting the row of previous window activation in VS
window_start_row <- extended_video_annotations[extended_video_annotations$PID == pid,][window_start_index,]
# Calculating the elapsed time in that particular window
elapsed_window_time <- row$Time - window_start_row$Time
# If the window was not C, S, N, P or was not NA (it is a numbered test case)
if(window_start_row$Window != "C" && window_start_row$Window != "S" && window_start_row$Window != "N" && window_start_row$Window != "P" && !is.na(window_start_row$Window)) {
# e.g., TC0 -> 2. column index
vs_time_summary_row[,as.numeric(window_start_row$Window)+2] <- vs_time_summary_row[,as.numeric(window_start_row$Window)+2] + elapsed_window_time
} else if(window_start_row$Window != "N" && !is.na(window_start_row$Window)) {
vs_time_summary_row[,window_start_row$Window] <- vs_time_summary_row[,window_start_row$Window] + elapsed_window_time
# Summarizing class time for given test
if(!is.na(window_start_row$Page) && window_start_row$Page != "H") {
if(window_start_row$Window == "C") {
cut_time_for_test_row[,as.numeric(window_start_row$Page)+2] <- cut_time_for_test_row[,as.numeric(window_start_row$Page)+2] + elapsed_window_time
}
if(window_start_row$Window == "S") {
sut_time_for_test_row[,as.numeric(window_start_row$Page)+2] <- sut_time_for_test_row[,as.numeric(window_start_row$Page)+2] + elapsed_window_time
}
if(window_start_row$Window == "P") {
put_time_for_test_row[,as.numeric(window_start_row$Page)+2] <- put_time_for_test_row[,as.numeric(window_start_row$Page)+2] + elapsed_window_time
}
}
}
}
window_start_index <- NA
} else if(row$Behavior == "VS activated") {
currently_active <- "VS"
# Start VS timer
vs_start_index <- i
window_start_index <- i
# stopping Portal timers
if(page_start_index != -1) {
page_start_row <- extended_video_annotations[extended_video_annotations$PID == pid,][page_start_index,]
elapsed_page_time <- row$Time - page_start_row$Time
if(page_start_row$Page != "H" && !is.na(page_start_row$Page)) {
# e.g., TC0 -> 2. column index
portal_time_summary_row[,as.numeric(page_start_row$Page)+2] <- portal_time_summary_row[,as.numeric(page_start_row$Page)+2] + elapsed_page_time
}
}
page_start_index <- NA
} else if(row$Behavior == "Changed page in portal") {
portal_page <- row$Modifier.1
if(page_start_index != -1) {
page_start_row <- extended_video_annotations[extended_video_annotations$PID == pid,][page_start_index,]
elapsed_page_time <- row$Time - page_start_row$Time
if(page_start_row$Page != "H" && !is.na(page_start_row$Page)) {
# e.g., TC0 -> 2. column index
portal_time_summary_row[,as.numeric(page_start_row$Page)+2] <- portal_time_summary_row[,as.numeric(page_start_row$Page)+2] + elapsed_page_time
}
}
page_start_index <- i
} else if(row$Behavior == "Changed window in VS") {
vs_window <- row$Modifier.1
if(window_start_index != -1) {
window_start_row <- extended_video_annotations[extended_video_annotations$PID == pid,][window_start_index,]
elapsed_window_time <- row$Time - window_start_row$Time
if(window_start_row$Window != "C" && window_start_row$Window != "S" && window_start_row$Window != "N" && window_start_row$Window != "P" && !is.na(window_start_row$Window)) {
# e.g., TC0 -> 2. column index
vs_time_summary_row[,as.numeric(window_start_row$Window)+2] <- vs_time_summary_row[,as.numeric(window_start_row$Window)+2] + elapsed_window_time
} else if(window_start_row$Window != "N" && !is.na(window_start_row$Window)) {
vs_time_summary_row[,window_start_row$Window] <- vs_time_summary_row[,window_start_row$Window] + elapsed_window_time
}
}
window_start_index <- i
} else if(row$Behavior == "Marked as OK") {
marked_case <- portal_page
marked_ok_summary_row[,as.numeric(marked_case)+2] <- marked_ok_summary_row[,as.numeric(marked_case)+2] +1
} else if(row$Behavior == "Marked as WRONG") {
marked_case <- portal_page
marked_wrong_summary_row[,as.numeric(marked_case)+2] <- marked_wrong_summary_row[,as.numeric(marked_case)+2] + 1
} else if(row$Behavior == "Running test") {
run_case <- vs_window
} else if(row$Behavior == "Submit") {
} else if(row$Behavior == "Other event") {
} else if(row$Behavior == "Debug test") {
if(row$Modifier.1 == "S") {
debug_start_index <- i
debug_case <- vs_window
} else if(row$Modifier.1 == "E") {
debug_case <- extended_video_annotations[extended_video_annotations$PID == pid,][debug_start_index,]$DebugCase
debug_time <- row$Time - annotations_for_participant[annotations_for_participant$pid == pid,][debug_start_index,]$Time
debug_start_index <- NA
}
} else if(row$Behavior == "Remove answer") {
marked_case <- row$Modifier.1
} else if(row$Behavior == "Missing test problem") {
if(row$Modifier.1 == "S") {
window_start_row <- extended_video_annotations[extended_video_annotations$PID == pid,][window_start_index,]
elapsed_window_time <- row$Time - window_start_row$Time
if(window_start_row$Window != "C" && window_start_row$Window != "S" && window_start_row$Window != "N" && window_start_row$Window != "P" && !is.na(window_start_row$Window)) {
# e.g., TC0 -> 2. column index
vs_time_summary_row[,as.numeric(window_start_row$Window)+2] <- vs_time_summary_row[,as.numeric(window_start_row$Window)+2] + elapsed_window_time
} else if(window_start_row$Window != "N" && !is.na(window_start_row$Window)) {
vs_time_summary_row[,window_start_row$Window] <- vs_time_summary_row[,window_start_row$Window] + elapsed_window_time
}
} else if(row$Modifier.1 == "E") {
window_start_index <- i
}
}
extended_row <- data.frame(PID=pid,Timestamp=row$Time,Behavior=row$Behavior,Modifier=row$Modifier.1,Active=currently_active,Page=portal_page,Window=vs_window,RunCase=run_case,DebugLength=debug_time,DebugCase=debug_case,stringsAsFactors = FALSE)
extended_video_annotations <- rbind(extended_video_annotations,extended_row)
}
portal_time_summary <- rbind(portal_time_summary, portal_time_summary_row)
vs_time_summary <- rbind(vs_time_summary, vs_time_summary_row)
marked_ok_summary <- rbind(marked_ok_summary, marked_ok_summary_row)
marked_wrong_summary <- rbind(marked_wrong_summary, marked_wrong_summary_row)
cut_time_for_test <- rbind(cut_time_for_test, cut_time_for_test_row)
sut_time_for_test <- rbind(sut_time_for_test, sut_time_for_test_row)
put_time_for_test <- rbind(put_time_for_test, put_time_for_test_row)
}
portal_time_summary$check_sums <- rowSums(portal_time_summary[,c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5")])
vs_time_summary$check_sums <- rowSums(vs_time_summary[,c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5","S","C","P")])
rm(cut_time_for_test_row,extended_row,marked_ok_summary_row, marked_wrong_summary_row, page_start_row,portal_time_summary_row,put_time_for_test_row,row,sut_time_for_test_row,vs_time_summary_row,window_start_row,currently_active,debug_case,debug_start_index,debug_time,elapsed_page_time,elapsed_vs_time,elapsed_window_time,i,marked_case,page_start_index,pid,portal_page,run_case,vs_start_index,vs_window,window_start_index)After this execution we have the following summaries:
- Portal time (
portal_time_summary): time spent on the portal for each test case - Visual Studio time (
vs_time_summary): time spent in each window of Visual Studio - OK marking summary (
marked_ok_summary): the number of times participants marked each case as OK - Wrong marking summary (
marked_wrong_summary): the number of times participants marked each case as Wrong - Class under test time for test {
cut_time_for_test): the time participants spent on analyzing the class for a given test - PUT time for test {
put_time_for_test}: the time participants spent on analyzing the parameterized unit test for a given test - SUT time for test {
sut_time_for_Test}: the time participants spent on analyzing the system under test (excluding the CUT and PUT)
4 Exploratory analysis
The following sections contain different aspects of data analysis.
4.1 Binary classification of answers
Source: http://www.damienfrancois.be/blog/files/modelperfcheatsheet.pdf, https://en.wikipedia.org/wiki/Evaluation_of_binary_classifiers, https://en.wikipedia.org/wiki/Sensitivity_and_specificity
Calculating: accuracy, misclassification rate, true positive rate (sensitivity), true negative rate (specificity), false positive rate (1-specificity)
participant_result_summary$Accuracy <- (participant_result_summary$TP + participant_result_summary$TN) / 15 # (TP+TN)/(TP+TN+FP+FN)
participant_result_summary$Misclassification <- (participant_result_summary$FP + participant_result_summary$FN) / 15
participant_result_summary$Sensitivity <- (participant_result_summary$TP)/(participant_result_summary$TP + participant_result_summary$FN) # (hit rate)
participant_result_summary$Specificity <- (participant_result_summary$TN)/(participant_result_summary$TN + participant_result_summary$FP)
participant_result_summary$FalsePositiveRate <- 1- participant_result_summary$Specificity # False positive rate
prs <- participant_result_summary
participant_result_summary$MCC <- ((prs$TP*prs$TN)-(prs$FP*prs$FN))/sqrt((prs$TP+prs$FP)*(prs$TP+prs$FN)*(prs$TN+prs$FP)*(prs$TN+prs$FN))
rm(prs)
print(median(1- participant_result_summary$Specificity)) # FP## [1] 0.25print(median(1- participant_result_summary$Sensitivity)) # FN## [1] 0.3333333The following box plots are showing different measures of binary classification separated by the projects.
ggplot(participant_result_summary, aes(x=factor(Project), y=Accuracy)) + geom_boxplot() + ylim(0,1) # Accuracy
ggplot(participant_result_summary, aes(x=factor(Project), y=Misclassification)) + geom_boxplot() + ylim(0,1) # Misclassification
ggplot(participant_result_summary, aes(x=factor(Project), y=FalsePositiveRate)) + geom_boxplot() + ylim(0,1) # False positive rate
# IN PAPER
#pdf(file="boxplot-sensitivity.pdf",width=3.5,height=2.5)
ggplot(participant_result_summary, aes(x=factor(Project), y=Sensitivity)) + geom_boxplot() + xlab("Project") + ylab("TPR") + guides(fill=FALSE) + theme_hc() + scale_y_continuous(limits=c(0,1), breaks=seq(0,1,0.1))# Sensitivity (TPR)
#dev.off()
# IN PAPER
#pdf(file="boxplot-tnr.pdf",width=3.5,height=2.5)
ggplot(participant_result_summary, aes(x=factor(Project), y=Specificity)) + geom_boxplot() + xlab("Project") + ylab("TNR")+ guides(fill=FALSE) + theme_hc() + scale_y_continuous(limits=c(0,1), breaks=seq(0,1,0.1)) # Specificity (true negative rate)
#dev.off()
# IN PAPER
#pdf(file="boxplot-matthews.pdf",width=3.5,height=2.5)
ggplot(participant_result_summary, aes(x=factor(Project), y=MCC)) + geom_boxplot() + xlab("Project") + ylab("MCC") + guides(fill=FALSE) + theme_hc() + scale_y_continuous(limits=c(-1,1), breaks=seq(-1,1,0.2)) # Matthews correlation coefficient
#dev.off()The following plots summarizes the binary classification for the test cases that are faulty.
palette <- c("#cb181d", "#238b45")
participant_results_nbitcoin_faulty <- participant_results[(participant_results$Results.Id == 5 | participant_results$Results.Id == 7 | participant_results$Results.Id == 10) & (as.numeric(participant_results$pid) < 41),]
participant_results_mathnet_faulty <- participant_results[(participant_results$Results.Id == 2 | participant_results$Results.Id == 4 | participant_results$Results.Id == 11) & as.numeric(participant_results$pid) > 40,]
participant_results_nbitcoin_faulty$Project <- rep("NBitcoin",nrow(participant_results_nbitcoin_faulty))
participant_results_mathnet_faulty$Project <- rep("MathNet",nrow(participant_results_mathnet_faulty))
participant_results_on_faulty_tests <- rbind(participant_results_nbitcoin_faulty, participant_results_mathnet_faulty)
participant_results_on_faulty_tests <- participant_results_on_faulty_tests %>%
group_by(Project, Check) %>%
summarise(count=n()) %>%
mutate(perc=count/sum(count))
ggplot(participant_results_on_faulty_tests, aes(x=Project,y=perc*100,fill=factor(Check))) + scale_fill_manual(values = palette) + geom_bar(stat="identity") + ylab("Percentage") + labs(fill="Result") + theme_hc()
The following plot summarizes the results for the complex test cases (both the faulties and not faulties).
palette <- c("#cb181d", "#238b45")
participant_results_nbitcoin_complex <- participant_results[(participant_results$Results.Id == 7 | participant_results$Results.Id == 10 | participant_results$Results.Id == 13 | participant_results$Results.Id == 14) & (as.numeric(participant_results$pid) < 41),]
participant_results_mathnet_complex <- participant_results[(participant_results$Results.Id == 4 | participant_results$Results.Id == 10 | participant_results$Results.Id == 11 | participant_results$Results.Id == 14) & as.numeric(participant_results$pid) > 40,]
participant_results_nbitcoin_complex$Project <- rep("NBitcoin",nrow(participant_results_nbitcoin_complex))
participant_results_mathnet_complex$Project <- rep("MathNet",nrow(participant_results_mathnet_complex))
participant_results_on_complex_tests <- rbind(participant_results_nbitcoin_complex, participant_results_mathnet_complex)
# Checking not faulty complex tests
complex_not_faulty <- participant_results_on_complex_tests[(participant_results_on_complex_tests$Project == "NBitcoin" & (participant_results_on_complex_tests$Results.Id == 13 | participant_results_on_complex_tests$Results.Id == 14)) |(participant_results_on_complex_tests$Project == "MathNet" & (participant_results_on_complex_tests$Results.Id == 10 | participant_results_on_complex_tests$Results.Id == 14)),]
complex_not_faulty_summarized <- complex_not_faulty %>%
group_by(Project, Check) %>%
summarise(count=n()) %>%
mutate(perc=count/sum(count))
ggplot(complex_not_faulty_summarized, aes(x=Project,y=perc*100,fill=factor(Check))) + geom_bar(stat="identity") + theme_hc() + scale_fill_manual(values = palette) + labs(fill="Result") + ylab("Percentage")
# Checking faulty complex tests
complex_faulty <- participant_results_on_complex_tests[(participant_results_on_complex_tests$Project == "NBitcoin" & (participant_results_on_complex_tests$Results.Id == 7 | participant_results_on_complex_tests$Results.Id == 10)) |(participant_results_on_complex_tests$Project == "MathNet" & (participant_results_on_complex_tests$Results.Id == 4 | participant_results_on_complex_tests$Results.Id == 11)),]
complex_faulty_summarized <- complex_faulty %>%
group_by(Project, Check) %>%
summarise(count=n()) %>%
mutate(perc=count/sum(count))
ggplot(complex_faulty_summarized, aes(x=Project,y=perc*100,fill=factor(Check))) + geom_bar(stat="identity") + theme_hc() + scale_fill_manual(values = palette) + labs(fill="Result") + ylab("Percentage")
The following coloured table contains the evaluation of all participant answers for each test case.
cb_palette <- c("#680008", "#e22828", "#2cba5a", "#01703e")
# IN PAPER
#pdf(file="nbitcoin-tile-map.pdf",width=6.5,height = 3.5)
ggplot(participant_results[participant_results$pid < 41,], aes(x=factor(pid), y=reorder(factor(TestId),Results.Id), fill=as.factor(Check))) +
geom_tile(alpha=0.8,width=.9, height=.9) +theme(axis.line = element_line(colour = "black"),
panel.grid.major = element_blank(),
panel.grid.minor = element_blank(),
panel.border = element_blank(),
panel.background = element_blank()) + scale_fill_manual(values=cb_palette) + xlab("Participant ID") + ylab("Test ID") + labs(fill="Result")
#dev.off()
# IN PAPER
#pdf(file="mathnet-tile-map.pdf",width=5.5,height = 3.5)
ggplot(participant_results[participant_results$pid > 40,], aes(x=factor(pid), y=reorder(factor(TestId),Results.Id), fill=as.factor(Check))) +
geom_tile(alpha=0.8,width=.9, height=.9) +theme(axis.line = element_line(colour = "black"),
panel.grid.major = element_blank(),
panel.grid.minor = element_blank(),
panel.border = element_blank(),
panel.background = element_blank()) + scale_fill_manual(values=cb_palette) + xlab("Participant ID") + ylab("Test ID") + labs(fill="Result")
#dev.off()4.2 Time spent in Visual Studio and Portal
In the followings, the time spent in Visual Studio and Portal is presented for each possible window/page (T0-14, S, C and P).
library(ggthemes)
project_column <- data.frame()
for(i in 1:nrow(vs_time_summary)) {
if(as.numeric(vs_time_summary[i,]$PID) > 40) {
project_column[i,"Project"] <- "MathNet"
} else {
project_column[i,"Project"] <- "NBitcoin"
}
}
vs_time_summary_with_projects <- cbind(vs_time_summary[order(vs_time_summary$PID),],project_column)
project_column <- data.frame()
for(i in 1:nrow(portal_time_summary)) {
if(as.numeric(portal_time_summary[i,]$PID) > 40) {
project_column[i,"Project"] <- "MathNet"
} else {
project_column[i,"Project"] <- "NBitcoin"
}
}
portal_time_summary_with_projects <- cbind(portal_time_summary[order(portal_time_summary$PID),],project_column)
# Tests individually
time_spent_tests_vs <- melt(vs_time_summary_with_projects,id.vars=c("PID","Project"), measure.vars=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"),variable.name="Window",value.name="Time")
time_spent_tests_vs <- time_spent_tests_vs[order(time_spent_tests_vs$PID,time_spent_tests_vs$Window),]
time_spent_tests_portal <- melt(portal_time_summary_with_projects,id.vars=c("PID","Project"), measure.vars=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"),variable.name="Page",value.name="Time")
time_spent_tests_portal <- time_spent_tests_portal[order(time_spent_tests_portal$PID,time_spent_tests_portal$Page),]
time_spent_tests <- time_spent_tests_vs
time_spent_tests$Time = time_spent_tests$Time + time_spent_tests_portal$Time
time_spent_others <- melt(vs_time_summary_with_projects,id.vars=c("PID","Project"), measure.vars=c("S","C","P"),variable.name="Window",value.name="Time")
time_spent_tests_nbitcoin <- time_spent_tests[time_spent_tests$Project == "NBitcoin",]
time_spent_tests_nbitcoin$TestCategory <- ifelse(time_spent_tests_nbitcoin$Window == "T5.2","Faulty", ifelse(time_spent_tests_nbitcoin$Window == "T7.3" | time_spent_tests_nbitcoin$Window == "T10.4","Faulty", ifelse(time_spent_tests_nbitcoin$Window == "T13.5" | time_spent_tests_nbitcoin$Window == "T14.5", "Not faulty", "Not faulty")))
time_spent_tests_mathnet <- time_spent_tests[time_spent_tests$Project == "MathNet",]
time_spent_tests_mathnet$TestCategory <- ifelse(time_spent_tests_mathnet$Window == "T2.1","Faulty", ifelse(time_spent_tests_mathnet$Window == "T4.2" | time_spent_tests_mathnet$Window == "T11.4","Faulty", ifelse(time_spent_tests_mathnet$Window == "T10.4" | time_spent_tests_mathnet$Window == "T14.5", "Not faulty", "Not faulty")))
# Tests grouped by methods
vs_time_summary_with_projects <- vs_time_summary_with_projects[order(vs_time_summary_with_projects$PID),]
portal_time_summary_with_projects <- portal_time_summary_with_projects[order(portal_time_summary_with_projects$PID),]
pts <- portal_time_summary_with_projects
vts <- vs_time_summary_with_projects
time_spent_tests_grouped <- data.frame(M1=c(1:54),M2=c(1:54),M3=c(1:54),M4=c(1:54),M5=c(1:54))
time_spent_tests_grouped$M1 <- (pts$T0.1+pts$T1.1+pts$T2.1)+(vts$T0.1+vts$T1.1+vts$T2.1)
time_spent_tests_grouped$M2 <- (pts$T3.2+pts$T4.2+pts$T5.2)+(vts$T3.2+vts$T4.2+vts$T5.2)
time_spent_tests_grouped$M3 <- (pts$T6.3+pts$T7.3+pts$T8.3)+(vts$T6.3+vts$T7.3+vts$T8.3)
time_spent_tests_grouped$M4 <- (pts$T9.4+pts$T10.4+pts$T11.4)+(vts$T9.4+vts$T10.4+vts$T11.4)
time_spent_tests_grouped$M5 <- (pts$T12.5+pts$T13.5+pts$T14.5)+(vts$T12.5+vts$T13.5+vts$T14.5)
# Other windows - SUT, CUT and PUT
time_spent_others_nbitcoin <- time_spent_others[time_spent_others$Project == "NBitcoin",]
time_spent_others_mathnet <- time_spent_others[time_spent_others$Project == "MathNet",]
#pdf(file="nbitcoin-time-spent-tests.pdf",width=8,height = 3.5)
ggplot(time_spent_tests_nbitcoin, aes(x=factor(Window), y=Time, fill=factor(TestCategory))) + geom_boxplot() + scale_y_continuous(limits=c(0,500),oob = rescale_none) + theme_hc() + scale_fill_manual(values=c("#BCBABE","#F1F1F2")) + ylab("Time spent [s]") + xlab("Test") + labs(fill="Category") # NBitcoin tests
#dev.off()
#pdf(file="mathnet-time-spent-tests.pdf",width=8,height = 3.5)
ggplot(time_spent_tests_mathnet, aes(x=factor(Window), y=Time, fill=factor(TestCategory))) + geom_boxplot() + scale_y_continuous(limits=c(0,500),oob = rescale_none) + theme_hc() + scale_fill_manual(values=c("#BCBABE","#F1F1F2")) + ylab("Time spent [s]") + xlab("Test") + labs(fill="Category") # MathNet tests
#dev.off()
#pdf(file="nbitcoin-time-spent-others.pdf",width=5,height = 3.5)
ggplot(time_spent_others_nbitcoin, aes(x=factor(Window), y=Time, fill=factor(Window))) + geom_boxplot(fatten=4) + scale_y_continuous(limits=c(0,1200),oob = rescale_none) + theme_hc() + scale_fill_manual(values=c("#A4CABC","#EAB364","#D09683")) + guides(fill=FALSE) + ylab("Time spent [s]") + xlab("Window") # NBitcoin others
#dev.off()
#pdf(file="mathnet-time-spent-others.pdf",width=5,height = 3.5)
ggplot(time_spent_others_mathnet, aes(x=factor(Window), y=Time, fill=factor(Window))) + geom_boxplot(fatten=4)+ theme_hc() + scale_fill_manual(values=c("#A4CABC","#EAB364","#D09683")) + scale_y_continuous(limits=c(0,1200),oob = rescale_none) + guides(fill=FALSE) + ylab("Time spent [s]") + xlab("Window") # MathNet others
#dev.off()
# Joining the dataset
summarized_test_times_portal <- time_spent_tests_portal %>% group_by(PID, Project) %>% summarize(Time=sum(Time)) %>% bind_cols(data.frame(Location=rep("Portal",54)))
summarized_test_time_vs <- time_spent_tests_vs %>% group_by(PID, Project) %>% summarize(Time=sum(Time)) %>% bind_cols(data.frame(Location=rep("VS",54)))
names(time_spent_others)[3] <- "Location"
full_summarized_times <- summarized_test_times_portal %>% bind_rows(summarized_test_time_vs) %>% bind_rows(time_spent_others)## Warning in rbind_all(x, .id): Unequal factor levels: coercing to characterfull_summarized_times$Location <- factor(full_summarized_times$Location, levels=c("Portal", "VS", "C", "S", "P"))
levels(full_summarized_times$Location) <- list("Portal"="Portal", "Test code"="VS", "CUT" = "C", "SUT"="S","PUT"="P")
# IN PAPER
#pdf(file="full-time-spent-nbitcoin.pdf",width=4,height = 2.5)
ggplot(data=filter(full_summarized_times, Project=="NBitcoin"), aes(x=Location,y=Time)) +geom_boxplot() + theme_hc() + scale_y_continuous(limits=c(0,1600), breaks=seq(0,1500,250)) + ylab("Time [s]")## Warning: Removed 7 rows containing non-finite values (stat_boxplot).
#dev.off()
# IN PAPER
#pdf(file="full-time-spent-mathnet.pdf",width=4,height = 2.5)
ggplot(data=filter(full_summarized_times, Project=="MathNet"), aes(x=Location,y=Time)) +geom_boxplot() + theme_hc()+ scale_y_continuous(limits=c(0,1600), breaks=seq(0,1500,250)) + ylab("Time [s]")
#dev.off()4.3 Time spent everywhere for test cases
In the followings, the average time spent everywhere for each case opened in the portal.
cut_times <- melt(cut_time_for_test,id.vars = c("PID"), measure.vars=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), variable.name = "Test",value.name="Time")
all_times_for_tests <- cut_times
put_times <- melt(put_time_for_test, id.vars = c("PID"), measure.vars=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), variable.name = "Test",value.name="Time")
all_times_for_tests <- rbind(all_times_for_tests, put_times)
sut_times <- melt(sut_time_for_test, id.vars = c("PID"), measure.vars=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"), variable.name = "Test",value.name="Time")
all_times_for_tests <- rbind(all_times_for_tests, sut_times)
vs_time_for_tests <- subset(time_spent_tests_vs, select=c(PID,Window,Time))
names(vs_time_for_tests) <- c("PID","Test","Time")
portal_time_for_tests <- subset(time_spent_tests_portal, select=c(PID,Page,Time))
names(portal_time_for_tests) <- c("PID","Test","Time")
portal_plus_vs_time_for_tests <- dplyr::left_join(vs_time_for_tests,portal_time_for_tests,by=c('PID','Test'))
portal_plus_vs_time_for_tests$Time <- portal_plus_vs_time_for_tests$Time.x + portal_plus_vs_time_for_tests$Time.y
portal_plus_vs_time_for_tests <- subset(portal_plus_vs_time_for_tests, select=c(PID,Test,Time))
all_times_for_tests_with_sum <- rbind(all_times_for_tests, portal_plus_vs_time_for_tests)
all_times_for_tests <- rbind(all_times_for_tests, vs_time_for_tests)
all_times_for_tests <- rbind(all_times_for_tests, portal_time_for_tests)
all_times_for_tests <- cbind(all_times_for_tests, c(rep("CUT",nrow(cut_times)),rep("PUT",nrow(put_times)),rep("SUT",nrow(sut_times)),rep("VS",nrow(vs_time_for_tests)),rep("Portal",nrow(portal_time_for_tests))))
names(all_times_for_tests)[4] <- "Location"
all_times_for_tests_with_sum <- cbind(all_times_for_tests_with_sum, c(rep("CUT",nrow(cut_times)),rep("PUT",nrow(put_times)),rep("SUT",nrow(sut_times)),rep("Test",nrow(portal_plus_vs_time_for_tests))))
names(all_times_for_tests_with_sum)[4] <- "Location"
all_times_for_tests <- cbind(all_times_for_tests, c(rep(NA,nrow(all_times_for_tests))))
names(all_times_for_tests)[5] <- "Project"
all_times_for_tests$Project[all_times_for_tests$PID > 40] <- "MathNet"
all_times_for_tests$Project[all_times_for_tests$PID < 41] <- "NBitcoin"
all_times_for_tests_with_sum <- cbind(all_times_for_tests_with_sum, c(rep(NA,nrow(all_times_for_tests_with_sum))))
names(all_times_for_tests_with_sum)[5] <- "Project"
all_times_for_tests_with_sum$Project[all_times_for_tests_with_sum$PID > 40] <- "MathNet"
all_times_for_tests_with_sum$Project[all_times_for_tests_with_sum$PID < 41] <- "NBitcoin"
# Participants 55 and 59 had missing videos in the beginning caused by unexpected shutdowns, thus removing their video timing data
all_times_for_tests <- dplyr::filter(all_times_for_tests, !((PID == 59 & (Test == "T0.1" | Test == "T1.1")) | (PID == 55 & (Test == "T0.1" | Test == "T1.1")) ))
jj <- inner_join(x=all_times_for_tests, y=result_answers, by=c("PID" = "pid", "Test" = "TestId", "Project" = "Project"))## Warning in inner_join_impl(x, y, by$x, by$y): joining factors with
## different levels, coercing to character vectortime_summary <- jj %>%
group_by(PID,Test,Project) %>%
summarise(sum=sum(Time)) %>%
group_by(Project) %>%
summarize(mean=mean(sum), median=median(sum), sd=sd(sum), min=min(sum), max=max(sum))
time_summary_plot <- jj %>%
group_by(PID,Test,Project,Method) %>%
summarise(Time=sum(Time)) %>%
mutate(Faulty=ifelse( (Project=="NBitcoin" && (Test=="T5.2" || Test=="T7.3" || Test=="T10.4") ) || (Project=="MathNet" && (Test=="T2.1" || Test=="T4.2" || Test=="T11.4") ) ,"YES","NO"))
levels(time_summary_plot$Method)[levels(time_summary_plot$Method)=="CombinationsWithRepetition"] <- "CWithRepetition"
levels(time_summary_plot$Method)[levels(time_summary_plot$Method)=="VariationsWithRepetition"] <- "VWithRepetition"
time_summary_plot$Test <- factor(time_summary_plot$Test, levels=c("T0.1","T1.1","T2.1","T3.2","T4.2","T5.2","T6.3","T7.3","T8.3","T9.4","T10.4","T11.4","T12.5","T13.5","T14.5"))
palette <- c("#FFFFFF","#ff821c")
# IN PAPER
#pdf(file="mathnet-time-spent-tests.pdf",width=7,height = 3.5)
ggplot(data=filter(time_summary_plot, Project == "MathNet"), aes(x=Test,y=Time,fill=factor(Faulty)) ) + geom_boxplot() + facet_grid(~Method, scales="free_x") + theme_hc() + ylab("Time [s]") + xlab("Test ID") + scale_fill_manual(values=palette, guide=FALSE) + scale_y_continuous(limits=c(0,900), breaks=seq(0,900,100))
#dev.off()
# IN PAPER
#pdf(file="nbitcoin-time-spent-tests.pdf",width=7,height = 3.5)
ggplot(data=filter(time_summary_plot, Project == "NBitcoin"), aes(x=Test,y=Time,fill=factor(Faulty))) + geom_boxplot() + facet_grid(~Method, scales="free_x") + theme_hc() + ylab("Time [s]") + xlab("Test ID") +scale_fill_manual(values=palette, guide=FALSE) + scale_y_continuous(limits=c(0,900), breaks=seq(0,900,100))
#dev.off()
qplot(data=filter(all_times_for_tests_with_sum, Project == "MathNet"), Time, geom="histogram") + facet_grid(Location~Test)## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
qplot(data=filter(all_times_for_tests_with_sum, Project == "NBitcoin"), Time, geom="histogram") + facet_grid(Location~Test)## `stat_bin()` using `bins = 30`. Pick better value with `binwidth`.
ggplot(data=filter(all_times_for_tests,Project == "MathNet")) +
geom_bar(aes(x = factor(PID), y = Time, fill = Location), stat = "identity") +
facet_wrap(~Test) + theme_bw()
ggplot(data=filter(all_times_for_tests,Project == "NBitcoin")) +
geom_bar(aes(x = factor(PID), y = Time, fill = Location), stat = "identity") +
facet_wrap(~Test) + theme_bw()
ggplot(data=filter(all_times_for_tests_with_sum,Project == "MathNet")) +
geom_bar(aes(x = factor(PID), y = Time, fill = Location), stat = "identity") +
facet_grid(Test~.)
print(time_summary)## Source: local data frame [2 x 6]
##
## Project mean median sd min max
## (chr) (dbl) (dbl) (dbl) (dbl) (dbl)
## 1 MathNet 113.9035 86.2370 87.49944 6.068 555.028
## 2 NBitcoin 146.8705 117.8215 121.89108 1.823 818.1204.4 Statistical and correlation analysis
The following section contains the statistical analysis we performed in order to capture the key factors found in the data.
4.4.1 Relation between Project and Specificity/Sensitivity
# Difference between projects - Sensitivity
sensitivity_nbitcoin <- participant_result_summary$Sensitivity[participant_result_summary$Project == "NBitcoin"]
sensitivity_mathnet <- participant_result_summary$Sensitivity[participant_result_summary$Project == "MathNet"]
sensitivity_sd <- mean(c(sd(sensitivity_nbitcoin),sd(sensitivity_mathnet))) # 0.309
sensitivity_A <- measureA(sensitivity_nbitcoin, sensitivity_mathnet) # 0.651 -> medium-large difference
sensitivity_w <- wilcox.test(sensitivity_nbitcoin, sensitivity_mathnet, exact=FALSE) # p=0.04514 < 0.05 -> reject (samples are from different distribution)
print(mean(participant_result_summary$Sensitivity[participant_result_summary$Project == "NBitcoin"]))## [1] 0.7666667print(mean(participant_result_summary$Sensitivity[participant_result_summary$Project == "MathNet"]))## [1] 0.5972222print(sensitivity_sd)## [1] 0.3089999print(sensitivity_A)## [1] 0.6513889print(sensitivity_w)##
## Wilcoxon rank sum test with continuity correction
##
## data: sensitivity_nbitcoin and sensitivity_mathnet
## W = 469, p-value = 0.04514
## alternative hypothesis: true location shift is not equal to 0# Difference between projects - Specificity
specificity_nbitcoin <- participant_result_summary$Specificity[participant_result_summary$Project == "NBitcoin"]
specificity_mathnet <- participant_result_summary$Specificity[participant_result_summary$Project == "MathNet"]
specificity_sd <- mean(c(sd(specificity_nbitcoin),sd(specificity_mathnet))) # 0.130
specificity_A <- measureA(specificity_nbitcoin, specificity_mathnet) # 0.5569 -> small difference
specificity_w <- wilcox.test(specificity_nbitcoin, specificity_mathnet, exact=FALSE) # p=0.473 > 0.05 -> not rejecting (samples are likely from same distribution)
print(mean(participant_result_summary$Specificity[participant_result_summary$Project == "NBitcoin"]))## [1] 0.8print(mean(participant_result_summary$Specificity[participant_result_summary$Project == "MathNet"]))## [1] 0.7708333print(specificity_sd)## [1] 0.1301958print(specificity_A)## [1] 0.5569444print(specificity_w)##
## Wilcoxon rank sum test with continuity correction
##
## data: specificity_nbitcoin and specificity_mathnet
## W = 401, p-value = 0.4733
## alternative hypothesis: true location shift is not equal to 04.4.2 Relation between programming experience and Specificity/Sensitivity
Note: we excluded participants who selected the “None” or the “One year or less options” (3 participants) to have balanced groups.
# SENSITIVITY
# Ranking programming experience
result_bg_answers$Prog[result_bg_answers$Prog == '5-10'] <- 5
result_bg_answers$Prog[result_bg_answers$Prog == 'none'] <- 0
participant_result_summary_with_bg_answers <- plyr::join(participant_result_summary, result_bg_answers, by=c("PID"))
# Calculating Kruskal-Wallis statistics for the groups
g1_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Prog == 2]
g2_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Prog == 3]
g3_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Prog == 4]
g4_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Prog == 5]
kruskal_test_data_sensitivity <- list(
g1=g1_,
g2=g2_,
g3=g3_,
g4=g4_
)
print(mean(g1_))## [1] 0.5925926print(mean(g2_))## [1] 0.5416667print(mean(g3_))## [1] 0.8070175print(mean(g4_))## [1] 0.6666667print(mean(c(sd(g1_),sd(g2_),sd(g3_),sd(g4_))))## [1] 0.3162253kruskal_result_sensitivity <- kruskal.test(kruskal_test_data_sensitivity) # 0.1223 > 0.05 -> not rejecting (groups are likely equal, experience less likely influence)
print(kruskal_result_sensitivity)##
## Kruskal-Wallis rank sum test
##
## data: kruskal_test_data_sensitivity
## Kruskal-Wallis chi-squared = 5.7894, df = 3, p-value = 0.1223# SPECIFICITY
# Calculating Kruskal-Wallis statistics for the groups
g1_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Prog == 2]
g2_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Prog == 3]
g3_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Prog == 4]
g4_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Prog == 5]
kruskal_test_data_specificity <- list(
g1=g1_,
g2=g2_,
g3=g3_,
g4=g4_
)
print(mean(g1_))## [1] 0.7037037print(mean(g2_))## [1] 0.8020833print(mean(g3_))## [1] 0.8157895print(mean(g4_))## [1] 0.8111111print(mean(c(sd(g1_),sd(g2_),sd(g3_),sd(g4_))))## [1] 0.1228748kruskal_result_specificity <- kruskal.test(kruskal_test_data_specificity) # 0.182 > 0.05 -> not reject (groups are likely equal, experience less likely influence)
print(kruskal_result_specificity)##
## Kruskal-Wallis rank sum test
##
## data: kruskal_test_data_specificity
## Kruskal-Wallis chi-squared = 4.8695, df = 3, p-value = 0.18164.4.3 Relation between industrial experience and Specificity/Sensitivity
# SENSITIVITY
# Ranking industrial experience
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == 'none'] <- 0
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '6'] <- 1
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '7-12'] <- 2
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '1-2'] <- 3
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '3-5'] <- 4
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '6-10'] <- 5
participant_result_summary_with_bg_answers$Work[participant_result_summary_with_bg_answers$Work == '10'] <- 6
# Calculating Kruskal-Wallis statistics for the groups
g1_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Work == 0]
g2_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Work == 1]
g3_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Work == 2]
g4_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Work == 3]
g5_ <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$Work == 4]
kruskal_test_data_sensitivity <- list(
g1=g1_,
g2=g2_,
g3=g3_,
g4=g4_,
g5=g5_
)
print(mean(g1_))## [1] 0.5333333print(mean(g2_))## [1] 0.7407407print(mean(g3_))## [1] 0.8148148print(mean(g4_))## [1] 0.5740741print(mean(g5_))## [1] 0.9166667print(mean(c(sd(g1_),sd(g2_),sd(g3_),sd(g4_),sd(g5_))))## [1] 0.2715446kruskal_result_sensitivity <- kruskal.test(kruskal_test_data_sensitivity) # 0.109 > 0.05 -> not rejecting (groups are likely equal, industrial experience less likely influence)
print(kruskal_result_sensitivity)##
## Kruskal-Wallis rank sum test
##
## data: kruskal_test_data_sensitivity
## Kruskal-Wallis chi-squared = 7.5415, df = 4, p-value = 0.1099# SPECIFICITY
g1_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 0]
g2_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1]
g3_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 2]
g4_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 3]
g5_ <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 4]
# Calculating Kruskal-Wallis statistics for the groups
kruskal_test_data_specificity <- list(
g1=g1_,
g2=g2_,
g3=g3_,
g4=g4_,
g5=g5_
)
print(mean(g1_))## [1] 0.85print(mean(g2_))## [1] 0.8425926print(mean(g3_))## [1] 0.7222222print(mean(g4_))## [1] 0.7314815print(mean(g5_))## [1] 0.8541667print(mean(c(sd(g1_),sd(g2_),sd(g3_),sd(g4_),sd(g5_))))## [1] 0.1226816kruskal_result_specificity <- kruskal.test(kruskal_test_data_specificity) # 0.02638 < 0.05 -> reject (groups not equal, industrial experience may influence)
print(kruskal_result_specificity)##
## Kruskal-Wallis rank sum test
##
## data: kruskal_test_data_specificity
## Kruskal-Wallis chi-squared = 11.016, df = 4, p-value = 0.02638# Comparing less than 6 months experience and 1-2 years
industrial_exp_1_3_A <- measureA(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1],participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 3])
industrial_exp_1_3_w <- wilcox.test(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1],participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 3], exact = FALSE)
industrial_exp_1_3_sd <- mean(c(sd(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1]),sd(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 3])))
print(mean(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1]))## [1] 0.8425926print(mean(participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 3]))## [1] 0.7314815print(mean(participant_result_summary$Specificity[participant_result_summary$Project == "MathNet"]))## [1] 0.7708333print(industrial_exp_1_3_sd) # 0.116## [1] 0.1168131print(industrial_exp_1_3_A) # 0.75 -> large diff## [1] 0.7515432print(industrial_exp_1_3_w) # p=0.0091 < 0.05 -> reject (samples are from different distribution)##
## Wilcoxon rank sum test with continuity correction
##
## data: participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == and participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$Work == 1] and 3]
## W = 243.5, p-value = 0.009109
## alternative hypothesis: true location shift is not equal to 04.4.4 Relation between Unit testing habits and Specificity/Sensitivity
Note: Excluding 6 participants who have chosen ‘Never’ or ‘Often’ due to their small numbers. Thus, two groups can be compared with Mann-Whitney U-Test.
# Ranking programming experience
participant_result_summary_with_bg_answers$UT[participant_result_summary_with_bg_answers$UT == 'never'] <- 0
participant_result_summary_with_bg_answers$UT[participant_result_summary_with_bg_answers$UT == 'rarely'] <- 1
participant_result_summary_with_bg_answers$UT[participant_result_summary_with_bg_answers$UT == 'occasionally'] <- 2
participant_result_summary_with_bg_answers$UT[participant_result_summary_with_bg_answers$UT == 'often'] <- 3
# No participant gave 'Always' as an answer
# SENSITIVITY
sensitivity_ut_rarely <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$UT == 1]
sensitivity_ut_occasionally <- participant_result_summary_with_bg_answers$Sensitivity[participant_result_summary_with_bg_answers$UT == 2]
sensitivity_sd <- mean(c(sd(sensitivity_ut_rarely),sd(sensitivity_ut_occasionally))) # 0.327
sensitivity_A <- measureA(sensitivity_ut_rarely, sensitivity_ut_occasionally) # 0.422 -> small diff
sensitivity_w <- wilcox.test(sensitivity_ut_rarely, sensitivity_ut_occasionally, exact=FALSE) # p=0.371 > 0.05 -> not rejecting (samples are likely from same distribution)
print(sensitivity_sd)## [1] 0.3276099print(sensitivity_A)## [1] 0.4222222print(sensitivity_w)##
## Wilcoxon rank sum test with continuity correction
##
## data: sensitivity_ut_rarely and sensitivity_ut_occasionally
## W = 209, p-value = 0.3714
## alternative hypothesis: true location shift is not equal to 0# SPECIFICITY
# Difference between projects - Specificity
specificity_ut_rarely <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$UT == 1]
specificity_ut_occasionally <- participant_result_summary_with_bg_answers$Specificity[participant_result_summary_with_bg_answers$UT == 2]
specificity_sd <- mean(c(sd(specificity_ut_rarely),sd(specificity_ut_occasionally))) # 0.309
specificity_A <- measureA(specificity_ut_rarely, specificity_ut_occasionally) # 0.498 -> no difference
specificity_w <- wilcox.test(specificity_ut_rarely, specificity_ut_occasionally, exact=FALSE) # p=1 > 0.05 -> not rejecting (samples are likely from same distribution)
print(specificity_sd)## [1] 0.1298307print(specificity_A)## [1] 0.4989899print(specificity_w)##
## Wilcoxon rank sum test with continuity correction
##
## data: specificity_ut_rarely and specificity_ut_occasionally
## W = 247, p-value = 1
## alternative hypothesis: true location shift is not equal to 04.5 Other analyses
4.5.1 Background questionnaire test score
participant_result_summary_with_bg_answers$TestScore <- c(rep(0,54))
for(pid in unique(participant_result_summary_with_bg_answers$PID)) {
row_for_pid <- filter(participant_result_summary_with_bg_answers, PID == pid)
a1 <- ifelse(row_for_pid$A1 == "true",1,0)
a2 <- ifelse(row_for_pid$A2 == "true",1,0)
a3 <- ifelse(row_for_pid$A3 == "true",1,0)
a4 <- ifelse(row_for_pid$A4 == "true",1,0)
a5 <- ifelse(row_for_pid$A5 == "false",1,0)
participant_result_summary_with_bg_answers$TestScore[participant_result_summary_with_bg_answers$PID == pid] <- a1 + a2 + a3 + a4 + a5
}4.5.2 Likert chart for exit questionnaire
Note: does not work with R markdown, use it separately.
# result_exit_answers_corrected <- result_exit_answers[result_exit_answers$agreement2 != "",]
# options = c("Fully agree","Partially agree","Neither agree nor disagree","Partially disagree","Fully disagree")
# questions = c ("A: I had enough time to understand the class under test.",
# "B: I had enough time to review the generated tests.",
# "C: It was easy to understand the class under test.",
# "D: It was easy to understand the generated tests.",
# "E: I am certain I chose the right answers",
# "F: Generated tests are difficult to read.",
# "G: Generated tests are too long to understand.",
# "H: Generated tests are too short to exercise useful behavior.",
# "I: Generated tests had too many assertions.",
# "J: Generated tests had meaningful assertions.",
# "K: It was easy to select test cases with wrong assertions."
# )
#
# data2 <- subset(result_exit_answers_corrected, select = c(agreement1,agreement2,agreement3,agreement4,agreement5,agreement6,agreement7, agreement8, agreement9, agreement10, agreement11))
# names(data2) <- questions
#
#
# for(i in 1:length(questions)){
# data[data[,i] == 'fagree',i] <- "Fully agree"
# data[data[,i] == 'pagree',i] <- "Partially agree"
# data[data[,i] == 'neither',i] <- "Neither agree nor disagree"
# data[data[,i] == 'pdisagree',i] <- "Partially disagree"
# data[data[,i] == 'fdisagree',i] <- "Fully disagree"
#
# data[,i] <- factor(data[,i], levels = options)
# }
# Likert-figure can only be printed to a PDF file, comment out the lines below to generate.
#ldt <- likert(subset(data, select=questions), nlevels = length(options))
#pdf(file="likert.pdf", width=15)
#plot(ldt, ordered=FALSE) + theme(text = element_text(size=15))
#dev.off()