sink("output-z-test.txt")

### Iteration 1

print("Iteration 1")

## looking for significant difference among the models

print("zero-shot")

# zero-shot

codegemma <- 11
llama3 <- 41
mistral <- 3


print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


# few-shot 1
print("few-shot 1")

codegemma <- 26
llama3 <- 14
mistral <- 24

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


# few-shot 2

print("few-shot 2")

codegemma <- 14
llama3 <- 53
mistral <- 7

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))

# few-shot 3

print("few-shot 3")

codegemma <- 7
llama3 <- 15
mistral <- 12

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


print("Comparing percentages of <syntactic> vs <semantic (+ semantic)> results per model, per approach")
### syntactic x semantic


print("codegemma: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(11,18), n=c(100,100))

print("few-shot 1")
prop.test(x = c(26,39), n=c(100,100))

print("few-shot 2")
prop.test(x = c(14,34), n=c(100,100))

print("few-shot 3")
prop.test(x = c(7,29), n=c(100,100))


print("llama3: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(41,53), n=c(100,100))

print("few-shot 1")
prop.test(x = c(14,61), n=c(100,100))

print("few-shot 2")
prop.test(x = c(53,69), n=c(100,100))

print("few-shot 3")
prop.test(x = c(15,35), n=c(100,100))


print("Mistral: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(3,27), n=c(100,100))

print("few-shot 1")
prop.test(x = c(6,25), n=c(100,100))

print("few-shot 2")
prop.test(x = c(7,10), n=c(100,100))

print("few-shot 3")
prop.test(x = c(12,28), n=c(100,100))


print("----------------Iteration 2-----------------")

## looking for significant difference among the models

print("zero-shot")

# zero-shot

codegemma <- 96
llama3 <- 47
mistral <- 8


print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


# few-shot 1
print("few-shot 1")

codegemma <- 1
llama3 <- 0
mistral <- 26

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


# few-shot 2

print("few-shot 2")

codegemma <- 27
llama3 <- 2
mistral <- 47

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))

# few-shot 3

print("few-shot 3")

codegemma <- 19
llama3 <- 22
mistral <- 27

print("z-test between codegemma and llama3")
prop.test(x = c(codegemma ,llama3), n=c(100,100))

print("z-test between codegemma and mistral")
prop.test(x = c(codegemma,mistral), n=c(100,100))

print("z-test between mistral and llama3")
prop.test(x = c(mistral,llama3), n=c(100,100))


print("Comparing percentages of <syntactic> vs <semantic (+ semantic)> results per model, per approach")
### syntactic x semantic


print("codegemma: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(96,97), n=c(100,100))

print("few-shot 1")
prop.test(x = c(1,2), n=c(100,100))

print("few-shot 2")
prop.test(x = c(27,31), n=c(100,100))

print("few-shot 3")
prop.test(x = c(19,35), n=c(100,100))


print("llama3: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(47,49), n=c(100,100))

print("few-shot 1")
prop.test(x = c(0,37), n=c(100,100))

print("few-shot 2")
prop.test(x = c(2,20), n=c(100,100))

print("few-shot 3")
prop.test(x = c(22,32), n=c(100,100))


print("Mistral: semantic vs syntactic")

print("zero-shot")
prop.test(x = c(8,11), n=c(100,100))

print("few-shot 1")
prop.test(x = c(26,30), n=c(100,100))

print("few-shot 2")
prop.test(x = c(47,48), n=c(100,100))

print("few-shot 3")
prop.test(x = c(17,17), n=c(100,100))

sink()