---
title: "Evaluation"
author: "Víctor López Juan"
date: '2020-06-15'
output: pdf_document
---

```{r}
library(ggplot2)
library(tidyr)
library(dplyr)
library(extrafont)
library(boot)
library(cowplot)
library(ggpubr)
library(svglite)
library(xtable)
library(Cairo)
library(ggthemes)
theme_set(theme_cowplot())
theme_set(theme_light() + theme(text = element_text(family = "DejaVu Serif")))
#pdf.options(encoding='ISOLatin2')
#extrafont::font_import()
extrafont::loadfonts()
library(RColorBrewer)
```

```{r}
# This palete is colour-blind friendly and grayscale friendly"
custom.palette <- brewer.pal(4, "OrRd")[2:4]
```


```{r}
benchmarks <- read.csv("benchmark-tyde2020.csv")
benchmarks$config_name <- as.factor(benchmarks$config_name)
benchmarks
```


```{r}
mean.ci <- function(x, p=0.975) {
  n = length(x)
  mean(x)+sd(x)/sqrt(n)*qt(p, n-1)
}

quantile.ci <- function(x, p=0.975) {
  n = length(x)
  mean(x)+sd(x)/sqrt(n)*qt(p, n-1)
}

fix <- function(x, default) { if(is.null(x)) { default } else {x}}
calc.boot <- function(x) {
  set.seed(2398329)
  boot.obj <- boot(data=x,
                   statistic=function(data,indices){median(data[indices])},
                   R=1000)
  sink("aux")
  ci <- boot.ci(boot.obj, type="perc")
  sink(NULL)
  qs <- quantile(x, c(0.025, 0.5, 0.975))
  list(fix(ci$perc[4], qs[1]),
       fix(ci$t0, qs[2]),
       fix(ci$perc[5], qs[3]),
       length(x[!is.null(x)]))
}
calc.in <- calc.boot

benchmarks.avg <- benchmarks %>% filter(exit_code == 0) %>% 
  group_by(program, commit, config_name, family_index, family_name) %>%
  do(cpu.in = calc.in(.$cpu_time_s),
     mem.in = calc.in(.$memory_mb)) %>%
  rowwise() %>%
  mutate(cpu = cpu.in[[2]],
         mem = mem.in[[2]],
         cpu.up = cpu.in[[3]],
         cpu.dn = cpu.in[[1]],
         mem.up = mem.in[[3]],
         mem.dn = mem.in[[1]],
         n.samples.cpu = cpu.in[[4]],
         n.samples.mem = mem.in[[4]],
         cpu.in = NULL,
         mem.in = NULL)
```


```{r}
lookup <- function (source, target) {
  stopifnot(length(source) == length(target))
  function (x) {
    res <- target[match(x,source)]
    res[is.na(res)] <- x[is.na(res)]
    res
  }
}
```

```{r}
avg.lang <- benchmarks.avg %>% filter(family_name == "Language")
avg.lang
```

```{r}
benchmarks %>% filter(exit_code != 0)
```


```{r, message=FALSE}
levels.config <- c("W;fast;HC4;fe;noFastEq","W;fast;HC4;fe","Agda-")
language.data <- avg.lang %>% filter(config_name %in% c("Agda-","W;fast;HC4;fe","W;fast;HC4;fe;noFastEq"))
labels.language = lookup(source=c(0,1,2,3,4,5,6),
                  target=c("Type", "Pointed", "Subset", "Multigraph",
                           "Refl. graph","Precat.", "Setoid"))
labels.config   = lookup(source=c("Agda-","W;fast;HC4;fe","W;fast;HC4;fe;noFastEq","W;fast;HC4"),
                         target=c("Agda","Tog⁺","Tog⁺ without the syntactic equality rule","Tog⁺ without the fast elaboration shorcut"))

p1 <- ggplot(language.data %>% filter(family_index > 4), aes(x=family_index, y=cpu*1000, fill=config_name)) +
  geom_bar(stat="identity",position=position_dodge()) +
  geom_errorbar(aes(ymin=cpu.up*1000, ymax=cpu.dn*1000), position=position_dodge(.9), width=.4) +
  ylab("CPU (ms)") +
  xlab("Structure") +
  labs(fill="Implementation") +
  scale_fill_manual(values=custom.palette,labels=labels.config,breaks=levels.config) +
  scale_x_continuous(breaks=0:6,labels=labels.language) +
  coord_flip()

p2 <- ggplot(language.data %>% filter(family_index > 4), aes(x=family_index, y=mem, fill=config_name)) +
  geom_bar(stat="identity",position=position_dodge()) +
  geom_errorbar(aes(ymin=mem.up, ymax=mem.dn), position=position_dodge(.9), width=.4) +
  ylab("Memory (MB)") +
  xlab("Structure") +
  labs(fill="Implementation") +
  scale_x_continuous(breaks=0:6,labels=labels.language) +
  scale_fill_manual(values=custom.palette,labels=labels.config,breaks=levels.config) +
  theme(axis.title.y = element_blank()) +
  coord_flip()

p3 <- ggplot(language.data %>% filter(family_index <= 4), aes(x=family_index, y=cpu*1000, fill=config_name)) +
  geom_bar(stat="identity",position=position_dodge()) +
  geom_errorbar(aes(ymin=cpu.up*1000, ymax=cpu.dn*1000), position=position_dodge(.9), width=.4) +
  ylab("CPU (ms)") +
  xlab("Structure") +
  labs(fill="Implementation") +
  scale_fill_manual(values=custom.palette,labels=labels.config,breaks=levels.config) +
  scale_x_continuous(breaks=0:6,labels=labels.language) +
  coord_flip()

p4 <- ggplot(language.data %>% filter(family_index <= 4), aes(x=family_index, y=mem, fill=config_name)) +
  geom_bar(stat="identity",position=position_dodge()) +
  geom_errorbar(aes(ymin=mem.up, ymax=mem.dn), position=position_dodge(.9), width=.4) +
  ylab("Memory (MB)") +
  xlab("Structure") +
  labs(fill="Implementation") +
  scale_x_continuous(breaks=0:6,labels=labels.language) +
  scale_fill_manual(values=custom.palette,labels=labels.config,breaks=levels.config) +
  theme(axis.title.y = element_blank()) +
  coord_flip()

ggarrange(p1,p2,p3,p4,ncol=2,nrow=2,align="v",common.legend=TRUE, legend="bottom")
ggsave(filename="lang_all.pdf", width=8, height=4.5, device=cairo_pdf)
```