---
title: "Understanding Peer Review of Software EngineeringPapers"
output:
  html_document: default
  pdf_document: 
    keep_tex: yes
date: "27-02-2021"
---

Data Analysis script for the paper

**"Understanding Peer Review of Software EngineeringPapers"** 

by Neil A. Ernst, Jeffrey C. Carver, Daniel Mendez, Marco Torchiano.

These scripts are released under the CC-BY (Creative Commons, Attributions) 4.0 License.

You are free to:

- Share — copy and redistribute the material in any medium or format
- Adapt — remix, transform, and build upon the material for any purpose, even commercially.
 
The licensor cannot revoke these freedoms as long as you follow the license terms.

Under the following terms:

- Attribution — You must give appropriate credit, provide a link to the license, and indicate if changes were made. You may do so in any reasonable manner, but not in any way that suggests the licensor endorses you or your use.

- No additional restrictions — You may not apply legal terms or technological measures that legally restrict others from doing anything the license permits.

The full license is available at: <https://creativecommons.org/licenses/by/4.0/legalcode>

```{r setup, include=FALSE}

library(ggplot2)
library(tidyr)
library(dplyr)
library(stringr)
library(knitr)
library(effsize)
library(pander)
library(ggrepel)
library(treemapify)
library(ggrepel)


knitr::opts_chunk$set(echo = FALSE)
knitr::opts_chunk$set(warning = FALSE)
ggplot2::update_geom_defaults("bar", list(fill = "steelblue",color=NA))
ggplot2::update_geom_defaults("violin", list(fill = "steelblue",color="navy"))
theme_set(theme_light()+theme(axis.title.y = element_text(hjust=1,vjust=1,angle=0),
                              legend.position = "top"))
```
```{r load data}

d = read.csv("clean_raw_data.csv")
questions = read.csv("items.csv")


for( v in grep("^Q.*",names(d),value = TRUE)){
  d[[v]][d[[v]]==""] <- NA
}

for( v in grep("^Q.*[0-9]$",names(d),value = TRUE)){
  d[[v]] <- factor(d[[v]],sort(unique(d[[v]])))
}

for( v in grep("^Q.*[0-9]+_TEXT$",names(d),value = TRUE)){
  d[[v]] <- as.character(d[[v]])
}

for( v in grep("^Q11_.*",names(d),value = TRUE)){
  d[[v]] <- as.numeric(as.character(d[[v]]))
}


levels(d$Q4) = c("NoAward","Award")
levels(d$Q6) = c("PlainReviewer","ChairOrEditor")
d$Q9 <- factor(d$Q9,c("Less than 1 year", "1-4 years","5-9 years", "10-19 years", "More than 20 years"),ordered=TRUE)
levels(d$Q9) <- c("Less than 1", "1-4","5-9", "10-19", "More than 20")

numrev.levels <- c("0","1-5","6-10","11-20","20+")
d$Q10_1 <- factor(d$Q10_1,numrev.levels,ordered=TRUE)
d$Q10_2 <- factor(d$Q10_2,numrev.levels,ordered=TRUE)
d$Q10_3 <- factor(d$Q10_3,numrev.levels,ordered=TRUE)

d$Q13 <- str_replace(d$Q13,"interest, independent","interest; independent")

timerev.levels <- c("Less than 30 mins","30-59 mins", "1-2 hours","More than 2 hours")
d$Q14_1 <- factor(d$Q14_1,timerev.levels,ordered=TRUE)
d$Q14_2 <- factor(d$Q14_2,timerev.levels,ordered=TRUE)
d$Q14_3 <- factor(d$Q14_3,timerev.levels,ordered=TRUE)

numread.levels <- c("Never, focus on sections","Once", "Twice","Three or more times")
d$Q16_1 <- factor(d$Q16_1,numread.levels,ordered=TRUE)
d$Q16_2 <- factor(d$Q16_2,numread.levels,ordered=TRUE)

d$Q18 <- gsub(", ","  ",d$Q18)

d$Q24 <- factor(d$Q24,c("Little impact: I pay them little to no attention.",
                        "Moderate impact: I consider the overall aim, but not the detailed templates.",
                        "Big impact: I strive to make my review comply in every way with the PC guidelines."),
                        ordered=TRUE)

d <- d %>% mutate_at(vars(matches("Q27_\\d+$")),function(x)
                    factor(x,c("Extremely easy","Somewhat easy", 
                                     "Neither easy nor difficult", "Somewhat difficult",
                                     "Extremely difficult"),ordered=T))

d$Q27_4_TEXT <- gsub("^precis.+","Precision Conference",d$Q27_4_TEXT,ignore.case = TRUE)


d <- d %>% mutate_at(vars(matches("Q30_\\d+$")),function(x)
                     factor(x,rev(c("Strongly Disagree", "Disagree", "Neutral", "Agree", 
                                    "Strongly Agree")),ordered = TRUE))
                     
d <- d %>% mutate_at(vars(matches("Q33_\\d+$")),function(x)
                    factor(x,c("Extremely good", "Somewhat good", "Neither good nor bad", 
                            "Somewhat bad", "Extremely bad"),ordered=TRUE))

d <- d %>% mutate_at(vars(starts_with("Q36")),function(x) 
                  factor(x,c("Extremely important", "Very important", "Moderately important", 
                             "Slightly important", "Not at all important" ),ordered=T))
```

```{r utility, include=FALSE}
keep_order <- function(x){
  factor(x,as.character(x),ordered=TRUE)
}

#' Wrap factor levels
#' 
#' Change the labels of a factor by wrapping them
#' to a given length
#'
#' @param faktor The factor whose level are top be wrapped-
#' @param ...    Additional parameters passed to \code{\link{stringr::str_wrap}}.
#'
fctr_wrap <- function(faktor,...){
   if(!is.factor(faktor)) stop("only accept factors");
   levels(faktor) <- str_wrap(levels(faktor),...)
   faktor
   #return ( factor(str_wrap(faktor,width),str_wrap(levels(faktor),width),ordered=is.ordered(faktor)) );
}

fctr_replace <- function(faktor, ...){
   if(!is.factor(faktor)) stop("only accept factors");
   levels(faktor) <- str_replace(levels(faktor),...)
   faktor
}

count_all_options <- function(.data,varName,measureName,by="Q4"){
  reps = 1 + gsub("[^,;]+","",.data[[varName]]) %>% str_length %>% max(na.rm=TRUE)
  re = paste0(rep("([^,]+)?",reps),collapse=",?")
  if( is.null(by) || is.na(by) || nchar(by)==0){
    by = "All"
    .data <- .data %>% mutate(All = "All")
  }
  measure.popularity <- .data %>% select(by,varName) %>% 
        extract(varName,paste0(measureName,1:reps),re) %>%
        gather(key=!!paste0(measureName,"Num"),value=!!measureName,
               starts_with(measureName),na.rm = TRUE) %>%
        group_by(!!as.symbol(by),!!as.symbol(measureName)) %>% count(.drop=FALSE) %>% ungroup() %>%
        filter(!!as.symbol(measureName)!="") %>%
        inner_join(.data %>% count(!!as.symbol(by)) %>% rename(N=n),by=by) %>%
        mutate(Prop = n/N)
  
        
  ordered.measure =  measure.popularity %>% group_by(!!as.symbol(measureName)) %>% 
                    summarize(n=sum(n)) %>% arrange(n) 
  
  measure.popularity <- measure.popularity %>% 
    mutate(!!measureName := factor(!!as.symbol(measureName),
                                   ordered.measure[[measureName]],ordered = T))
  return ( measure.popularity )
}

analyze_ranks <- function(.data,varName,measure,by="Q4"){
  if( is.null(by) || is.na(by) || nchar(by)==0){
    by = "All"
    .data <- .data %>% mutate(All = "All")
  }
  pcfactor.relevance <- .data %>% select(by,starts_with(varName),-ends_with("TEXT")) %>%
        mutate_at(vars(starts_with(varName)),function(x) as.numeric(as.character(x))) %>%
        gather(key=!!measure,value="Rank",starts_with(varName)) %>%
        mutate(!!as.symbol(measure) := gsub(".* - ","",unlist(questions[,!!as.symbol(measure)])) ) %>%
        filter(!is.na(Rank))
  
  pcfactor.relevance.summary <- pcfactor.relevance %>% 
            group_by(!!as.symbol(measure)) %>%
            summarize(MedianRank=median(Rank),
                      MeanRank=mean(Rank,na.rm=TRUE),
                      SdRank=sd(Rank,na.rm=TRUE)) %>% 
            arrange(MeanRank)
  
  pcfactor.relevance[[measure]] <- factor(pcfactor.relevance[[measure]],
                                          pcfactor.relevance.summary[[measure]],ordered=TRUE)
  return( pcfactor.relevance )
}

parallel_ranks <- function(.data,var,by="Q9",xlabel="Experience"){
  
.data %>% filter(!is.na(!!as.symbol(by))) %>% 
  analyze_ranks(var,"measure",by=by) %>%
  group_by(measure,!!as.symbol(by)) %>% 
  summarize(Rank=mean(Rank)) %>%
  show_parallel(by,"Rank",xlabel)
}

show_parallel <- function(.data,by,score="Rank",xlabel="Experience"){
  direct.labels = .data %>% group_by(measure) %>% 
                  summarize(!!as.symbol(by):= max(!!as.symbol(by))) %>%
                  left_join(.data %>% select(measure,by,score))
  
  percentage = max( .data[[score]] ) <= 1
  
  if(percentage){
     score.range = seq(floor(min(.data[[score]])*10)/10, ceiling(max(.data[[score]])*10)/10, length.out = 6)
     lab = "Mean\nProportion"
  }else{ # scores
      score.range = seq(floor(min(.data[[score]])), ceiling(max(.data[[score]])))
      lab = "Mean\nScore"
  }

g = ggplot(.data,aes(x=!!as.symbol(by),y=!!as.symbol(score),group=measure,color=measure))+
  #geom_vline(aes(xintercept=as.numeric(Q9)-1),size=1,color="gray")+
  annotate("segment",x=1:4,xend=1:4,y=min(score.range),yend=max(score.range), color = 'gray', size=2)+
  annotate("segment",y=score.range,yend=score.range,x=.8,xend=4, color = 'gray', size=.5)+
  geom_line()+geom_point()+
  geom_text_repel(data = direct.labels, aes(label=str_wrap(measure,30)), hjust=0, nudge_x=0.1)+
  #annotate("text_repel",x=direct.labels[[by]],y=direct.labels$measure, aes(label=str_wrap(measure,30)), hjust=0, nudge_x=0.1)+
  xlab(xlabel)+ylab(lab)

if(!percentage){
  g <- g + scale_y_reverse(breaks=score.range)
}else{
  g <- g + scale_y_continuous(breaks=score.range,labels = scales::percent)
}

if(length(levels(.data$measure))<=8){
  g <- g +   scale_color_brewer(type="qual",palette=2)
}else{
  g <- g +   scale_color_viridis_d(option="magma")
}

g + guides(color="none")+
  scale_x_discrete(expand=expand_scale(0,c(.2,2)))+

  theme(panel.border = element_blank(),panel.grid=element_blank())
}

showrank <- function(.data, var,wrap_width=25,by="Q4"){
  
  res=list()
  res$ranks = .data
  
  .data <- .data %>% mutate(!!var := fctr_wrap(!!as.symbol(var),wrap_width))
  .summary <- .data %>% group_by(!!as.symbol(by),!!as.symbol(var)) %>%
              summarize(MeanRank=mean(Rank,na.rm=TRUE)) %>% 
              arrange(MeanRank) 
  res$summary <- .summary

  .data.aggregate <- 
  .data %>% group_by(!!as.symbol(var),!!as.symbol(by),Rank) %>% count() %>%
            group_by(!!as.symbol(var),!!as.symbol(by)) %>% mutate(Prop = n / sum(n))
  
  nranks = max(.data$Rank)
  
  res$plot <- 
  .data %>%
  ggplot(aes(x=Rank,group=paste0(!!as.symbol(var),!!as.symbol(by))))+

  geom_vline(data=.summary,aes(xintercept=MeanRank,color=""))+
  geom_text(data=.summary,aes(x=MeanRank,label=round(MeanRank,1)),
            y=max(.data.aggregate$Prop),vjust=0.5,hjust="inward",color="firebrick")+
  theme(legend.position = "top", legend.box.just = "right")+
  scale_color_manual(name="mean rank",values = "firebrick")+

  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%")),
            stat="count",vjust=1.1,color='black',fontface="bold",size=2)+
  geom_bar(aes(y=..prop..))+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%")),
            stat="count",vjust=1.1,color='white',size=2)+

  scale_x_continuous(breaks=1:nranks,minor_breaks = NULL)+
  scale_y_continuous(labels = scales::percent)+
  xlab("Rank")+ylab("Freq")+
  facet_grid(as.formula(paste0(by,"~",var))) 
  
  return( res )
}

#.data=d
#varName = "Q30"
#measure = "Method"
analyze_grade <- function(.data,varName,measure,by="Q4"){
  grade.levels = levels(.data[[grep(paste0("^",varName),names(.data),value = TRUE)[1]]])
  pcfactor.relevance <- .data %>% select(by,starts_with(varName),-ends_with("TEXT")) %>%
        #mutate_at(vars(starts_with(varName)),function(x) as.numeric(as.character(x))) %>%
        gather(key=!!measure,value="Grade",starts_with(varName)) %>%
        mutate(!!as.symbol(measure) := gsub(".* - ","",unlist(questions[,!!as.symbol(measure)])) ) %>%
        mutate(Grade = factor(Grade,grade.levels,ordered=TRUE)) 
  
  pcfactor.relevance.summary <- pcfactor.relevance %>% 
            group_by(!!as.symbol(measure)) %>%
            summarize(MeanGrade=mean(as.numeric(Grade),na.rm=TRUE),
                      SdGrade=sd(as.numeric(Grade),na.rm=TRUE)) %>% 
            arrange(MeanGrade)
  
  pcfactor.relevance[[measure]] <- factor(pcfactor.relevance[[measure]],
                                          pcfactor.relevance.summary[[measure]],ordered=TRUE)
  return( pcfactor.relevance )
}

showgrade <- function(.data, var,wrap_width=25,scale_boundaries=FALSE,by="Q4"){
  
  res=list()
  res$ranks = .data
  
  .data <- .data %>% mutate(!!var := fctr_wrap(!!as.symbol(var),wrap_width))
  .summary <- .data %>% group_by(!!as.symbol(by),!!as.symbol(var)) %>%
            summarize(MeanGrade=mean(as.numeric(Grade),na.rm=TRUE)) %>% 
            arrange(MeanGrade)

  res$summary <- .summary

  .data.aggregate <- 
  .data %>% group_by(!!as.symbol(var),!!as.symbol(by),Grade) %>% count() %>%
            group_by(!!as.symbol(var),!!as.symbol(by)) %>% mutate(Prop = n / sum(n))
  
  
#x_scale_labels = gsub(" ","\n",levels(.data$Grade))
x_scale_labels = str_wrap(levels(.data$Grade),10)

if(scale_boundaries){
   x_scale_labels[2:(length(x_scale_labels)-1)] <- "." 
}

res$plot <-
  .data.aggregate %>% 
ggplot(aes(x=as.numeric(Grade),y=Prop))+
  geom_bar(stat="identity")+
  geom_text(aes(label=paste0(round(Prop*100),"%"), vjust=(Prop>0.05)*1.2-0.1, color=Prop>0.05),size=2)+
  scale_color_manual(values=c("black","white"), guide=FALSE)+
#  geom_text(aes(label=paste0(round(Prop*100),"%")),vjust=1.1,color='white',size=2)+
  
#  geom_vline(data=.summary,aes(xintercept=MeanGrade,color=""))+
#  geom_text(data=.summary,aes(x=MeanGrade,label=round(MeanGrade,1)),
#            y=max(.data.aggregate$Prop),vjust=0.5,hjust=-0.2,color="firebrick")+
#  scale_color_manual(name="mean agreement",values = "firebrick")+
#  theme(legend.position = "top", legend.box.just = "right")+

  scale_x_continuous(breaks=1:5,labels = x_scale_labels,
                     minor_breaks = NULL)+
    scale_y_continuous(labels = scales::percent)+     
  xlab("Agreement")+ylab("Proportion")+
  theme(axis.title.y = element_text(vjust=1.08,angle=pi/2))+
  facet_grid(as.formula(paste0(by,"~",var)))
  
  return( res )
}

```

# Questions analyzed

```{r,results="asis"}
questions %>% select(starts_with("Q"),-ends_with("TEXT")) %>% gather(ID,QUESTION) %>%
  extract(ID,c("ID","SUB"),"([^_]+)_?([^_]+)?") %>% 
  mutate(QUESTION = str_replace(QUESTION,"(?<=\\?).*$","")) %>%
  mutate(QUESTION = str_replace(QUESTION," - .*$","")) %>%
  select(ID,QUESTION) %>% distinct() %>%
  pander::pandoc.table(split.cells=120,split.tables=200,justify="ll")
```

# Response completeness

Completeness of data records (i.e. number of non NA values across each line)

```{r completeness, fig.width=5,fig.height=3}
completeness = apply(d,1,function(x) sum(!is.na(x))) / dim(d)[2]
#completeness = apply(d.all,1,function(x) sum(!is.na(x))) / dim(d.all)[2]

ggplot(data.frame(completeness=completeness),aes(x=completeness))+
  geom_bar()+ scale_x_continuous(labels = scales::percent,limits=c(0,1))

```

We decide to drop all cases with completeness lower than 66% (`r sum(completeness<=.66)` cases).

```{r clean, results='asis'}
paste0(d$ResponseId[completeness<=.66],collapse = ", ")
d <- d[completeness>.66,]
```

# Demographics

### Award recipients

Recipient of best reviewer award award:
```{r award,results="asis"}
table(d$Q4) %>% pandoc.table(row.names = c("Freq"),caption=as.character(questions[["Q4"]]))

table(d$Q5) %>% pandoc.table(row.names = "Freq",caption=as.character(questions[["Q5"]]))

#with(d,table(Q4,Q5,useNA = "ifany"))
```

```{r treemap award}

d %>% count(Q4,Q5) %>% 
  mutate(Label=ifelse(is.na(Q5),as.character(Q4),paste0(Q4,":",Q5)),
         Prop=n/sum(n)) %>%
  ggplot(aes(area=n,subgroup=Q4,subgroup1=Q5,fill=Label))+
  geom_treemap()+geom_treemap_text(aes(label=paste0(Label,"\n",n," (",round(Prop*100),"%)")))+
  scale_fill_brewer(type="qual",palette=2,name="Best paper award")
  
```

### Decisional role experience

Played decisional role using reviews:
```{r decisional role}
table(d$Q6,d$Q4,useNA = "ifany") %>% addmargins(2) %>% kable(caption="Award recipients vs. decisional roles")

fisher.test(table(d$Q6,d$Q4))
```

The odds of having received an award doubles for people who had any decisional role experience (though not significant).

### Years of experience

```{r experience, fig.width=7, fig.height=4}
full.experience = d%>% select(Q4) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q9)%>%distinct() %>% na.omit)

d %>% filter(!is.na(Q9)) %>% group_by(Q4,Q9) %>% tally() %>% 
  full_join(full.experience,by = c("Q4", "Q9")) %>% mutate(n=coalesce(n,as.integer(0))) %>%
  group_by(Q4) %>% mutate(Prop = n/sum(n)) %>%
ggplot(aes(x=Q9,y=Prop,fill=Q4))+
  geom_bar(aes(y = Prop),stat="identity",position="dodge")+
  geom_text(aes(label= paste0(round(Prop*100,0),"%"),y=Prop,group=Q4),  
             color=rep(c("black","white"),each=4),hjust=1.1,position=position_dodge(width=1))+
  #xlab("Experience in\nreviewing papers")+
  xlab("")+
  ylab("Proportion of respondents")+
  annotate("text",x=c(4.25,3.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+
  scale_y_continuous(labels = scales::percent)+     
  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  coord_flip()
```

```{r experience-violin, fig.width=7, fig.height=4}
full.experience = d%>% select(Q4) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q9)%>%distinct() %>% na.omit)

d %>% filter(!is.na(Q9)) %>%
ggplot(aes(y=Q9,x=Q4,fill=Q4,group=Q4))+
  geom_violin(adjust=1.8,draw_quantiles=c(.5),alpha=.5)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  geom_dotplot(binaxis = "y",dotsize = .25, stackdir="center")+
    coord_flip()+
  xlab("Experience in\nreviewing papers")     

```

A quick consistency check:
```{r experience check}
table(d$Q9,d$Q6) %>% kable()
cliff.delta(Q9 ~ Q6,data=d)
```

As expected, the effect size of being an editor on the experience is significantly large.


### Comfortable topics

```{r topic-overall, fig.width=10,fig.height=5}
topic.popularity <- d %>% count_all_options("Q7","Topic",by=NULL)

ggplot(topic.popularity,aes(x=Topic,y=Prop))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "),color=Prop<0.5), position=position_dodge(1),hjust="inward",size=2)+
  scale_color_manual(values=c("white","black"),guide="none")+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion comfortable with reviewing that topic")+xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())
```


```{r topic, fig.width=8,fig.height=6}
topic.popularity <- d %>% count_all_options("Q7","Topic") %>% arrange(Topic,Q4) %>% filter(Topic!="")

ggplot(topic.popularity,aes(x=Topic,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")),
            color=c(rep("black",2),rep(c("black","white"),12)),hjust=1.1,
            #color=rep(c("black","white"),each=13),hjust=1.1,
            position=position_dodge(1),hjust="inward",size=2)+
  annotate("text",x=c(13.25,12.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion comfortable with reviewing that topic")+xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())
```


```{r other topics}
d$Q7_10_TEXT <- d$Q7_10_TEXT %>% tolower() %>%
                str_replace(".*maint.*evol.*","maintenance and evolution") %>%
                str_replace(".*global.*","global software development") %>%
                str_replace("\\(.*\\)","") %>%
                str_replace(".*product lines.*","software product lines") %>%
                str_replace(".*quality.*","software quality") %>%
                str_replace(".*model.*","model-driven development") %>%
                str_replace(".*process.*","software process")
  

d$Q7_10_TEXT %>% strsplit("[,;]") %>% unlist() %>% tolower() %>% 
  str_replace_all("^\\s+|\\s+$","") %>% str_replace_all("&","and") %>%
  table() %>% as_tibble() %>% arrange(desc(n)) %>% kable(col.names = c("Other topics","Freq"))

#as.data.frame(table(unlist(lapply(as.character(d$Q7_10),function(x) gsub("^ *| *$","",tolower(unlist(strsplit(x,"[,;]")))))))) %>% arrange(desc(Freq)) %>% kable(col.names = c("Other topics","Freq"))
```

### Comfortable methods

```{r strategy, fig.width=7,fig.height=5 }
 
strategy.popularity <- d %>% count_all_options("Q8","Strategy")

ggplot(strategy.popularity,aes(x=Strategy,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Poroportion comfortable with reviewing that research method")+
  xlab("Research Methods")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r other strategies}
d$Q8_9_TEXT %>% strsplit("[,;]") %>% unlist() %>% tolower() %>% str_replace("^\\s*|\\s*$","") %>% 
  table() %>% as_tibble() %>% arrange(desc(n)) %>% kable(col.names = c("Other strategies","Freq"))
```



### Number of reviews per year

Number of papers and reviewers

```{r number reviews,fig.width=8,fig.height=4}
d %>% select(Recipient=Q4,Journal=Q10_1,Conference=Q10_2,Workshop=Q10_3) %>% 
      gather("Venue","Reviews",Journal,Conference,Workshop) %>% 
      mutate(Reviews = factor(Reviews,levels(d$Q10_1),ordered=TRUE)) %>% 
      filter(!is.na(Reviews)) %>%
ggplot(aes(x=Reviews,group=Venue))+
  #geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%"),vjust=if_else(..prop..>0.05,1.2,-0.1)),stat="count",color='black',fontface="bold",size=3)+
  geom_bar(aes(y=..prop..))+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%"),
                vjust=if_else(..prop..>0.05,1.2,-0.2),color=..prop..>0.05),stat="count",size=3)+
  scale_color_manual(values=c("black","white"),guide="none")+
  xlab("Number of papers")+ylab("")+
  scale_y_continuous(labels = scales::percent_format(10))+
  facet_grid(Recipient~Venue)
```

```{r number reviews conf journal}
ggplot(d %>% filter(!is.na(Q10_1),!is.na(Q10_2)),aes(x=Q10_1,y=Q10_2))+
  geom_tile(stat="bin2d",aes(fill=..count..),color='white')+
  geom_text(stat="bin2d",aes(label=..count..,color=..count..>15))+
  scale_color_manual(values=c("white","black"),guide=FALSE)+
  xlab("Journal reviews")+ylab("Conference reviews")
```

Detail of number of reviews of conference and Journal by Award type.

```{r number reviews by award type,fig.width=6,fig.height=3}
ggplot(d %>% filter(!is.na(Q10_1),!is.na(Q10_2)),aes(x=Q10_1,y=Q10_2))+
  geom_tile(stat="bin2d",aes(fill=..count..))+
  geom_text(stat="bin2d",aes(label=..count..,color=..count..>15))+
  scale_color_manual(values=c("white","black"),guide=FALSE)+
  facet_grid(.~Q5)+
  xlab("Journal reviews")+ylab("Conference reviews")
```


# Attitude and motivation

### Factors affectinge review acceptance 

What factors influence your decision to agree to a journal review invitation?
(ranking options)


```{r accept factors,fig.width=12,fig.height=5}

accept.factor <- d %>% filter(!is.na(Q4)) %>% analyze_ranks("Q11","Motivation") %>% showrank("Motivation")

# To avoid grouping by award
# accept.factor <- d %>% filter(!is.na(Q4)) %>% analyze_ranks("Q11","Motivation",by=NULL) %>% showrank("Motivation",by="All")

accept.factor$summary %>% spread(Q4,MeanRank) %>%
                          mutate(Motivation = str_replace(Motivation,"\n"," ")) %>% 
                          kable

accept.factor$plot

```

Factors by experience

```{r accept factors by experience,fig.width=10,fig.height=5}

parallel_ranks(d,"Q11")

```

What factors influence your decision to accept a Program Committee invitation?
(ranking options)

```{r pc factors, fig.width=12,fig.height=5}

pc.factor <- d %>% filter(!is.na(Q4)) %>% analyze_ranks("Q12","Motivation") %>% showrank("Motivation")

pc.factor$summary %>% spread(Q4,MeanRank) %>%
                          mutate(Motivation = str_replace(Motivation,"\n"," ")) %>% kable

pc.factor$plot


```

```{r pc factors by experience,fig.width=10,fig.height=5}

parallel_ranks(d,"Q12")

```

Comparison of reasons for reviewing journal vs joining PC

```{r comparison of motivation factors}

motivation.factors <- pc.factor$summary %>% 
            mutate(Motivation=gsub("submission topics","sumission/article topic",as.character(Motivation))) %>% inner_join(accept.factor$summary %>% 
            mutate(Motivation=gsub("article","sumission/article",as.character(Motivation))),
            by=c("Q4","Motivation"),suffix=c("_PC","_JOURNAL"))

motivation.factors %>%  mutate(Motivation = str_replace(Motivation,"\n"," ")) %>% kable()
```

```{r comparison motivation dot plots}
motivation.factors %>% select(Q4,Motivation,starts_with("MeanRank")) %>%   
    gather(key="Venue",value="MeanRank",MeanRank_JOURNAL,MeanRank_PC) %>%
    mutate(Venue= factor(gsub(".+_","",as.character(Venue))) ) %>%
    arrange(desc(MeanRank)) %>% ungroup() %>%
    mutate(Motivation = factor(Motivation,unique(as.character(Motivation)),ordered = TRUE)) %>%
ggplot(aes(x=Motivation,y=MeanRank,color=Venue,shape=Venue))+
  geom_point(size=3)+
  ylab("Mean rank")+
  #annotate("text",x=1,y=1,label="Most important",hjust=0,vjust=2)+
  #annotate("text",x=1,y=7,label="Least important",hjust=1,vjust=2)+
  #annotate("segment",x=.7,xend=.7,y=2.7,yend=5.3,arrow=arrow(length = unit(5, "pt"),ends="both"))+
  scale_color_brewer(type="qual",palette=6)+
  scale_y_continuous(breaks=1:7)+
  coord_flip()+facet_grid(.~Q4)

```

### Bidding motivations

```{r bidding motivation,fig.width=7,fig.height=4}
motivation.popularity <- d%>%count_all_options("Q13","Motivation")%>%filter(Motivation!="")

motivation.popularity %>%
ggplot(aes(x=Motivation,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),
            color=c(rep(c("black","white"),each=2),rep("black",6)),hjust="inward",size=2)+
  
    annotate("text",x=c(5.25,4.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents citing that motivation for bidding")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r bidding motivation single,fig.width=7,fig.height=3}
motivation.popularity <- d%>%count_all_options("Q13","Motivation",by = NA)

motivation.popularity %>%
ggplot(aes(x=Motivation,y=Prop))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "), color=Prop>0.5), position=position_dodge(1), hjust="inward",size=2)+
  scale_color_manual(values=c("black","white"), guide=FALSE)+
  
#    annotate("text",x=c(5.25,4.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

#  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents citing that motivation for bidding")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```


Bidding motivation by experince

```{r bidding motivation by experience,fig.width=10,fig.height=5}
motivation.popularity.exp <- d%>%count_all_options("Q13","Motivation",by="Q9")

motivation.popularity.exp %>% 
  rename(measure=Motivation) %>%
  show_parallel("Q9","Prop")

```

# Review method

### Time review  (`Q14`)

```{r review time,fig.width=8,fig.height=4}
d %>% select(Q4,Journals=Q14_1,Conferences=Q14_2,Workshops=Q14_3) %>%
      gather(Venue,Time,Journals,Conferences,Workshops) %>% ungroup() %>%
      mutate(Time = factor(Time,levels(d$Q14_1),ordered=TRUE),
             Venue = factor(Venue,c("Journals","Conferences","Workshops"),ordered=TRUE)) %>%
      filter(!is.na(Time)) %>%
ggplot(aes(x=Time,fill=Q4,group=paste0(Q4,Venue)))+
  geom_bar(aes(y=..prop..),position="dodge")+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%")),stat="count",
            hjust="inward",color='black',size=3,position=position_dodge(width=1))+
  
    geom_text(data=data.frame(Time=c(4.25,3.75),Q4=c("Award","NoAward"),
                              Venue=factor(rep("Journals",2),c("Journals","Conferences","Workshops"),ordered=TRUE)),
              aes(label=Q4,y=0),
             color=c("white","black"), hjust=-0.1, size=5, fontface="bold")+

    scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
    scale_y_continuous(labels = scales::percent)+
  #xlab("Time spent\non review")+ylab("Freq")+
  xlab("")+ylab("Proportion of respondents")+
  coord_flip()+
  facet_grid(.~Venue)+theme(panel.grid.major.y = element_blank())
```


```{r review time single,fig.width=8,fig.height=3}
d %>% select(Journals=Q14_1,Conferences=Q14_2,Workshops=Q14_3) %>%
      gather(Venue,Time,Journals,Conferences,Workshops) %>%
      mutate(Time = factor(Time,levels(d$Q14_1),ordered=TRUE),
             Venue = factor(Venue,c("Journals","Conferences","Workshops"),ordered=TRUE)) %>%
      filter(!is.na(Time)) %>%
ggplot(aes(x=Time,group=Venue))+
  geom_bar(aes(y=..prop..),position="dodge")+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%")),stat="count",
            hjust="inward",color='black',size=3,position=position_dodge(width=1))+
  
    scale_y_continuous(labels = scales::percent)+
  #xlab("Time spent\non review")+ylab("Freq")+
  xlab("")+ylab("Proportion of respondents")+
  coord_flip()+
  facet_grid(.~Venue)+theme(panel.grid.major.y = element_blank())
```


```{r time on review by experience,fig.width=10,fig.height=5, warning=FALSE}
d %>% select(Q9,Journals=Q14_1,Conferences=Q14_2,Workshops=Q14_3) %>%
      gather(Venue,Time,Journals,Conferences,Workshops) %>%
      mutate(Time = factor(Time,levels(d$Q14_1),ordered=TRUE),
             Venue = factor(Venue,c("Journals","Conferences","Workshops"),ordered=TRUE)) %>%
      filter(!is.na(Time)) %>%
      group_by(Q9,Venue,Time) %>% count() %>%
      group_by(Q9,Venue) %>% mutate(Prop = n / sum(n)) %>% 
      rename(measure=Time) %>%
      show_parallel("Q9","Prop") +
      facet_grid(Venue ~ .)

```

### Times re-read (`Q16`)

```{r review re-read times}
d %>% select(Q4,Journals=Q16_1,Conferences=Q16_2) %>%
      gather(Venue,Time,Journals,Conferences) %>%
      mutate(Time = factor(Time,levels(d$Q16_1),ordered=TRUE),
             Venue = factor(Venue,c("Journals","Conferences"),ordered=TRUE)) %>%
      filter(!is.na(Time)) %>%
ggplot(aes(x=Time,fill=Q4,group=paste0(Q4,Venue)))+
  geom_bar(aes(y=..prop..),position="dodge")+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"%")),stat="count",
            hjust="inward",color='black',size=3,position=position_dodge(width=1))+
    scale_fill_brewer(name="Recipient",palette=3,type="qual")+
    scale_y_continuous(labels = scales::percent)+
  xlab("Number of times\nread the paper")+ylab("Freq")+
  coord_flip()+
  facet_grid(.~Venue)+theme(panel.grid.major.y = element_blank())
```


```{r review re-read times by experience,fig.width=10,fig.height=5, warning=FALSE}
d %>% select(Q9,Journals=Q16_1,Conferences=Q16_2,Workshops=Q14_3) %>%
      gather(Venue,Time,Journals,Conferences) %>%
      mutate(Time = factor(Time,levels(d$Q16_1),ordered=TRUE),
             Venue = factor(Venue,c("Journals","Conferences"),ordered=TRUE)) %>%
      filter(!is.na(Time)) %>%
      group_by(Q9,Venue,Time) %>% count() %>%
      group_by(Q9,Venue) %>% mutate(Prop = n / sum(n)) %>% 
      rename(measure=Time) %>%
      show_parallel("Q9","Prop") +
      facet_grid(Venue ~ .)

```

### Work method (`Q15`)

```{r work method,fig.width=7,fig.height=4}
d %>% filter(!is.na(Q15)) %>% 
      mutate(Q15 = factor(Q15,names(sort(table(Q15))),ordered=TRUE)) %>% 
ggplot(aes(x=fctr_wrap(Q15,40),fill=Q4,group=Q4))+
  geom_bar(aes(y=..prop..),position="dodge")+
  #geom_text(aes(y=..prop..,group=NA,label=..count..),stat="count",hjust=1.2,color="white")+
  geom_text(aes(y=..prop..,label=paste0(" ",round(..prop..*100),"% (",..count..,") "),hjust=(..prop..>0.5)),stat="count", position=position_dodge(1),color='black',size=2)+
  annotate("text",x=c(4.25,3.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  xlab("")+ylab("Proportion of respondents")+
  scale_y_continuous(labels = scales::percent)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  coord_flip()+theme(panel.grid.major.y = element_blank())
```

```{r work method by experience,fig.width=10,fig.height=4}
d %>% filter(!is.na(Q15)) %>% 
      mutate(Q15 = factor(Q15,names(sort(table(Q15))),ordered=TRUE)) %>% 
      group_by(Q9,Q15) %>% tally %>% 
      mutate(Prop = n / sum(n), measure = fctr_wrap(Q15,40)) %>%
      show_parallel("Q9","Prop") 

```


### Student involvement (`Q18`)

```{r student involvement}
involvement.popularity <- d%>%count_all_options("Q18","Way")

inv.levels = str_wrap(levels(involvement.popularity$Way),60)

involvement.popularity %>% #mutate(Way = factor(str_wrap(Way,60),inv.levels,ordered=TRUE)) %>%
ggplot(aes(x=fctr_wrap(Way,50),y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents involving student in that way")+
  xlab("Method")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r student involvement by experiences,fig.width=10,fig.height=4}
student.popularity.exp <- d%>%count_all_options("Q18","Method",by="Q9")

student.popularity.exp %>% 
  rename(measure=Method) %>%
  show_parallel("Q9","Prop")
```
### Sections important for reviewing (`Q19`)

```{r section relevance, fig.width=12,fig.height=5}
section.relevance <- d %>% filter(!is.na(Q4)) %>% analyze_ranks("Q19","Section") %>% showrank("Section")

section.relevance$summary %>% spread(Q4,MeanRank) %>%
                          mutate(Section = str_replace(Section,"\n"," ")) %>% 
                          kable
section.relevance$plot
```

Other:

```{r other sections}
d %>% select(Q4,Q19_7,Q19_7_TEXT) %>% filter(!is.na(Q19_7_TEXT)) %>% arrange(as.character(Q19_7)) %>% kable

```

```{r section relevance by experience, fig.width=12,fig.height=5}
d %>% parallel_ranks("Q19")

```


### Top three features leading to negative reviews(`Q22`)

```{r features leading to negative reviews,fig.width=8,fig.height=6}
involvement.popularity <- d%>%count_all_options("Q22","Cue")

inv.levels = str_wrap(levels(involvement.popularity$Cue),60)

involvement.popularity %>% mutate(Cue = factor(str_wrap(Cue,60),inv.levels,ordered=TRUE)) %>%
ggplot(aes(x=Cue,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
#  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "), color=Prop>0.2&Q4=="Award", hjust=Prop>0.2), position=position_dodge(1), size=2)+
  scale_color_manual(values=c("black","white"),guide=FALSE)+

  annotate("text",x=c(9.25,8.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion of respondents deeming feature leads to negative review")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```


```{r features leading to negative reviews by experience,fig.width=10,fig.height=5}

involvement.popularity.exp <- d%>%count_all_options("Q22","Cue",by="Q9")

involvement.popularity.exp %>% 
  rename(measure=Cue) %>%
  show_parallel("Q9","Prop")
```

### Top three features leading to positive reviews (`Q23`)

```{r features leading to positive reviews,fig.width=8,fig.height=5}
positive.popularity <- d%>%count_all_options("Q23","Cue")

positive.popularity <- rbind(positive.popularity,
                             list(Q4="Award",Cue="Citations to my work",n=0,N=61,Prop=0.0))
inv.levels = str_wrap(levels(positive.popularity$Cue),60)

positive.popularity %>% mutate(Cue = factor(str_wrap(Cue,60),inv.levels, ordered=TRUE)) %>%
ggplot(aes(x=Cue,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge",width=0.8)+
#  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "),color=Prop>0.5&Q4=="Award", hjust=Prop>0.5), position=position_dodge(1), size=2)+
  scale_color_manual(values=c("black","white"),guide=FALSE)+
  annotate("text",x=c(7.25,6.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+
  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion of respondents deeming feature leads to positive review")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r features leading to positive reviews by experiences,fig.width=10,fig.height=4}
positive.popularity.exp <- d%>%count_all_options("Q23","Cue",by="Q9")

positive.popularity.exp %>% 
  rename(measure=Cue) %>%
  show_parallel("Q9","Prop")
```


### Impact of guidelines (`Q24`)

```{r guidelines impact}
d %>% filter(!is.na(Q4)) %>% 
ggplot(aes(x=fctr_wrap(Q24,40),fill=Q4,group=Q4))+
  geom_bar(aes(y=..prop..),position="dodge")+
  geom_text(aes(y=..prop..,label=paste0(round(..prop..*100),"% (",..count..,")")),stat="count",
            position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion of respondents claiming such impact of guidelines")+
  xlab("Impact")+
  coord_flip()+theme(panel.grid.major.y = element_blank())
```

```{r guidelines impact by experience,fig.width=10,fig.height=4}
d %>% filter(!is.na(Q24)) %>% 
      mutate(Q15 = factor(Q24,names(sort(table(Q24))),ordered=TRUE)) %>% 
      group_by(Q9,Q24) %>% tally %>% 
      mutate(Prop = n / sum(n), measure = fctr_wrap(Q24,40)) %>%
      show_parallel("Q9","Prop") 

```

### Tools/Artifact validation (`Q25`)

```{r tool validation, fig.widht=8,fig.height=5}
tool.popularity <- d%>%mutate(Q25=gsub("data, scripts,","data/ scripts",Q25))%>%
                       count_all_options("Q25","Approach")

tool.popularity %>%
ggplot(aes(x=fctr_wrap(Approach,40),y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "), color=Prop>0.5&Q4=="Award"), position=position_dodge(1), hjust="inward", size=2)+
  scale_color_manual(values=c("black","white"), guide="none")+
    annotate("text",x=c(8.25,7.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents using that approach")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r tool validation by experiences,fig.width=10,fig.height=4}
validation.popularity.exp <- d%>%mutate(Q25=gsub("data, scripts,","data/ scripts",Q25))%>%
                                          count_all_options("Q25","Approach",by="Q9")

validation.popularity.exp %>% 
  rename(measure=Approach) %>%
  show_parallel("Q9","Prop")
```
### Conference management tools (`Q27`)

```{r chairing tools relevance}

tool.relevance <- d %>% filter(!is.na(Q4)) %>% analyze_grade("Q27","Tool")

tool.relevance.summary <- tool.relevance %>% 
                     group_by(Q4,Tool) %>% 
                     summarize(MeanGrade = mean(as.numeric(Grade),na.rm=TRUE)) 
tool.relevance.summary %>% spread(Q4,MeanGrade) %>% kable
```

```{r chairing tools,fig.width=9,fig.height=4}
tool.relevance %>% group_by(Q4,Tool,Grade)%>%tally()%>%
  group_by(Tool,Q4)%>%mutate(Prop=n/sum(n)) %>%
ggplot(aes(x=Grade,y=Prop,fill=is.na(Grade)))+
  geom_text(aes(label=paste0(round(Prop*100),"%")),vjust=1.1,color='black',fontface="bold",size=2)+
  scale_fill_manual(values=c("steelblue","gray"),guide=FALSE)+
  geom_bar(stat="identity")+
  geom_text(aes(label=paste0(round(Prop*100),"%")),vjust=1.1,color='white',size=2)+
  scale_y_continuous(labels = scales::percent_format(accuracy = 1))+     
  xlab("Ease of use")+ylab("Prop")+
  coord_flip()+
  facet_grid(Q4~Tool)+theme(panel.grid.major.y = element_blank())
```

Other chairing tools

```{r other chairing tools table}
other.tools <- d %>% filter(!is.na(Q27_4_TEXT)) %>% 
      select(Q4,Tool=Q27_4_TEXT,Grade=Q27_4) %>%
      mutate(Grade=factor(Grade,levels(tool.relevance$Grade),ordered=TRUE)) %>%
      arrange(Tool)
other.tools %>% kable
```

```{r other chairing tools,fig.width=10,fig.height=4}
#tool.relevance %>% filter(Tool!="Other") %>% rbind(other.tools) %>%
other.tools %>%
  group_by(Q4,Tool,Grade)%>%tally()%>%group_by(Tool,Q4)%>%mutate(Prop=n/sum(n),N=sum(n)) %>%
ggplot(aes(x=Grade,y=Prop,fill=is.na(Grade)))+
  geom_text(aes(label=paste0(round(Prop*100),"%")),vjust=1.1,color='black',fontface="bold",size=2)+
  scale_fill_manual(values=c("steelblue","gray"),guide=FALSE)+
  geom_bar(stat="identity")+
  geom_text(aes(label=paste0(round(Prop*100),"%")),vjust=1.1,color='white',size=2)+
  scale_y_continuous(breaks=c(0,.5,1),labels = scales::percent_format(accuracy = 1))+     
  xlab("Ease of use")+ylab("Prop")+
  coord_flip()+
  facet_grid(Q4~paste0(Tool,"\n(",N," resp.)"))+theme(panel.grid.major.y = element_blank())

```

### Review model  (`Q28`)

```{r review model}
review.model.popularity <- d %>% mutate(Q28=gsub(", "," ",Q28)) %>%
                               count_all_options("Q28","Model")

review.model.popularity %>%
ggplot(aes(x=fctr_wrap(Model,40),y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents")+
  xlab("Review Model")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r review model by experience ,fig.width=10,fig.height=4}
review.model.popularity.exp <- d %>% mutate(Q28=gsub(", "," ",Q28)) %>%
                               count_all_options("Q28","Model",by="Q9")

review.model.popularity.exp %>% 
  rename(measure=Model) %>%
  show_parallel("Q9","Prop")

```
### Effort recognition  (`Q30`)

```{r recognition relevance,fig.width=12,fig.height=6,warning=FALSE,error=FALSE}
recognition.relevance <- d %>% filter(!is.na(Q4)) %>% analyze_grade("Q30","Recognition") %>% showgrade("Recognition",scale_boundaries=TRUE)


recognition.relevance$summary %>% spread(Q4,MeanGrade) %>% 
            mutate(Recognition = str_replace_all(Recognition,"\n"," ")) %>% 
                     kable

recognition.relevance$plot

```

```{r other recognition}
d %>% filter(!is.na(Q4),!is.na(Q30_6_TEXT)) %>% select(Q4,Q30_6_TEXT) %>% 
      kable(col.names = c("Recipient","Other recognition") )
```

```{r recognition relevance by experience,fig.width=12,fig.height=6,warning=FALSE,error=FALSE}
recognition.relevance.exp <- d %>% filter(!is.na(Q9)) %>% analyze_grade("Q30","Recognition",by="Q9") %>% showgrade("Recognition",scale_boundaries=TRUE,by="Q9")


recognition.relevance.exp$summary %>% 
  mutate(measure=Recognition) %>% 
  show_parallel(score = "MeanGrade",by="Q9")

```



### Impactful papers  (`Q31`)

```{r impactful papers, fig.widht=8, fig.heigh=4}
full.impact = d%>% select(Q4) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q31)%>%distinct() %>% na.omit)
d %>% filter(!is.na(Q4),!is.na(Q31)) %>%
      group_by(Q4,Q31) %>% tally() %>%
      full_join(full.impact,by = c("Q4", "Q31")) %>% mutate(n=coalesce(n,as.integer(0))) %>%
      group_by(Q4) %>% mutate(Prop=n/sum(n)) %>% ungroup %>%
      arrange(desc(Q4),Prop) %>% mutate(Q31=factor(Q31,unique(Q31),ordered=TRUE)) %>%
ggplot(aes(x=Q31,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "),color=Prop>0.15&Q4=="Award", hjust=Prop>0.15), position=position_dodge(1), size=2)+
  scale_color_manual(values=c("black","white"),guide=FALSE)+
    annotate("text",x=c(6.25,5.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+

  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion of respondents")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```


```{r impactful papers by experience, fig.width=10,fig.height=4}
full.impact.exp = d%>% select(Q9) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q31)%>%distinct() %>% na.omit)
d %>% filter(!is.na(Q9)) %>%
      group_by(Q9,Q31) %>% tally() %>%
      full_join(full.impact.exp,by = c("Q9", "Q31")) %>% mutate(n=coalesce(n,as.integer(0))) %>%
      group_by(Q9) %>% mutate(Prop=n/sum(n)) %>%
      mutate(measure = Q31) %>%
show_parallel(by="Q9",score="Prop")
```


### Features affecting PC/Editor decision  (`Q32`)

```{r effect on decision}
full.decision = d%>% select(Q4) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q32)%>%distinct() %>% na.omit)
d %>% filter(!is.na(Q4)) %>%
      group_by(Q4,Q32) %>% tally() %>%
      full_join(full.decision,by = c("Q4", "Q32")) %>% mutate(n=coalesce(n,as.integer(0))) %>%
      group_by(Q4) %>% mutate(Prop=n/sum(n)) %>%
ggplot(aes(x=Q32,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Number of respondents")+
  xlab("Feature")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

```{r  effect on decision by experience, fig.width=10,fig.height=4}
full.decision.exp = d%>% select(Q9) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q32)%>%distinct() %>% na.omit)
d %>% filter(!is.na(Q9)) %>%
      group_by(Q9,Q32) %>% tally() %>%
      full_join(full.decision.exp,by = c("Q9", "Q32")) %>% mutate(n=coalesce(n,as.integer(0))) %>%
      group_by(Q9) %>% mutate(Prop=n/sum(n)) %>%
      mutate(measure = Q32) %>%
show_parallel(by="Q9",score="Prop")
```

## Opinion on (received) reviews

### Observed quality of reviews (`Q33`)

```{r observed review quality distribution,fig.width=9,fig.height=5}
review.quality <- d %>% filter(!is.na(Q4)) %>% 
                    analyze_grade("Q33","Venue") %>% showgrade("Venue")


review.quality$summary %>% spread(Q4,MeanGrade) %>% 
            mutate(Venue = str_replace_all(Venue,"\n"," ")) %>% 
                     kable

review.quality$plot

```



### Recommendation to increase acceptance chances (`Q34`)

```{r increase acceptance, fig.width=8, fig.height=4}
full.acceptance = d%>% select(Q4) %>% unique() %>% na.omit %>% 
              crossing(d%>%select(Q34)%>%distinct() %>% na.omit)
d %>% filter(!is.na(Q4)) %>%
      group_by(Q4,Q34) %>% tally() %>%
      full_join(full.acceptance,by = c("Q4", "Q34")) %>% 
      mutate(n=coalesce(n,as.integer(0))) %>%
      group_by(Q4) %>% mutate(Prop=n/sum(n)) %>% ungroup %>%
      arrange(desc(Q4),Prop) %>% mutate(Q34=factor(Q34,unique(Q34),ordered=TRUE)) %>%
ggplot(aes(x=Q34,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
#  geom_text(aes(label=paste0(round(Prop*100),"% (",n,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  geom_text(aes(label=paste0(" ",round(Prop*100),"% (",n,") "),color=Prop>0.3&Q4=="Award", hjust=Prop>0.3), position=position_dodge(1), size=2)+
  scale_color_manual(values=c("black","white"),guide=FALSE)+
  annotate("text",x=c(7.25,6.75),y=c(0,0),label=c("Award","NoAward"),color=c("white","black"),hjust=-0.1,size=4,fontface="bold")+
  scale_fill_brewer(name="Recipient",palette=3,type="qual",guide="none")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion of respondents")+
  xlab("")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```

### Characteristics of a good peer review (`Q35`)

```{r good review,fig.width=10,fig.height=5}
good.relevance <- d %>% filter(!is.na(Q4)) %>% 
                  analyze_ranks("Q35","Characteristic") %>%
                  mutate(Characteristic = fctr_replace(Characteristic,":| \\(.*","")) %>%
                  showrank("Characteristic")

good.relevance$summary %>% spread(Q4,MeanRank) %>% kable

good.relevance$plot

```

```{r good reviewe by experience, fig.width=12,fig.height=5}
d %>% parallel_ranks("Q35")

```

```{r good review mean,fig.width=7, fig.height=4}
ggplot(good.relevance$summary,aes(x=MeanRank,y=Characteristic,color=Q4,shape=Q4))+
  geom_point(size=3)+
  scale_shape(name="Recipient") +
  scale_color_brewer(name="Recipient",type="qual",palette=3)+
  scale_x_continuous(limits=c(1,8),breaks=1:8)+
#  scale_x_reverse()+
  geom_text_repel(aes(label=round(MeanRank,1)),color="navy",size=3,nudge_y=0.4,direction="x")+
  scale_y_discrete(limits=rev(unique(good.relevance$summary$Characteristic)))+
  xlab("Mean rank")

```

### Important elements to provide in the review (`Q36`)

```{r points in review,fig.width=18,fig.height=5}
points.relevance <- d %>% filter(!is.na(Q4)) %>% 
                  analyze_grade("Q36","Point") %>%
                  showgrade("Point",scale_boundaries = TRUE)

points.relevance$summary %>% spread(Q4,MeanGrade) %>% mutate(Point=str_replace_all(Point,"\n"," ")) %>% kable

points.relevance$plot

```


```{r important points dots}
points.relevance$summary %>%
ggplot(aes(x=Point,y=6-MeanGrade,color=Q4,shape=Q4))+
  geom_point(size=3)+
  scale_shape(name="Recipient") +
  scale_color_brewer(name="Recipient",type="qual",palette=3)+
  scale_x_discrete(limits=rev(levels(points.relevance$ranks$Point)))+
  scale_y_continuous(breaks=1:5,labels=rev(sub(" (?=[^ ]+$)","\n",levels(points.relevance$ranks$Grade),perl=TRUE)),limits =c(1,5))+
  ylab("Mean importance score")+
  coord_flip()

```


```{r points relevance by experience,fig.width=12,fig.height=6,warning=FALSE,error=FALSE}
points.relevance.exp <- d %>% filter(!is.na(Q9)) %>% analyze_grade("Q36","Point",by="Q9") %>% showgrade("Point",scale_boundaries=TRUE,by="Q9")


points.relevance.exp$summary %>% 
  mutate(measure=Point) %>% 
  show_parallel(score = "MeanGrade",by="Q9")

```

# Qualitative analysis

## Quality assessment (`Q20`)
```{r quality assessment,fig.width=7,fig.height=10}
d %>% select(Q4,starts_with("Q20_")) %>% 
      group_by(Q4) %>% mutate(N=length(Q4)) %>%
      gather("Code","Present",starts_with("Q20_")) %>%
      #mutate(Code = str_replace(Code,"Q20_(.)\\.(.*)","\\1:\\2")) %>%
      extract(Code,c("Category","Code"),"Q20_([^.]+)\\.?(.*)?") %>%
      mutate( Category = factor(Category,names(sort(-table(Category))),ordered=TRUE )) %>%
      group_by(Q4,Category,Code,N) %>% summarize(Freq=sum(Present)) %>%
      group_by(Q4,Category) %>% mutate(Prop = Freq / N) %>%
      arrange(Category,desc(Q4),Prop) %>% ungroup() %>%
      mutate(Code = factor(Code,unique(Code),ordered=TRUE)) %>%


ggplot(aes(x=Code,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",Freq,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion mentioned factor")+
  xlab("Quality\nassessment\nfactor")+
  facet_grid(Category~.,scales="free_y",space="free_y")+
  coord_flip()+theme(panel.grid.major.y = element_blank())

```



## Quality review (author's viewpoint) (`Q53`)
```{r quality review,fig.width=6,fig.height=7}

d %>% select(Q4,starts_with("Q53_")) %>% 
      group_by(Q4) %>% mutate(N=length(Q4)) %>%
      gather("Code","Present",starts_with("Q53_")) %>%
      mutate(Code = str_remove(Code,"^Q53_")) %>%
      mutate(Code = str_replace(Code,"\\."," ")) %>%
      group_by(Q4,Code,N) %>% summarize(Freq=sum(Present)) %>%
      group_by(Q4) %>% mutate(Prop = Freq / N) %>%
      ungroup %>% arrange(desc(Q4),Prop) %>%
      mutate(Code = factor(Code,unique(Code),ordered=TRUE)) %>%


ggplot(aes(x=Code,y=Prop,fill=Q4))+
  geom_bar(stat="identity",position="dodge")+
  geom_text(aes(label=paste0(round(Prop*100),"% (",Freq,")")), position=position_dodge(1),color='black',hjust="inward",size=2)+
  scale_fill_brewer(name="Recipient",palette=3,type="qual")+
  scale_y_continuous(labels = scales::percent)+
  ylab("Proportion mentioned factor")+
  xlab("Factor")+
  coord_flip()+theme(panel.grid.major.y = element_blank())
```


