This R Notebook supports the electronic laboratory notebook (ELN) suvey data shown in the publication: “Considerations for Implementing Electronic Laboratory Notebooks in an Academic Research Environment”, S.G. Higgins, A.A. Nogiwa-Valdez, M.M. Stevens (2021).

Configure environment

Load required packages:

library(here)
here() starts at /Users/stuart/OneDrive - Imperial College London/_Papers/ELN-essay/product_survey_revised
library(tidyverse)
Registered S3 methods overwritten by 'dbplyr':
  method         from
  print.tbl_lazy     
  print.tbl_sql      
── Attaching packages ──────────────────────────────────────────────────────────────── tidyverse 1.3.0 ──
✓ ggplot2 3.3.3     ✓ purrr   0.3.4
✓ tibble  3.0.6     ✓ dplyr   1.0.4
✓ tidyr   1.1.2     ✓ stringr 1.4.0
✓ readr   1.4.0     ✓ forcats 0.5.1
── Conflicts ─────────────────────────────────────────────────────────────────── tidyverse_conflicts() ──
x dplyr::filter() masks stats::filter()
x dplyr::lag()    masks stats::lag()
library(plotly)
Registered S3 method overwritten by 'data.table':
  method           from
  print.data.table     
Registered S3 methods overwritten by 'htmltools':
  method               from         
  print.html           tools:rstudio
  print.shiny.tag      tools:rstudio
  print.shiny.tag.list tools:rstudio
Registered S3 method overwritten by 'htmlwidgets':
  method           from         
  print.htmlwidget tools:rstudio

Attaching package: ‘plotly’

The following object is masked from ‘package:ggplot2’:

    last_plot

The following object is masked from ‘package:stats’:

    filter

The following object is masked from ‘package:graphics’:

    layout
library(htmlwidgets)

Import data

Load in survey data from file, recode ‘ongoing’ tags in the date_defunct column to the year 2021, calculate the total number of years active, and create a logical vector for each row determining whether the ELN is active or not in the year 2021: (Note: this Notebook expects file ‘ELN_Review_Higgins_2021_Survey.csv’ to be present in the same directory as the working directory identified by the here package)

data <-
  read_csv(here("ELN_Review_Higgins_2021_Survey.csv")) %>%
  mutate(date_defunct_numeric = as.numeric(replace(date_defunct, date_defunct == "ongoing", 2021)),
         years_active = date_defunct_numeric - date_released,
         defunct_in_2021 =
           case_when(
             date_defunct == "ongoing" ~ FALSE,
             date_defunct == 2021 ~ TRUE,
             TRUE ~ TRUE
           ),
         row_number = row_number())

── Column specification ─────────────────────────────────────────────────────────────────────────────────
cols(
  product_name = col_character(),
  manufacturer = col_character(),
  date_released = col_double(),
  date_defunct = col_character(),
  codebase = col_character(),
  notes = col_character(),
  reference_1 = col_character(),
  reference_2 = col_character(),
  reference_3 = col_character(),
  reference_4 = col_character(),
  references_accessed = col_character()
)

Generate statistics

How many ELNs were surveyed?

data %>%
  count()

How many of the ELNs surveyed are active (FALSE) or defunct (TRUE) in 2021?

data %>%
  count(defunct_in_2021)

What is the average (and spread) of the lifetime (years_active) of the ELNs surveyed? (Note: the median absolute estimate here has a default scaling constant of 1.4826, so that it acts as as a consistent estimator of the standard deviation)

data %>%
  summarise(mean_years_active = mean(years_active),
            sd_years_active = sd(years_active),
            median_years_active = median(years_active),
            mad_years_active = mad(years_active),
            iqr_years_active = IQR(years_active),
            range_years_active = max(years_active)-min(years_active))

What are the average and spread of the lifetimes of ELNs, sub-divided by codebase?

data %>%
  group_by(codebase) %>%
  summarise(mean_years_active = mean(years_active),
            sd_years_active = sd(years_active),
            median_years_active = median(years_active),
            mad_years_active = mad(years_active),
            iqr_years_active = IQR(years_active),
            range_years_active = max(years_active)-min(years_active))

How many of the ELNs surveyed have open-source or proprietary codebases?

data %>%
  count(codebase)

Which are the longest running proprietary and open source ELNs (in the survey data)?

data %>%
  group_by(codebase) %>%
  slice_max(n=1, order_by=years_active) %>%
  select(product_name, manufacturer, years_active, date_defunct, codebase)

Generate figures

Define a theme for plotting figures:

mytheme <-
  theme_bw() +
  theme(
    panel.background = element_rect(fill = "white", colour = "black", size = 2),
    panel.grid.minor = element_blank(),
    panel.grid.major = element_blank(),
    text = element_text(size = 25, face = "plain", colour = "black"),
    axis.title.x = element_text(size = 25, face = "plain"),
    axis.title.y = element_text(size = 25),
    element_line(size = 2),
    axis.ticks.length = unit(0.15, "cm"))

Define functions for customising the appearance of plotted figures:

get_point_colour <- function(x){
  ifelse(x==TRUE, "grey", "grey30")
}

get_line_colour <- function(x){
  ifelse(x!="opensource", "#0072B2", "#CC79A7")
}

Produce the timeline plot featured in Figure 1 of the main manuscript:

p_timeline <-
  data %>%
  mutate(row_number = as_factor(row_number)) %>%
  mutate(row_number = fct_reorder(fct_reorder(row_number, years_active, .desc=FALSE), codebase, .desc=FALSE)) %>%
  mutate(row_number_new = as.numeric(row_number)) %>%
  ggplot() +
  geom_segment(aes(x=date_released, xend=date_defunct_numeric,y=row_number_new, yend=row_number_new),
               colour=get_line_colour(data$codebase),
               linetype="solid",
               size=0.5) +
  geom_point(aes(x=date_released, y=row_number_new), colour=get_point_colour(data$defunct_in_2021), shape=1, size=2 ) +
  geom_point(aes(x=date_defunct_numeric, y=row_number_new), colour=get_point_colour(data$defunct_in_2021), shape=16, size=0.5) + 
  scale_x_continuous(position="bottom", breaks=c(seq(1980,2021,5))) +
  coord_cartesian(xlim=c(1980,2021)) +
  theme_bw() +
  theme(
    plot.margin = margin(0.1, 0.1, 0.1, 0.1, "cm"),
    panel.border = element_blank(),
    panel.grid.major.y = element_line(colour="grey95", size=0.25),
    panel.grid.major.x = element_line(colour="grey95", size=0.25),
    panel.grid.minor.x = element_line(colour="grey95", size=0.25),
    axis.text.y = element_blank(),
    axis.text.x = element_blank(),
    axis.title.y = element_blank(),
    axis.title.x = element_blank(),
    axis.ticks.y = element_blank(),
    axis.ticks.x = element_blank(),
    legend.position = "bottom"
  )

print(p_timeline)

ggsave(here("ELN_Review_Higgins_2021_Timeline.pdf"), plot=p_timeline, width=18.0, height=10, device="pdf", dpi=600, units="cm")

Generate a separate interactive version of the timeline shown in Figure 1, and export as a standalone HTML file using plotly and htmlwidgets:

p_interactive <-
  data %>%
  mutate(row_number = as_factor(row_number)) %>%
  mutate(row_number = fct_reorder(fct_reorder(row_number, years_active, .desc=FALSE), codebase, .desc=FALSE)) %>%
  mutate(row_number_new = as.numeric(row_number)) %>%
  mutate(codebase_name = recode(codebase, proprietary = "Proprietary", opensource = "Open-source"),
         marker_colour = if_else(defunct_in_2021 == TRUE, "#C0C0C0", "#4d4d4d"),
         hovertext =
           paste0("<b>",
                  product_name,
                  "</b><br>Date released: ",
                  date_released,
                  "<br>Date defunct: ",
                  date_defunct,
                  "<extra></extra>")
  ) %>%
  plot_ly() %>%
  add_segments(
    x = ~date_released, y = ~row_number_new,
    xend = ~date_defunct_numeric, yend ~row_number_new,
    color = ~factor(codebase_name),
    colors = c("#CC79A7", "#0072B2")
  ) %>%
  add_markers(
    x = ~date_released,
    y = ~row_number_new,
    text = ~product_name,
    name = "Date released",
    customdata = ~hovertext,
    hovertemplate = "%{customdata}",
    marker =
      list(
        symbol = "circle-open",
        size = 10,
        color = ~marker_colour
      ),
    inherit = FALSE
  ) %>%
  add_markers(
    x = ~date_defunct_numeric,
    y = ~row_number_new,
    text = ~product_name,
    name = "Date defunct",
    customdata = ~hovertext,
    hovertemplate = "%{customdata}",
    marker =
      list(
        size = 3,
        color = ~marker_colour
      )
  ) %>%
  layout(
    title = "How long do electronic laboratory notebooks last?",
    xaxis = list(title = "Timeline of ELN products"),
    yaxis = list(title = "Lifetimes of surveyed ELN products",
                 showticklabels = FALSE,
                 zeroline = FALSE,
                 showline = FALSE)
  )
saveWidget(partial_bundle(p_interactive), here("ELN_Review_Higgins_2021_Lifetimes_Interactive_Figure1.html"), selfcontained = TRUE, title = "ELN survey")

Generate a plot showing the total number of new proprietary and open-source ELNs per year, as featured in Figure 1: (Note: axes for this plot were manually appended late in graphics software)

data_summarised <-
  data %>%
  group_by(date_released, codebase) %>%
  summarise(count = n(), .groups="drop_last")

p_releases <-
  data_summarised %>%
  ggplot(aes(x=date_released, y=as.factor(codebase), size=count)) +
  geom_point(shape=21, fill=get_line_colour(data_summarised$codebase),alpha=0.5) +
  scale_size(range=c(1,10)) +
  scale_x_continuous(position="bottom", breaks=c(seq(1980,2021,5))) +
  coord_cartesian(xlim=c(1980,2021)) +
  theme_bw() +
  theme(
    plot.margin = margin(0.1, 0.1, 0.1, 0.1, "cm"),
    panel.border = element_blank(),
    panel.grid.major.y = element_blank(),
    panel.grid.major.x = element_line(colour="grey95", size=0.25),
    panel.grid.minor.x = element_line(colour="grey95", size=0.25),
    axis.title.y = element_blank(),
    axis.title.x = element_blank(),
    axis.ticks.y = element_blank(),
    axis.ticks.x = element_blank(),
    axis.text.y = element_blank(),
    legend.position = "none"
  )

print(p_releases)

ggsave(here("ELN_Review_Higgins_2021_Releases-Per-Year.pdf"), plot=p_releases, width=18.0, height=2.5, device="pdf", dpi=600, units="cm")

Session information

sessionInfo()
R version 4.0.3 (2020-10-10)
Platform: x86_64-apple-darwin17.0 (64-bit)
Running under: macOS Big Sur 10.16

Matrix products: default
LAPACK: /Library/Frameworks/R.framework/Versions/4.0/Resources/lib/libRlapack.dylib

locale:
[1] en_GB.UTF-8/en_GB.UTF-8/en_GB.UTF-8/C/en_GB.UTF-8/en_GB.UTF-8

attached base packages:
[1] stats     graphics  grDevices utils     datasets  methods   base     

other attached packages:
 [1] htmlwidgets_1.5.3 plotly_4.9.3      forcats_0.5.1     stringr_1.4.0     dplyr_1.0.4      
 [6] purrr_0.3.4       readr_1.4.0       tidyr_1.1.2       tibble_3.0.6      ggplot2_3.3.3    
[11] tidyverse_1.3.0   here_1.0.1       

loaded via a namespace (and not attached):
 [1] tidyselect_1.1.0  xfun_0.20         haven_2.3.1       colorspace_2.0-0  vctrs_0.3.6      
 [6] generics_0.1.0    htmltools_0.5.1.1 viridisLite_0.3.0 yaml_2.2.1        rlang_0.4.10     
[11] pillar_1.4.7      glue_1.4.2        withr_2.4.1       DBI_1.1.1         dbplyr_2.1.0     
[16] modelr_0.1.8      readxl_1.3.1      lifecycle_0.2.0   munsell_0.5.0     gtable_0.3.0     
[21] cellranger_1.1.0  rvest_0.3.6       labeling_0.4.2    knitr_1.31        crosstalk_1.1.1  
[26] curl_4.3          broom_0.7.4       Rcpp_1.0.6        scales_1.1.1      backports_1.2.1  
[31] jsonlite_1.7.2    farver_2.0.3      fs_1.5.0          hms_1.0.0         digest_0.6.27    
[36] stringi_1.5.3     grid_4.0.3        rprojroot_2.0.2   cli_2.3.0         tools_4.0.3      
[41] magrittr_2.0.1    lazyeval_0.2.2    crayon_1.4.0      pkgconfig_2.0.3   ellipsis_0.3.1   
[46] data.table_1.13.6 xml2_1.3.2        reprex_1.0.0      lubridate_1.7.9.2 assertthat_0.2.1 
[51] httr_1.4.2        rstudioapi_0.13   R6_2.5.0          compiler_4.0.3   
LS0tCnRpdGxlOiAiQ29kZSBmb3IgZ2VuZXJhdGluZyB0aGUgc3RhdGlzdGljcyBhbmQgZGF0YSBmaWd1cmVzIHByZXNlbnRlZCBpbiB0aGUgcHVibGljYXRpb24gJ0NvbnNpZGVyYXRpb25zIGZvciBJbXBsZW1lbnRpbmcgRWxlY3Ryb25pYyBMYWJvcmF0b3J5IE5vdGVib29rcyBpbiBhbiBBY2FkZW1pYyBSZXNlYXJjaCBFbnZpcm9ubWVudCciCmF1dGhvcjogU3R1YXJ0IEcuIEhpZ2dpbnMKZGF0ZTogImByIGZvcm1hdChTeXMudGltZSgpLCAnJWQgJUIgJVknKWAiCm91dHB1dDoKICBodG1sX25vdGVib29rOgogICAgdGhlbWU6IHNhbmRzdG9uZQogICAgdG9jOiB0cnVlCiAgICB0b2NfZGVwdGg6IDUKICAgIHRvY19mbG9hdDogdHJ1ZQotLS0KClRoaXMgUiBOb3RlYm9vayBzdXBwb3J0cyB0aGUgZWxlY3Ryb25pYyBsYWJvcmF0b3J5IG5vdGVib29rIChFTE4pIHN1dmV5IGRhdGEgc2hvd24gaW4gdGhlIHB1YmxpY2F0aW9uOiAiQ29uc2lkZXJhdGlvbnMgZm9yIEltcGxlbWVudGluZyBFbGVjdHJvbmljIExhYm9yYXRvcnkgTm90ZWJvb2tzIGluIGFuIEFjYWRlbWljIFJlc2VhcmNoIEVudmlyb25tZW50IiwgUy5HLiBIaWdnaW5zLCBBLkEuIE5vZ2l3YS1WYWxkZXosIE0uTS4gU3RldmVucyAoMjAyMSkuCgojIENvbmZpZ3VyZSBlbnZpcm9ubWVudAoKTG9hZCByZXF1aXJlZCBwYWNrYWdlczoKYGBge3J9CmxpYnJhcnkoaGVyZSkKbGlicmFyeSh0aWR5dmVyc2UpCmxpYnJhcnkocGxvdGx5KQpsaWJyYXJ5KGh0bWx3aWRnZXRzKQpgYGAKCiMgSW1wb3J0IGRhdGEKCkxvYWQgaW4gc3VydmV5IGRhdGEgZnJvbSBmaWxlLCByZWNvZGUgJ29uZ29pbmcnIHRhZ3MgaW4gdGhlIGBkYXRlX2RlZnVuY3RgIGNvbHVtbiB0byB0aGUgeWVhciAyMDIxLCBjYWxjdWxhdGUgdGhlIHRvdGFsIG51bWJlciBvZiB5ZWFycyBhY3RpdmUsIGFuZCBjcmVhdGUgYSBsb2dpY2FsIHZlY3RvciBmb3IgZWFjaCByb3cgZGV0ZXJtaW5pbmcgd2hldGhlciB0aGUgRUxOIGlzIGFjdGl2ZSBvciBub3QgaW4gdGhlIHllYXIgMjAyMTogKE5vdGU6IHRoaXMgTm90ZWJvb2sgZXhwZWN0cyBmaWxlICdFTE5fUmV2aWV3X0hpZ2dpbnNfMjAyMV9TdXJ2ZXkuY3N2JyB0byBiZSBwcmVzZW50IGluIHRoZSBzYW1lIGRpcmVjdG9yeSBhcyB0aGUgd29ya2luZyBkaXJlY3RvcnkgaWRlbnRpZmllZCBieSB0aGUgYGhlcmVgIHBhY2thZ2UpCmBgYHtyfQpkYXRhIDwtCiAgcmVhZF9jc3YoaGVyZSgiRUxOX1Jldmlld19IaWdnaW5zXzIwMjFfU3VydmV5LmNzdiIpKSAlPiUKICBtdXRhdGUoZGF0ZV9kZWZ1bmN0X251bWVyaWMgPSBhcy5udW1lcmljKHJlcGxhY2UoZGF0ZV9kZWZ1bmN0LCBkYXRlX2RlZnVuY3QgPT0gIm9uZ29pbmciLCAyMDIxKSksCiAgICAgICAgIHllYXJzX2FjdGl2ZSA9IGRhdGVfZGVmdW5jdF9udW1lcmljIC0gZGF0ZV9yZWxlYXNlZCwKICAgICAgICAgZGVmdW5jdF9pbl8yMDIxID0KICAgICAgICAgICBjYXNlX3doZW4oCiAgICAgICAgICAgICBkYXRlX2RlZnVuY3QgPT0gIm9uZ29pbmciIH4gRkFMU0UsCiAgICAgICAgICAgICBkYXRlX2RlZnVuY3QgPT0gMjAyMSB+IFRSVUUsCiAgICAgICAgICAgICBUUlVFIH4gVFJVRQogICAgICAgICAgICksCiAgICAgICAgIHJvd19udW1iZXIgPSByb3dfbnVtYmVyKCkpCmBgYAoKIyBHZW5lcmF0ZSBzdGF0aXN0aWNzCgpIb3cgbWFueSBFTE5zIHdlcmUgc3VydmV5ZWQ/CmBgYHtyfQpkYXRhICU+JQogIGNvdW50KCkKYGBgCgpIb3cgbWFueSBvZiB0aGUgRUxOcyBzdXJ2ZXllZCBhcmUgYWN0aXZlIChGQUxTRSkgb3IgZGVmdW5jdCAoVFJVRSkgaW4gMjAyMT8KYGBge3J9CmRhdGEgJT4lCiAgY291bnQoZGVmdW5jdF9pbl8yMDIxKQpgYGAKCldoYXQgaXMgdGhlIGF2ZXJhZ2UgKGFuZCBzcHJlYWQpIG9mIHRoZSBsaWZldGltZSAoYHllYXJzX2FjdGl2ZWApIG9mIHRoZSBFTE5zIHN1cnZleWVkPyAoTm90ZTogdGhlIG1lZGlhbiBhYnNvbHV0ZSBlc3RpbWF0ZSBoZXJlIGhhcyBhIGRlZmF1bHQgc2NhbGluZyBjb25zdGFudCBvZiAxLjQ4MjYsIHNvIHRoYXQgaXQgYWN0cyBhcyBhcyBhIGNvbnNpc3RlbnQgZXN0aW1hdG9yIG9mIHRoZSBzdGFuZGFyZCBkZXZpYXRpb24pCmBgYHtyfQpkYXRhICU+JQogIHN1bW1hcmlzZShtZWFuX3llYXJzX2FjdGl2ZSA9IG1lYW4oeWVhcnNfYWN0aXZlKSwKICAgICAgICAgICAgc2RfeWVhcnNfYWN0aXZlID0gc2QoeWVhcnNfYWN0aXZlKSwKICAgICAgICAgICAgbWVkaWFuX3llYXJzX2FjdGl2ZSA9IG1lZGlhbih5ZWFyc19hY3RpdmUpLAogICAgICAgICAgICBtYWRfeWVhcnNfYWN0aXZlID0gbWFkKHllYXJzX2FjdGl2ZSksCiAgICAgICAgICAgIGlxcl95ZWFyc19hY3RpdmUgPSBJUVIoeWVhcnNfYWN0aXZlKSwKICAgICAgICAgICAgcmFuZ2VfeWVhcnNfYWN0aXZlID0gbWF4KHllYXJzX2FjdGl2ZSktbWluKHllYXJzX2FjdGl2ZSkpCmBgYAoKV2hhdCBhcmUgdGhlIGF2ZXJhZ2UgYW5kIHNwcmVhZCBvZiB0aGUgbGlmZXRpbWVzIG9mIEVMTnMsIHN1Yi1kaXZpZGVkIGJ5IGNvZGViYXNlPwpgYGB7cn0KZGF0YSAlPiUKICBncm91cF9ieShjb2RlYmFzZSkgJT4lCiAgc3VtbWFyaXNlKG1lYW5feWVhcnNfYWN0aXZlID0gbWVhbih5ZWFyc19hY3RpdmUpLAogICAgICAgICAgICBzZF95ZWFyc19hY3RpdmUgPSBzZCh5ZWFyc19hY3RpdmUpLAogICAgICAgICAgICBtZWRpYW5feWVhcnNfYWN0aXZlID0gbWVkaWFuKHllYXJzX2FjdGl2ZSksCiAgICAgICAgICAgIG1hZF95ZWFyc19hY3RpdmUgPSBtYWQoeWVhcnNfYWN0aXZlKSwKICAgICAgICAgICAgaXFyX3llYXJzX2FjdGl2ZSA9IElRUih5ZWFyc19hY3RpdmUpLAogICAgICAgICAgICByYW5nZV95ZWFyc19hY3RpdmUgPSBtYXgoeWVhcnNfYWN0aXZlKS1taW4oeWVhcnNfYWN0aXZlKSkKYGBgCgpIb3cgbWFueSBvZiB0aGUgRUxOcyBzdXJ2ZXllZCBoYXZlIG9wZW4tc291cmNlIG9yIHByb3ByaWV0YXJ5IGNvZGViYXNlcz8KYGBge3J9CmRhdGEgJT4lCiAgY291bnQoY29kZWJhc2UpCmBgYAoKV2hpY2ggYXJlIHRoZSBsb25nZXN0IHJ1bm5pbmcgcHJvcHJpZXRhcnkgYW5kIG9wZW4gc291cmNlIEVMTnMgKGluIHRoZSBzdXJ2ZXkgZGF0YSk/CmBgYHtyfQpkYXRhICU+JQogIGdyb3VwX2J5KGNvZGViYXNlKSAlPiUKICBzbGljZV9tYXgobj0xLCBvcmRlcl9ieT15ZWFyc19hY3RpdmUpICU+JQogIHNlbGVjdChwcm9kdWN0X25hbWUsIG1hbnVmYWN0dXJlciwgeWVhcnNfYWN0aXZlLCBkYXRlX2RlZnVuY3QsIGNvZGViYXNlKQpgYGAKCgojIEdlbmVyYXRlIGZpZ3VyZXMKCkRlZmluZSBhIHRoZW1lIGZvciBwbG90dGluZyBmaWd1cmVzOgpgYGB7cn0KbXl0aGVtZSA8LQogIHRoZW1lX2J3KCkgKwogIHRoZW1lKAogICAgcGFuZWwuYmFja2dyb3VuZCA9IGVsZW1lbnRfcmVjdChmaWxsID0gIndoaXRlIiwgY29sb3VyID0gImJsYWNrIiwgc2l6ZSA9IDIpLAogICAgcGFuZWwuZ3JpZC5taW5vciA9IGVsZW1lbnRfYmxhbmsoKSwKICAgIHBhbmVsLmdyaWQubWFqb3IgPSBlbGVtZW50X2JsYW5rKCksCiAgICB0ZXh0ID0gZWxlbWVudF90ZXh0KHNpemUgPSAyNSwgZmFjZSA9ICJwbGFpbiIsIGNvbG91ciA9ICJibGFjayIpLAogICAgYXhpcy50aXRsZS54ID0gZWxlbWVudF90ZXh0KHNpemUgPSAyNSwgZmFjZSA9ICJwbGFpbiIpLAogICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF90ZXh0KHNpemUgPSAyNSksCiAgICBlbGVtZW50X2xpbmUoc2l6ZSA9IDIpLAogICAgYXhpcy50aWNrcy5sZW5ndGggPSB1bml0KDAuMTUsICJjbSIpKQpgYGAKCkRlZmluZSBmdW5jdGlvbnMgZm9yIGN1c3RvbWlzaW5nIHRoZSBhcHBlYXJhbmNlIG9mIHBsb3R0ZWQgZmlndXJlczoKYGBge3J9CmdldF9wb2ludF9jb2xvdXIgPC0gZnVuY3Rpb24oeCl7CiAgaWZlbHNlKHg9PVRSVUUsICJncmV5IiwgImdyZXkzMCIpCn0KCmdldF9saW5lX2NvbG91ciA8LSBmdW5jdGlvbih4KXsKICBpZmVsc2UoeCE9Im9wZW5zb3VyY2UiLCAiIzAwNzJCMiIsICIjQ0M3OUE3IikKfQpgYGAKClByb2R1Y2UgdGhlIHRpbWVsaW5lIHBsb3QgZmVhdHVyZWQgaW4gRmlndXJlIDEgb2YgdGhlIG1haW4gbWFudXNjcmlwdDoKYGBge3J9CnBfdGltZWxpbmUgPC0KICBkYXRhICU+JQogIG11dGF0ZShyb3dfbnVtYmVyID0gYXNfZmFjdG9yKHJvd19udW1iZXIpKSAlPiUKICBtdXRhdGUocm93X251bWJlciA9IGZjdF9yZW9yZGVyKGZjdF9yZW9yZGVyKHJvd19udW1iZXIsIHllYXJzX2FjdGl2ZSwgLmRlc2M9RkFMU0UpLCBjb2RlYmFzZSwgLmRlc2M9RkFMU0UpKSAlPiUKICBtdXRhdGUocm93X251bWJlcl9uZXcgPSBhcy5udW1lcmljKHJvd19udW1iZXIpKSAlPiUKICBnZ3Bsb3QoKSArCiAgZ2VvbV9zZWdtZW50KGFlcyh4PWRhdGVfcmVsZWFzZWQsIHhlbmQ9ZGF0ZV9kZWZ1bmN0X251bWVyaWMseT1yb3dfbnVtYmVyX25ldywgeWVuZD1yb3dfbnVtYmVyX25ldyksCiAgICAgICAgICAgICAgIGNvbG91cj1nZXRfbGluZV9jb2xvdXIoZGF0YSRjb2RlYmFzZSksCiAgICAgICAgICAgICAgIGxpbmV0eXBlPSJzb2xpZCIsCiAgICAgICAgICAgICAgIHNpemU9MC41KSArCiAgZ2VvbV9wb2ludChhZXMoeD1kYXRlX3JlbGVhc2VkLCB5PXJvd19udW1iZXJfbmV3KSwgY29sb3VyPWdldF9wb2ludF9jb2xvdXIoZGF0YSRkZWZ1bmN0X2luXzIwMjEpLCBzaGFwZT0xLCBzaXplPTIgKSArCiAgZ2VvbV9wb2ludChhZXMoeD1kYXRlX2RlZnVuY3RfbnVtZXJpYywgeT1yb3dfbnVtYmVyX25ldyksIGNvbG91cj1nZXRfcG9pbnRfY29sb3VyKGRhdGEkZGVmdW5jdF9pbl8yMDIxKSwgc2hhcGU9MTYsIHNpemU9MC41KSArIAogIHNjYWxlX3hfY29udGludW91cyhwb3NpdGlvbj0iYm90dG9tIiwgYnJlYWtzPWMoc2VxKDE5ODAsMjAyMSw1KSkpICsKICBjb29yZF9jYXJ0ZXNpYW4oeGxpbT1jKDE5ODAsMjAyMSkpICsKICB0aGVtZV9idygpICsKICB0aGVtZSgKICAgIHBsb3QubWFyZ2luID0gbWFyZ2luKDAuMSwgMC4xLCAwLjEsIDAuMSwgImNtIiksCiAgICBwYW5lbC5ib3JkZXIgPSBlbGVtZW50X2JsYW5rKCksCiAgICBwYW5lbC5ncmlkLm1ham9yLnkgPSBlbGVtZW50X2xpbmUoY29sb3VyPSJncmV5OTUiLCBzaXplPTAuMjUpLAogICAgcGFuZWwuZ3JpZC5tYWpvci54ID0gZWxlbWVudF9saW5lKGNvbG91cj0iZ3JleTk1Iiwgc2l6ZT0wLjI1KSwKICAgIHBhbmVsLmdyaWQubWlub3IueCA9IGVsZW1lbnRfbGluZShjb2xvdXI9ImdyZXk5NSIsIHNpemU9MC4yNSksCiAgICBheGlzLnRleHQueSA9IGVsZW1lbnRfYmxhbmsoKSwKICAgIGF4aXMudGV4dC54ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aXRsZS54ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aWNrcy55ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aWNrcy54ID0gZWxlbWVudF9ibGFuaygpLAogICAgbGVnZW5kLnBvc2l0aW9uID0gImJvdHRvbSIKICApCgpwcmludChwX3RpbWVsaW5lKQoKZ2dzYXZlKGhlcmUoIkVMTl9SZXZpZXdfSGlnZ2luc18yMDIxX1RpbWVsaW5lLnBkZiIpLCBwbG90PXBfdGltZWxpbmUsIHdpZHRoPTE4LjAsIGhlaWdodD0xMCwgZGV2aWNlPSJwZGYiLCBkcGk9NjAwLCB1bml0cz0iY20iKQpgYGAKCkdlbmVyYXRlIGEgc2VwYXJhdGUgaW50ZXJhY3RpdmUgdmVyc2lvbiBvZiB0aGUgdGltZWxpbmUgc2hvd24gaW4gRmlndXJlIDEsIGFuZCBleHBvcnQgYXMgYSBzdGFuZGFsb25lIEhUTUwgZmlsZSB1c2luZyBgcGxvdGx5YCBhbmQgYGh0bWx3aWRnZXRzYDoKYGBge3J9CnBfaW50ZXJhY3RpdmUgPC0KICBkYXRhICU+JQogIG11dGF0ZShyb3dfbnVtYmVyID0gYXNfZmFjdG9yKHJvd19udW1iZXIpKSAlPiUKICBtdXRhdGUocm93X251bWJlciA9IGZjdF9yZW9yZGVyKGZjdF9yZW9yZGVyKHJvd19udW1iZXIsIHllYXJzX2FjdGl2ZSwgLmRlc2M9RkFMU0UpLCBjb2RlYmFzZSwgLmRlc2M9RkFMU0UpKSAlPiUKICBtdXRhdGUocm93X251bWJlcl9uZXcgPSBhcy5udW1lcmljKHJvd19udW1iZXIpKSAlPiUKICBtdXRhdGUoY29kZWJhc2VfbmFtZSA9IHJlY29kZShjb2RlYmFzZSwgcHJvcHJpZXRhcnkgPSAiUHJvcHJpZXRhcnkiLCBvcGVuc291cmNlID0gIk9wZW4tc291cmNlIiksCiAgICAgICAgIG1hcmtlcl9jb2xvdXIgPSBpZl9lbHNlKGRlZnVuY3RfaW5fMjAyMSA9PSBUUlVFLCAiI0MwQzBDMCIsICIjNGQ0ZDRkIiksCiAgICAgICAgIGhvdmVydGV4dCA9CiAgICAgICAgICAgcGFzdGUwKCI8Yj4iLAogICAgICAgICAgICAgICAgICBwcm9kdWN0X25hbWUsCiAgICAgICAgICAgICAgICAgICI8L2I+PGJyPkRhdGUgcmVsZWFzZWQ6ICIsCiAgICAgICAgICAgICAgICAgIGRhdGVfcmVsZWFzZWQsCiAgICAgICAgICAgICAgICAgICI8YnI+RGF0ZSBkZWZ1bmN0OiAiLAogICAgICAgICAgICAgICAgICBkYXRlX2RlZnVuY3QsCiAgICAgICAgICAgICAgICAgICI8ZXh0cmE+PC9leHRyYT4iKQogICkgJT4lCiAgcGxvdF9seSgpICU+JQogIGFkZF9zZWdtZW50cygKICAgIHggPSB+ZGF0ZV9yZWxlYXNlZCwgeSA9IH5yb3dfbnVtYmVyX25ldywKICAgIHhlbmQgPSB+ZGF0ZV9kZWZ1bmN0X251bWVyaWMsIHllbmQgfnJvd19udW1iZXJfbmV3LAogICAgY29sb3IgPSB+ZmFjdG9yKGNvZGViYXNlX25hbWUpLAogICAgY29sb3JzID0gYygiI0NDNzlBNyIsICIjMDA3MkIyIikKICApICU+JQogIGFkZF9tYXJrZXJzKAogICAgeCA9IH5kYXRlX3JlbGVhc2VkLAogICAgeSA9IH5yb3dfbnVtYmVyX25ldywKICAgIHRleHQgPSB+cHJvZHVjdF9uYW1lLAogICAgbmFtZSA9ICJEYXRlIHJlbGVhc2VkIiwKICAgIGN1c3RvbWRhdGEgPSB+aG92ZXJ0ZXh0LAogICAgaG92ZXJ0ZW1wbGF0ZSA9ICIle2N1c3RvbWRhdGF9IiwKICAgIG1hcmtlciA9CiAgICAgIGxpc3QoCiAgICAgICAgc3ltYm9sID0gImNpcmNsZS1vcGVuIiwKICAgICAgICBzaXplID0gMTAsCiAgICAgICAgY29sb3IgPSB+bWFya2VyX2NvbG91cgogICAgICApLAogICAgaW5oZXJpdCA9IEZBTFNFCiAgKSAlPiUKICBhZGRfbWFya2VycygKICAgIHggPSB+ZGF0ZV9kZWZ1bmN0X251bWVyaWMsCiAgICB5ID0gfnJvd19udW1iZXJfbmV3LAogICAgdGV4dCA9IH5wcm9kdWN0X25hbWUsCiAgICBuYW1lID0gIkRhdGUgZGVmdW5jdCIsCiAgICBjdXN0b21kYXRhID0gfmhvdmVydGV4dCwKICAgIGhvdmVydGVtcGxhdGUgPSAiJXtjdXN0b21kYXRhfSIsCiAgICBtYXJrZXIgPQogICAgICBsaXN0KAogICAgICAgIHNpemUgPSAzLAogICAgICAgIGNvbG9yID0gfm1hcmtlcl9jb2xvdXIKICAgICAgKQogICkgJT4lCiAgbGF5b3V0KAogICAgdGl0bGUgPSAiSG93IGxvbmcgZG8gZWxlY3Ryb25pYyBsYWJvcmF0b3J5IG5vdGVib29rcyBsYXN0PyIsCiAgICB4YXhpcyA9IGxpc3QodGl0bGUgPSAiVGltZWxpbmUgb2YgRUxOIHByb2R1Y3RzIiksCiAgICB5YXhpcyA9IGxpc3QodGl0bGUgPSAiTGlmZXRpbWVzIG9mIHN1cnZleWVkIEVMTiBwcm9kdWN0cyIsCiAgICAgICAgICAgICAgICAgc2hvd3RpY2tsYWJlbHMgPSBGQUxTRSwKICAgICAgICAgICAgICAgICB6ZXJvbGluZSA9IEZBTFNFLAogICAgICAgICAgICAgICAgIHNob3dsaW5lID0gRkFMU0UpCiAgKQpzYXZlV2lkZ2V0KHBhcnRpYWxfYnVuZGxlKHBfaW50ZXJhY3RpdmUpLCBoZXJlKCJFTE5fUmV2aWV3X0hpZ2dpbnNfMjAyMV9MaWZldGltZXNfSW50ZXJhY3RpdmVfRmlndXJlMS5odG1sIiksIHNlbGZjb250YWluZWQgPSBUUlVFLCB0aXRsZSA9ICJFTE4gc3VydmV5IikKYGBgCgpHZW5lcmF0ZSBhIHBsb3Qgc2hvd2luZyB0aGUgdG90YWwgbnVtYmVyIG9mIG5ldyBwcm9wcmlldGFyeSBhbmQgb3Blbi1zb3VyY2UgRUxOcyBwZXIgeWVhciwgYXMgZmVhdHVyZWQgaW4gRmlndXJlIDE6IChOb3RlOiBheGVzIGZvciB0aGlzIHBsb3Qgd2VyZSBtYW51YWxseSBhcHBlbmRlZCBsYXRlIGluIGdyYXBoaWNzIHNvZnR3YXJlKQpgYGB7cn0KZGF0YV9zdW1tYXJpc2VkIDwtCiAgZGF0YSAlPiUKICBncm91cF9ieShkYXRlX3JlbGVhc2VkLCBjb2RlYmFzZSkgJT4lCiAgc3VtbWFyaXNlKGNvdW50ID0gbigpLCAuZ3JvdXBzPSJkcm9wX2xhc3QiKQoKcF9yZWxlYXNlcyA8LQogIGRhdGFfc3VtbWFyaXNlZCAlPiUKICBnZ3Bsb3QoYWVzKHg9ZGF0ZV9yZWxlYXNlZCwgeT1hcy5mYWN0b3IoY29kZWJhc2UpLCBzaXplPWNvdW50KSkgKwogIGdlb21fcG9pbnQoc2hhcGU9MjEsIGZpbGw9Z2V0X2xpbmVfY29sb3VyKGRhdGFfc3VtbWFyaXNlZCRjb2RlYmFzZSksYWxwaGE9MC41KSArCiAgc2NhbGVfc2l6ZShyYW5nZT1jKDEsMTApKSArCiAgc2NhbGVfeF9jb250aW51b3VzKHBvc2l0aW9uPSJib3R0b20iLCBicmVha3M9YyhzZXEoMTk4MCwyMDIxLDUpKSkgKwogIGNvb3JkX2NhcnRlc2lhbih4bGltPWMoMTk4MCwyMDIxKSkgKwogIHRoZW1lX2J3KCkgKwogIHRoZW1lKAogICAgcGxvdC5tYXJnaW4gPSBtYXJnaW4oMC4xLCAwLjEsIDAuMSwgMC4xLCAiY20iKSwKICAgIHBhbmVsLmJvcmRlciA9IGVsZW1lbnRfYmxhbmsoKSwKICAgIHBhbmVsLmdyaWQubWFqb3IueSA9IGVsZW1lbnRfYmxhbmsoKSwKICAgIHBhbmVsLmdyaWQubWFqb3IueCA9IGVsZW1lbnRfbGluZShjb2xvdXI9ImdyZXk5NSIsIHNpemU9MC4yNSksCiAgICBwYW5lbC5ncmlkLm1pbm9yLnggPSBlbGVtZW50X2xpbmUoY29sb3VyPSJncmV5OTUiLCBzaXplPTAuMjUpLAogICAgYXhpcy50aXRsZS55ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aXRsZS54ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aWNrcy55ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50aWNrcy54ID0gZWxlbWVudF9ibGFuaygpLAogICAgYXhpcy50ZXh0LnkgPSBlbGVtZW50X2JsYW5rKCksCiAgICBsZWdlbmQucG9zaXRpb24gPSAibm9uZSIKICApCgpwcmludChwX3JlbGVhc2VzKQoKZ2dzYXZlKGhlcmUoIkVMTl9SZXZpZXdfSGlnZ2luc18yMDIxX1JlbGVhc2VzLVBlci1ZZWFyLnBkZiIpLCBwbG90PXBfcmVsZWFzZXMsIHdpZHRoPTE4LjAsIGhlaWdodD0yLjUsIGRldmljZT0icGRmIiwgZHBpPTYwMCwgdW5pdHM9ImNtIikKYGBgCgojIFNlc3Npb24gaW5mb3JtYXRpb24KCmBgYHtyfQpzZXNzaW9uSW5mbygpCmBgYAoK