---
title: "Example R Markdown Notebook"
output:
  html_document:
    df_print: paged
jupyter:
  kernelspec:
    display_name: R
    language: R
    name: ir
  language_info:
    codemirror_mode: r
    file_extension: .r
    mimetype: text/x-r-source
    name: R
    pygments_lexer: r
    version: 3.4.3
---

## NLE Digital Newspaper Collection (DEA)

This notebook presents a pilot workflow on accessing digitized newspaper data in the National Library of Estonia. The collection currently includes 386,950 issues from 2200 periodical publications (including journals, magazines and newspapers). This amounts to a total of 3,635,503 pages of text and 6,298,369 articles segmented from this. Some of these texts are free to access, others need to follow some access restrictions due to copyright and other legal requirements. This notebook connects to a subsample of freely accessible texts, amounting to ~1,000,000 articles. 

NB! These texts have the licence: Free access - restricted use. This publication's copyright protection has expired but the rights of works contained in the publication may be protected. The works may be used for private purposes or study and research purposes. In other cases please ascertain that the copyright term has expired.

### Usage

The Notebook runs R-3.6.0 on the Jupyter Notebook, so regular R commands will work. To run a block in Jupyter Notebook, select the block and click on he the > 'run' icon above. To run a block in RStudio find the green > on top of the block.


### Required libraries

```{r, results=F, warning=F,messages=F}
library(data.table)
library(tidyverse)
library(lubridate)
library(plotly)
library(utils)
library(DT)

```


### Reading the files

This repository contains an index for the DEA periodical collection. `all_issues_all.tsv.zip` contains metainformation on each of the 383,388 issues of 2200 periodicals currently digitally available in the National Library of Estonia. 



```{r}

# Step 1. Download the latest metadata file for the collection (if you have not done so already)
system(paste0("wget --quiet data.digar.ee/text/all_issues_access.zip -O all_issues_access.zip"))

```


```{r}

# Read in the metadata file for all issues in the collection
all_issues <- fread("unzip -p all_issues_access.zip",sep="\t")

# Add some new variables
all_issues[,year:=as.numeric(year)]
all_issues[,firstyear:=min(year,na.rm=T),.(keyid)]
all_issues[,lastyear:=max(year,na.rm=T),by=.(keyid)]
all_issues[,title:=str_remove(DocumentTitle,";.*")]
all_issues[,number_of_issues:=.N,.(keyid)]

```


### Overview of the publications

The following block prints an overview of the metadata in an interactive table. The interactive table can be accessed below, or from the link [here](summarytable.html).

```{r}


summary_allissues <- unique(all_issues[,.SD[1],.(keyid)][,.(title,keyid,firstyear,lastyear,number_of_issues)])

#In Rstudio, run this line instead
#datatable(summary_allissues, filter = list(position = 'top', clear = FALSE),options = list(autoWidth = FALSE,   columnDefs = list(list(width = '200px', targets = c(1)),list(width = '200px', targets = c(2)),list(width = '50px', targets = c(2)),list(width = '50px', targets = c(2))),scrollX=T,     pageLength = 25))

DT::saveWidget(datatable(summary_allissues, filter = list(position = 'top', clear = FALSE),options = list(autoWidth = FALSE,
  columnDefs = list(list(width = '200px', targets = c(1)),list(width = '200px', targets = c(2)),list(width = '50px', targets = c(2)),list(width = '50px', targets = c(2))),scrollX=T,
    pageLength = 25)),"tables/summarytable.html")
IRdisplay::display_html('<iframe src="tables/summarytable.html" width=100%, height=1000></iframe> ')

```


### Graphic overviews of the collection

The number of issues over time in the collection.

```{r}
all_issues[,.N,year] %>% 
   ggplot(aes(x=year,y=N))+
   geom_col()
```

The issues colored by publication type.

```{r}
all_issues[,.N,.(year,DocumentType)] %>% 
   ggplot(aes(x=year,y=N, fill=DocumentType))+
   geom_col()
```

The issues with largest number of articles.

```{r}

toplist <- all_issues[,.N,keyid][order(-N)][1:20]
toplist[,keyid:=factor(keyid, levels=unique(keyid))]

toplist %>% 
   ggplot(aes(x=keyid,y=N, fill=keyid))+
   geom_col()+
   coord_flip()
```

The publication spans of the 20 publications with the largest collections.

```{r}

durations <- unique(all_issues[,.(keyid,firstyear,lastyear)])[toplist,on="keyid"][order(firstyear)]
durations[,keyid:=factor(keyid, levels=unique(keyid))]

durations %>% 
   ggplot(aes(y=keyid,color=keyid))+
   geom_segment(aes(yend=keyid,x=firstyear,xend=lastyear),size=3)
   coord_flip()

```


### Access and licences of the files

```{r}

#Metainformation on the publications, with licences and access rights of the files, and other data.
all_issues_plus_access <- all_issues[order(access)]
all_issues_plus_access[,keyid:=factor(keyid,levels=unique(keyid))]
p<- all_issues_plus_access[,.N,.(year,keyid,access)] %>% 
  ggplot(aes(x=year,y=N,text=keyid, fill=access))+
  geom_col()
ggplotly(p)

htmlwidgets::saveWidget(as_widget(ggplotly(p)), "tables/issue_access_time.html")


IRdisplay::display_html('<iframe src="tables/issue_access_time.html" width=100%, height=1000></iframe> ')
```

```{r}

# Languages in the collection
all_issues_plus_access <- all_issues_plus_access[order(language)]
all_issues_plus_access[,keyid:=factor(keyid,levels=unique(keyid))]
p<- all_issues_plus_access[,.N,.(year,keyid,language)] %>% 
  ggplot(aes(x=year,y=N,text=keyid, fill=language))+
  geom_col()
ggplotly(p)


```

```{r}

# Countries of publication in the collection
all_issues_plus_access <- all_issues_plus_access[order(country)]
all_issues_plus_access[,keyid:=factor(keyid,levels=unique(keyid))]
p<- all_issues_plus_access[,.N,.(year,keyid,country)] %>% 
  ggplot(aes(x=year,y=N,text=keyid, fill=country))+
  geom_col()
ggplotly(p)


```


```{r}

# Genre of the publication
all_issues_plus_access <- all_issues_plus_access[order(section)]
all_issues_plus_access[,keyid:=factor(keyid,levels=unique(keyid))]
p<- all_issues_plus_access[,.N,.(year,keyid,section)] %>% 
  ggplot(aes(x=year,y=N,text=keyid, fill=section))+
  geom_col()
ggplotly(p)

```


```{r}

# Overview of one genre in the dataset
p<- all_issues_plus_access[section=="Art, music, theatre, film"][,.N,.(year,keyid,section)] %>% 
  ggplot(aes(x=year,y=N,text=keyid, fill=keyid))+
  geom_col()
ggplotly(p)


```


```{r}


summary_all_plus_access <- unique(all_issues_plus_access[,.SD[1],.(keyid)][,.(title,keyid,firstyear,lastyear,number_of_issues, access, section)])


#In Rstudio, run this line instead
#datatable(summary_allissues, filter = list(position = 'top', clear = FALSE),options = list(autoWidth = FALSE,   columnDefs = list(list(width = '200px', targets = c(1)),list(width = '200px', targets = c(2)),list(width = '50px', targets = c(2)),list(width = '50px', targets = c(2))),scrollX=T,     pageLength = 25))

DT::saveWidget(datatable(summary_all_plus_access, filter = list(position = 'top', clear = FALSE),options = list(autoWidth = FALSE,
  columnDefs = list(list(width = '200px', targets = c(1)),list(width = '200px', targets = c(2)),list(width = '50px', targets = c(2)),list(width = '50px', targets = c(2))),scrollX=T,
    pageLength = 25)),"tables/summary_plus_access.html")
IRdisplay::display_html('<iframe src="tables/summary_plus_access.html" width=100%, height=1000></iframe> ')

```

