---
title: Processing the Rizzardi/Hickey brain dataset
author: Peter Hickey
date: 28 September 2020
output:
  BiocStyle::html_document:
    titlecaps: false
    toc_float: true
bibliography: ref.bib
editor_options: 
  chunk_output_type: console
---

```{r style, echo = FALSE, results = "hide", message = FALSE}
library(BiocStyle)
knitr::opts_chunk$set(error = FALSE, message = FALSE, warning = FALSE)
```

# Download the data

We obtain a whole-genome bisulfite-sequencing dataset of human brain from @rizzardi2019neuronal.
Count matrices of methylated reads and total reads for various samples, cytosine contexts (CpG, CpA, CpT), and strands (unstranded, forward, reverse) are provided from the Gene Expression Omnibus using accession code [GSE96612](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE96612).
We download these using `r Biocpkg("BiocFileCache")` to cache the results:

```{r}
library(BiocFileCache)
bfc <- BiocFileCache("raw_data", ask = FALSE)

base_url <- "https://ftp.ncbi.nlm.nih.gov/geo/series/GSE96nnn/GSE96612/suppl"

cov_bulk_cpg_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.bulk.CpG_unstranded.txt.gz"))
m_bulk_cpg_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.bulk.CpG_unstranded.txt.gz"))

cov_sorted_cpg_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.sorted.CpG_unstranded.txt.gz"))
m_sorted_cpg_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.sorted.CpG_unstranded.txt.gz"))

cov_sorted_cpa_fwd_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.sorted.CpA_forward_strand.txt.gz"))
m_sorted_cpa_fwd_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.sorted.CpA_forward_strand.txt.gz"))

cov_sorted_cpa_rev_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.sorted.CpA_reverse_strand.txt.gz"))
m_sorted_cpa_rev_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.sorted.CpA_reverse_strand.txt.gz"))

cov_sorted_cpt_fwd_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.sorted.CpT_forward_strand.txt.gz"))
m_sorted_cpt_fwd_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.sorted.CpT_forward_strand.txt.gz"))

cov_sorted_cpt_rev_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_Cov_matrix.sorted.CpT_reverse_strand.txt.gz"))
m_sorted_cpt_rev_fname <- bfcrpath(
  bfc,
  file.path(base_url, "GSE96612_M_matrix.sorted.CpT_reverse_strand.txt.gz"))
```

# Process the data and save for upload

We create a function to extract the loci, sample-level metadata, and count matrices from each file and write all of the relevant components to to a [HDF5 file](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) for upload to `r Biocpkg("ExperimentHub")`.

```{r}
library(data.table)
library(GenomicRanges)
library(HDF5Array)

FUN <- function(cov_fnames, m_fnames, out_fname) {
  stopifnot(identical(length(cov_fnames), length(m_fnames)))
  n <- length(cov_fnames)
  
  # Load data
  sample_data <- lapply(seq_along(cov_fnames), function(j) {
    cov_dt <- fread(cov_fnames[[j]], data.table = FALSE)
    m_dt <- fread(m_fnames[[j]], data.table = FALSE)
    stopifnot(
      identical(colnames(cov_dt), colnames(m_dt)),
      identical(cov_dt$chr, m_dt$chr),
      identical(cov_dt$start, m_dt$start),
      identical(cov_dt$strand, m_dt$strand))
    
    # rowRanges
    if (grepl("CpG", out_fname)) {
      # CpG data are aggregated across strands and thus unstranded.
      rowranges <- GRanges(
        seqnames = cov_dt$chr, 
        ranges = IRanges(start = cov_dt$start, width = 2L),
        strand = cov_dt$strand, 
        seqinfo = Seqinfo(genome = "hg19"))
    } else {
      rowranges <- GPos(
        seqnames = cov_dt$chr, 
        pos = cov_dt$start, 
        strand = cov_dt$strand, 
        seqinfo = Seqinfo(genome = "hg19"))
    }
    
    # colData
    cns_ <- setdiff(colnames(cov_dt), c("chr", "start", "strand"))
    cns <- sub("NA", "NAcc", cns_)
    cn_split <- strsplit(cns, "_")
    donor <- vapply(cn_split, "[[", 1, FUN.VALUE = character(1L))
    tissue <- vapply(cn_split, "[[", 2, FUN.VALUE = character(1L))
    neun <- if (all(lengths(cn_split) == 3)) {
      vapply(cn_split, "[[", 3, FUN.VALUE = character(1L))
    } else {
      "unsorted"
    }
    coldata <- data.frame(donor = donor, tissue = tissue, neun = neun)
    rownames(coldata) <- paste(
      coldata$donor,
      coldata$tissue,
      coldata$neun,
      sep = "_")
    
    # Assays
    M <- DelayedArray(m_dt[, cns_])
    colnames(M) <- rownames(coldata)
    type(M) <- "integer"
    rownames(M) <- NULL
    Cov <- DelayedArray(cov_dt[, cns_])
    colnames(Cov) <- rownames(coldata)
    type(Cov) <- "integer"
    rownames(Cov) <- NULL
    
    list(M = M, Cov = Cov, rowRanges = rowranges, colData = coldata)
  })
  
  # rowRanges
  # NOTE: rowRanges are only checked for sortedness if n > 1 because if n > 1 then 
  #       the data are sorted but they are not if n == 1.
  if (n > 1) {
    rowranges <- sort(unique(Reduce(c, lapply(sample_data, "[[", "rowRanges"))))
  } else {
    rowranges <- sample_data[[1]][["rowRanges"]]
    if (is.unsorted(rowranges)) {
      stop("rowranges are unsorted.")
    }
  }
  h5write(
    # TODO: https://github.com/Bioconductor/GenomicRanges/issues/46
    obj = as.data.frame(rowranges), 
    file = out_fname, 
    name = "rowRanges",
    level = level,
    # NOTE: Needed to ensure factor columns aren't converted to their underlying 
    #       integer types.
    DataFrameAsCompound = FALSE)
  # NOTE: as.data.frame,Seqinfo-method stores seqnames as rownames, which aren't 
  #       preserved by the call to h5write(), so have to explicitly add these 
  #       back as a column.
  seqinfo_df <- as.data.frame(seqinfo(rowranges))
  seqinfo_df[["seqnames"]] <- rownames(seqinfo_df)
  h5write(
    obj = seqinfo_df, 
    file = out_fname, 
    name = "rowRanges/seqinfo",
    level = level,
    # NOTE: Needed to ensure factor columns aren't converted to their underlying 
    #       integer types.
    DataFrameAsCompound = FALSE)
  
  # colData
  coldata <- do.call(rbind, lapply(sample_data, "[[", "colData"))
  h5write(
    obj = coldata, 
    file = out_fname, 
    name = "colData",
    level = level,
    # NOTE: Needed to ensure factor columns aren't converted to their underlying 
    #       integer types.
    DataFrameAsCompound = FALSE)

  # M and Cov
  if (n > 1) {
    # NOTE: This creates a large matrix in-memory, so only do with care.
    M <- matrix(
      0L, 
      nrow = length(rowranges), 
      ncol = nrow(coldata),
      dimnames = list(NULL, rownames(coldata)))
    Cov <- M
    for (j in seq_along(sample_data)) {
      sd <- sample_data[[j]]
      ol <- findOverlaps(sd$rowRanges, rowranges, type = "equal")
      M[subjectHits(ol), colnames(sd$M)] <- as.matrix(sd$M[queryHits(ol), ])
      Cov[subjectHits(ol), colnames(sd$Cov)] <- as.matrix(
        sd$Cov[queryHits(ol), ])
    }
  } else {
    M <- sample_data[[1L]][["M"]]
    Cov <- sample_data[[1L]][["Cov"]]
  }
  # TODO: Think about best chunkdim. Default is getHDF5DumpChunkDim(dim(x)),
  #       which currently results in blocks along columns.
  chunkdim <- makeCappedVolumeBox(
    getHDF5DumpChunkLength(),
    dim(M),
    "hypercube")
  M <- writeHDF5Array(
    x = M,
    filepath = out_fname,
    name = "M",
    chunkdim = chunkdim,
    level = level,
    with.dimnames = TRUE)
  Cov <- writeHDF5Array(
    x = Cov,
    filepath = out_fname,
    name = "Cov",
    chunkdim = chunkdim,
    level = level,
    with.dimnames = TRUE)

  list(M = M, Cov = Cov, rowRanges = rowranges, colData = coldata)
}
```

We run this on all the datasets, also combining the 'bulk' and 'sorted' CpG data into a single dataset.

```{r}
options(DelayedArray.block.size = 1e10) # 10GB block size.
path <- file.path("MethylSeqData", "rizzardi_hickey_brain")
dir.create(path, showWarnings = FALSE, recursive = TRUE)
level <- 9

cpg <- FUN(
  c(cov_bulk_cpg_fname, cov_sorted_cpg_fname),
  c(m_bulk_cpg_fname, m_sorted_cpg_fname),
  file.path(path, "rizzardi_hickey_brain.CpG.h5"))
sorted_cpa_fwd <- FUN(
  cov_sorted_cpa_fwd_fname, 
  m_sorted_cpa_fwd_fname, 
  file.path(path, "rizzardi_hickey_brain.CpA_fwd.h5"))
sorted_cpa_rev <- FUN(
  cov_sorted_cpa_rev_fname, 
  m_sorted_cpa_rev_fname, 
  file.path(path, "rizzardi_hickey_brain.CpA_rev.h5"))
sorted_cpt_fwd <- FUN(
  cov_sorted_cpt_fwd_fname, 
  m_sorted_cpt_fwd_fname, 
  file.path(path, "rizzardi_hickey_brain.CpT_fwd.h5"))
sorted_cpt_rev <- FUN(
  cov_sorted_cpt_rev_fname, 
  m_sorted_cpt_rev_fname, 
  file.path(path, "rizzardi_hickey_brain.CpT_rev.h5"))
```

# Session info

```{r}
sessionInfo()
```

# References
