---
title: Processing the Chen mammary dataset
author: Peter Hickey
date: 28 September 2020
output:
  BiocStyle::html_document:
    titlecaps: false
    toc_float: true
bibliography: ref.bib
editor_options: 
  chunk_output_type: console
---

```{r style, echo = FALSE, results = "hide", message = FALSE}
library(BiocStyle)
knitr::opts_chunk$set(error = FALSE, message = FALSE, warning = FALSE)
```

# Download the data

We obtain a reduced representation bisulfite-sequencing dataset of mouse mammary gland used in @chen2017mammary.
Bismark output files for each sample and a sample metadata file are provided from Zenodo using DOI [10.5281/zenodo.1052871](https://doi.org/10.5281/zenodo.1052871).
We download these using `r Biocpkg("BiocFileCache")` to cache the results:

```{r}
library(BiocFileCache)
bfc <- BiocFileCache("raw_data", ask = FALSE)

base_url <- "https://zenodo.org/record/1052871/files"

p6_1_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P6_1.bismark.cov.gz"))

p6_4_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P6_4.bismark.cov.gz"))

p7_2_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P7_2.bismark.cov.gz"))

p7_5_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P7_5.bismark.cov.gz"))

p8_3_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P8_3.bismark.cov.gz"))

p8_6_fname <- bfcrpath(
  bfc,
  file.path(base_url, "P8_6.bismark.cov.gz"))

targets_fname <- bfcrpath(
  bfc,
  file.path(base_url, "targets.txt"))
```

# Process the data

We create a function to extract the loci and count matrices from each file.

```{r}
library(GenomicRanges)
path <- file.path("MethylSeqData", "chen_mammary")
dir.create(path, showWarnings = FALSE, recursive = TRUE)
FUN <- function(fname, prefix) {
  bismark_df <- read.delim(fname, sep = "\t", header = FALSE)
  
  rowranges <- GPos(
    seqnames = bismark_df$V1, 
    pos = bismark_df$V2, 
    seqinfo = Seqinfo(genome = "mm10"))
  
  M <- matrix(as.integer(bismark_df$V5), dimnames = list(NULL, prefix))
  U <- matrix(as.integer(bismark_df$V6), dimnames = list(NULL, prefix))

  list(M = M, U = U, rowRanges = rowranges)
}
```

We run this on all the datasets.

```{r}
p6_1 <- FUN(p6_1_fname, "P6_1")
p6_4 <- FUN(p6_4_fname, "P6_4")
p7_2 <- FUN(p7_2_fname, "P7_2")
p7_5 <- FUN(p7_5_fname, "P7_5")
p8_3 <- FUN(p8_3_fname, "P8_3")
p8_6 <- FUN(p8_6_fname, "P8_6")
```

We then combine the sample-level loci and count matrices.

```{r}
sample_data <- list(
  "P6_1" = p6_1, "P6_4" = p6_4, 
  "P7_2" = p7_2, "P7_5" = p7_5, 
  "P8_3" = p8_3, "P8_6" = p8_6)

rowranges <- sort(unique(Reduce(c, lapply(sample_data, "[[", "rowRanges"))))

M <- matrix(
  0L, 
  nrow = length(rowranges), 
  ncol = length(sample_data),
  dimnames = list(NULL, names(sample_data)))
U <- M
for (j in seq_along(sample_data)) {
  sd <- sample_data[[j]]
  ol <- findOverlaps(sd$rowRanges, rowranges, type = "equal")
  M[subjectHits(ol), colnames(sd$M)] <- sd$M[queryHits(ol), ]
  U[subjectHits(ol), colnames(sd$U)] <- sd$U[queryHits(ol), ]
}
```

Finally, we load in the sample metadata.

```{r}
coldata <- read.delim(targets_fname, sep = "\t")
```

# Save for upload

We write all of the relevant components to to a [HDF5 file](https://en.wikipedia.org/wiki/Hierarchical_Data_Format) for upload to `r Biocpkg("ExperimentHub")`.

```{r}
library(HDF5Array)
options(DelayedArray.block.size = 1e10) # 10GB block size.
path <- file.path("MethylSeqData", "chen_mammary")
dir.create(path, showWarnings = FALSE, recursive = TRUE)
fname <- file.path(path, "chen_mammary.h5")
level <- 9

# M and U.
# TODO: Think about best chunkdim. Default is getHDF5DumpChunkDim(dim(x)),
#       which currently results in blocks along columns.
chunkdim <- makeCappedVolumeBox(
    getHDF5DumpChunkLength(),
    dim(M),
    "hypercube")
M <- writeHDF5Array(
  x = M, 
  filepath = fname,
  name = "M",
  chunkdim = chunkdim,
  level = level,
  with.dimnames = TRUE)
U <- writeHDF5Array(
  x = U, 
  filepath = fname,
  name = "U",
  chunkdim = chunkdim,
  level = level,
  with.dimnames = TRUE)

# rowRanges
h5write(
  # TODO: https://github.com/Bioconductor/GenomicRanges/issues/46
  obj = as.data.frame(rowranges), 
  file = fname, 
  name = "rowRanges",
  level = level,
  # NOTE: Needed to ensure factor columns aren't converted to their underlying 
  #       integer types.
  DataFrameAsCompound = FALSE)
# NOTE: as.data.frame,Seqinfo-method stores seqnames as rownames, which aren't 
#       preserved by the call to h5write(), so have to explicitly add these back 
#       as a column.
seqinfo_df <- as.data.frame(seqinfo(rowranges)) 
seqinfo_df[["seqnames"]] <- rownames(seqinfo_df)
h5write(
  obj = seqinfo_df, 
  file = fname, 
  name = "rowRanges/seqinfo",
  level = level,
  # NOTE: Needed to ensure factor columns aren't converted to their underlying 
  #       integer types.
  DataFrameAsCompound = FALSE)

# colData
h5write(
  obj = coldata, 
  file = fname, 
  name = "colData",
  level = level,
  # NOTE: Needed to ensure factor columns aren't converted to their underlying 
  #       integer types.
  DataFrameAsCompound = FALSE)
```

# Session information

```{r}
sessionInfo()
```

# References
