---
title: "10x Genomics datasets"
output: html_document
date: "2023-05-22"
---

```{r setup, include=FALSE}
knitr::opts_chunk$set(echo = TRUE)
```

```{r}
library(R.utils)
library(SingleCellExperiment)
library(DropletUtils)
library(rtracklayer)
```

## Raer package data

### Mouse 10x single cell RNA-seq

Make a tiny bam file from a 10x experiment, in this case a mouse brain single cell RNA-seq experiment.

First, make a tiny fasta file from a few regions with editing sites.

```{r}
dir.create("pkg_data", showWarnings = FALSE)
regions <- c("chr2:116032569-116033683",
             "chr6:48079510-48079909",
             "chr11:75300212-75300704",
             "chr8:65639891-65640286")
regions <- GRanges(regions)
strand(regions) <- "+"

seqs <- getSeq(FaFile("dbases/GRCm38.primary_assembly.genome.fa"), regions)
names(seqs) <- sub("chr", "", names(seqs))
export(seqs, "pkg_data/mouse_tiny.fasta")


gtf_lines <- c(
    '2\tunknown\texon\t1\t1115\t.\t-\t.\tgene_id Meis2; transcript_id Meis2; gene_name Meis2; gene_biotype "protein_coding";',
    '6\tunknown\texon\t1\t400\t.\t-\t.\tgene_id Zfp746; transcript_id Zfp746; gene_name Zfp746; gene_biotype "protein_coding";',
    '10\tunknown\texon\t1\t616\t.\t+\t.\tgene_id Ppfia2; transcript_id Ppfia2; gene_name Ppfia2; gene_biotype "protein_coding";',
    '11\tunknown\texon\t1\t493\t.\t-\t.\tgene_id Rpa1; transcript_id Rpa1; gene_name Rpa1; gene_biotype "protein_coding";')
writeLines(gtf_lines, "pkg_data/mouse_tiny.gtf")
```

Next, build a cellranger reference for the tiny fasta

```{bash}
mkdir -p mouse_5k_neuron
cd mouse_5k_neuron

cellranger mkref --genome=mm10_tiny --fasta=pkg_data/mouse_tiny.fasta --genes=pkg_data/mouse_tiny.gtf
```

Download bam file, downsample to 1%, convert to fastqs, and requantify using cellranger count. 

```{bash}
cd mouse_5k_neuron
wget https://cg.10xgenomics.com/samples/cell-exp/3.0.2/5k_neuron_v3_nextgem/5k_neuron_v3_nextgem_possorted_genome_bam.bam 
wget https://cg.10xgenomics.com/samples/cell-exp/3.0.2/5k_neuron_v3_nextgem/5k_neuron_v3_nextgem_possorted_genome_bam.bam.bai 

# downsample
samtools view -@ 8 -b -s 42.01 5k_neuron_v3_nextgem_possorted_genome_bam.bam > 5k_neuron_v3_nextgem_possorted_genome_bam_0.1.bam

# bamtofastq from 10x genomics
~/bin/bamtofastq-1.3.2 5k_neuron_v3_nextgem_possorted_genome_bam_0.1.bam ./fastq/

# cellranger from 10x genomics
cellranger count \
 --id 5k_neuron_tiny \
 --transcriptome=mouse_5k_neuron/mm10_tiny \
 --fastqs fastq/5k_neuron_v3_nextgem_0_1_HKF7VDSXX \
 --sample bamtofastq \
 --jobmode local \
 --localcores 6 \
 --localmem 4 \
 --chemistry SC3Pv3
```

Next extract a small subset of reads. 

```{bash}
cd mouse_5k_neuron/5k_neuron_tiny/outs 

# find UMIs duplicated in file
samtools view -f 1024 possorted_genome_bam.bam 2 \
    | egrep -o "UB:Z:[A-Z]+" \
    | sed 's/UB:Z://'g > umi_tags_to_keep.txt

# get reads from UMIs
samtools view -b -D UB:umi_tags_to_keep.txt possorted_genome_bam.bam > tmp.bam

# get all other reads from cbs with these reads
samtools view tmp.bam | cut -f 1 | sort | uniq > tmp_reads.txt

# and a few random reads
samtools view -s 42.01 possorted_genome_bam.bam 2 | head -n 500 >> tmp_reads.txt
cat tmp_reads.txt | sort | uniq > tmp_uniq_reads.txt

samtools view -b -N tmp_uniq_reads.txt possorted_genome_bam.bam > tmp2.bam
samtools merge -f -o 5k_neuron_mouse_possort.bam tmp.bam tmp2.bam
samtools index 5k_neuron_mouse_possort.bam

mv 5k_neuron_mouse_possort.bam* ../../../pkg_data/
```

Also get the matrix and make a sce 

```{bash}
cd mouse_5k_neuron

wget https://cf.10xgenomics.com/samples/cell-exp/3.0.2/5k_neuron_v3_nextgem/5k_neuron_v3_nextgem_filtered_feature_bc_matrix.tar.gz
tar zxvf 5k_neuron_v3_nextgem_filtered_feature_bc_matrix.tar.gz
```

```{r}
library(DropletUtils)
library(scran)
library(scater)

data_dir <- "mouse_5k_neuron"

sce <- read10xCounts(file.path(data_dir, "filtered_feature_bc_matrix"),
                     col.names = TRUE)

rownames(sce) <- uniquifyFeatureNames(
  rowData(sce)$ID, rowData(sce)$Symbol)

seed_val <- 42
sce <- logNormCounts(sce)
set.seed(seed_val)
dec <- modelGeneVarByPoisson(sce)
top_hvgs <- getTopHVGs(dec, prop = 0.1)
set.seed(seed_val)
sce <- denoisePCA(sce, technical=dec, subset.row=top_hvgs)
set.seed(seed_val)
sce <- runUMAP(sce, dimred="PCA")
snn.gr <- buildSNNGraph(sce, use.dimred="PCA", k=30)
colLabels(sce) <- factor(igraph::cluster_walktrap(snn.gr)$membership)
sce$clusters <- colLabels(sce)

if(!file.exists(file.path(data_dir, "allen_brain_reference.csv"))){
  download.file("https://idk-etl-prod-download-bucket.s3.amazonaws.com/aibs_mouse_ctx-hpf_10x/trimmed_means.csv", file.path(data_dir, "allen_brain_reference.csv"))

  download.file("https://storage.googleapis.com/linnarsson-lab-loom/l5_all.agg.loom",
                file.path(data_dir, "l5_all.agg.loom"))
}

ct_ref <- read.csv(file.path(data_dir, "allen_brain_reference.csv"))
rownames(ct_ref) <- make.unique(ct_ref$feature)
ct_ref$feature <- NULL

metadata(sce)$top_hvgs <- top_hvgs
metadata(sce)$ct_ref <- ct_ref
sce$type <- NULL
sce$r <- NULL
sce <- clustify(sce,
                cluster_col = "clusters",
                ref_mat = ct_ref,
                query_genes = top_hvgs[1:500],
                pseudobulk_method = "median",
                compute_method = "cosine")

sce$celltype <- gsub("^X[0-9]+_", "", sce$type)
plotUMAP(sce, colour_by = "celltype")
colLabels(sce) <- sce$celltype

sce <- sce[, sce$type != "unassigned"]
mgs <- scoreMarkers(sce)

n_to_keep <- 10
mgs_fil <- lapply(names(mgs), function(i) {
  message(i)
  x <- mgs[[i]]
  # Filter and keep relevant marker genes, those with AUC > 0.7
  x <- x[x$mean.AUC > 0.7, ]
  # Sort the genes from highest to lowest weight
  x <- x[order(x$mean.AUC, decreasing = TRUE), ]
  # Add gene and cluster id to the dataframe
  x$gene <- rownames(x)
  x$cluster <- i
  n_mkrs <- min(nrow(x), n_to_keep)
  x <- x[seq_len(n_mkrs), ]
  data.frame(x)
})
mgs_df <- do.call(rbind, mgs_fil)
metadata(sce)$mkrs <- mgs_df
sce.mb <- sce
save(sce.mb, file = "pkg_data/mouse_brain_sce.rds")

library(Rsamtools)
sbp <- ScanBamParam(tag = "CB")
bamdat <- scanBam("pkg_data/5k_neuron_mouse_possort.bam", param = sbp)
cbs <- unique(na.omit(bamdat[[1]]$tag$CB))
sce.mb <- sce.mb[, intersect(cbs, colnames(sce.mb))]
sce.mb <- sce.mb[rowSums(counts(sce.mb) > 0) >= 10, ]
assay(sce.mb, "logcounts") <- NULL
save(sce.mb, file = "pkg_data/mouse_brain_sce.rds")

```


clean up 

```{bash}
rm -rf mouse_5k_neuron pkg_data/mouse_tiny.gtf
```

### Human-pbmc 10x single cell RNA-seq

This is a larger dataset that will be used for the vignette.

```{bash}
mkdir -p temp-human-pbmc-data
cd temp-human-pbmc-data

# # from https://www.10xgenomics.com/resources/datasets/10k-human-pbmcs-3-v3-1-chromium-x-with-intronic-reads-3-1-high

wget https://cf.10xgenomics.com/samples/cell-exp/6.1.2/10k_PBMC_3p_nextgem_Chromium_X_intron/10k_PBMC_3p_nextgem_Chromium_X_intron_filtered_feature_bc_matrix.tar.gz
tar zxvf 10k_PBMC_3p_nextgem_Chromium_X_intron_filtered_feature_bc_matrix.tar.gz

# ~ 30Gb file
wget https://s3-us-west-2.amazonaws.com/10x.files/samples/cell-exp/6.1.2/10k_PBMC_3p_nextgem_Chromium_X_intron/10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_genome_bam.bam
wget https://s3-us-west-2.amazonaws.com/10x.files/samples/cell-exp/6.1.2/10k_PBMC_3p_nextgem_Chromium_X_intron/10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_genome_bam.bam.bai

```

Subset the bam file to keep ~2 million unspliced reads from chr16 overlapping sites from rediportal

```{r}
load("rediportal/rediportal_coords_hg38.rda")
gr_simple <- gr_simple[seqnames(gr_simple) == "chr16", ]
gr_simple$name <- as.character(gr_simple)
gr_simple$score <- 0L
export(gr_simple, "temp-human-pbmc-data/rediportal_chr16.bed.gz")
```

```{bash}
cd temp-human-pbmc-data

samtools view \
  -@ 4 \
  -h \
  -L rediportal_chr16.bed.gz \
  10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_genome_bam.bam \
  chr16 \
  | awk 'BEGIN {OFS=FS="\t"} $1 == "@" || $6 !~ /N/' \
  | head -n 2000000 \
  | samtools view -b  \
  > 10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_chr16_rp.bam
  
samtools index -@ 4 10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_chr16_rp.bam
```

```{bash}
mv temp-human-pbmc-data/10k_PBMC_3p_nextgem_Chromium_X_intron_possorted_chr16_rp.bam* \
10x/

mv temp-human-pbmc-data/rediportal_chr16.bed.gz 10x/
```

Next make a `SingleCellExperiment` object, with cell-types and a UMAP. 

```{r}

library(clustifyr)
library(clustifyrdatahub)
data_dir <- "temp-human-pbmc-data/10k_PBMC_3p_nextgem_Chromium_X_intron"

sce <- read10xCounts(file.path(data_dir, "filtered_feature_bc_matrix"),
                     col.names = TRUE)

rownames(sce) <- uniquifyFeatureNames(rowData(sce)$ID, rowData(sce)$Symbol)

seed_val <- 42
sce <- logNormCounts(sce)
set.seed(seed_val)
dec <- modelGeneVarByPoisson(sce)
top_hvgs <- getTopHVGs(dec, prop = 0.1)
set.seed(seed_val)
sce <- denoisePCA(sce, technical=dec, subset.row=top_hvgs)
set.seed(seed_val)
sce <- runUMAP(sce, dimred="PCA")
snn.gr <- buildSNNGraph(sce, use.dimred="PCA", k=30)
colLabels(sce) <- factor(igraph::cluster_walktrap(snn.gr)$membership)
sce$clusters <- colLabels(sce)

sce$type <- NULL
sce$r <- NULL
sce <- clustify(sce,
                cluster_col = "clusters",
                ref_mat = clustifyrdatahub::ref_hema_microarray(),
                query_genes = top_hvgs[1:500])

sce$celltype <- sce$type
colLabels(sce) <- sce$celltype

sce <- sce[, sce$celltype != "unassigned"]
mgs <- scoreMarkers(sce)

n_to_keep <- 10
mgs_fil <- lapply(names(mgs), function(i) {
  message(i)
  x <- mgs[[i]]
  # Filter and keep relevant marker genes, those with AUC > 0.7
  x <- x[x$mean.AUC > 0.7, ]
  # Sort the genes from highest to lowest weight
  x <- x[order(x$mean.AUC, decreasing = TRUE), ]
  # Add gene and cluster id to the dataframe
  x$gene <- rownames(x)
  x$cluster <- i
  n_mkrs <- min(nrow(x), n_to_keep)
  x <- x[seq_len(n_mkrs), ]
  data.frame(x)
})
mgs_df <- do.call(rbind, mgs_fil)
metadata(sce)$mkrs <- mgs_df

save(sce.pbmc, file = "10x/human_pbmc_sce.rda")

```

```{bash}
rm -rf temp-human-pbmc-data
```

<details><summary>Show session info</summary>

```{r code}
sessionInfo()
```
</details>
