tidydragen
tidydragen is an R package for parsing and tidying output from Illumina’s DRAGEN secondary-analysis pipelines. The following pipelines are supported: DNA tumor-normal (somatic), DNA germline-only, RNA tumor-only, and TSO500 ctDNA.
A DRAGEN run produces dozens of files per sample across mapping, coverage, variant calling, RNA quantification, and (for the ctTSO500 app) combined-variant and coverage reports. Consuming them downstream is fragile: most metrics land in headerless section,rg,variable,count[,pct] CSVs, some files fan out into several logical tables, region/phenotype variants share basenames, and column layouts drift between DRAGEN versions.
tidydragen addresses this with a schema-driven parsing layer built on the nemo base R6 classes, supplying DRAGEN-specific schemas and parsers that turn raw outputs into consistently structured, versioned, analysis-ready tables. These can be written to Apache Parquet, PostgreSQL, TSV, CSV, or RDS. Each run also produces a metadata.parquet file alongside the tidy tables, capturing IDs, paths, and R package versions.
Documentation
- Installation: https://tidywf.github.io/tidydragen/articles/installation
- Quickstart: https://tidywf.github.io/tidydragen/articles/quickstart
- Files supported: https://tidywf.github.io/tidydragen/articles/schema_table
- Output naming: https://tidywf.github.io/tidydragen/articles/output_naming
- Structure: https://tidywf.github.io/tidydragen/articles/structure
- Changelog: https://tidywf.github.io/tidydragen/articles/NEWS
- R6: https://tidywf.github.io/nemo/articles/structure
Quickstart
Single tool
Each DRAGEN tool has its own R6 class. Most DRAGEN metrics files share a headerless section,rg,variable,count[,pct] layout e.g.:
indir_map <- system.file("extdata/dragenmap", package = "tidydragen")
writeLines(head(
readLines(
file.path(indir_map, "sampleA.mapping_metrics.csv")
),
5
))
#> TUMOR MAPPING/ALIGNING SUMMARY,,Total input reads,2635326658,100.00
#> TUMOR MAPPING/ALIGNING SUMMARY,,Number of duplicate marked reads,524058237,19.89
#> TUMOR MAPPING/ALIGNING SUMMARY,,Number of duplicate marked and mate reads removed,NA
#> TUMOR MAPPING/ALIGNING SUMMARY,,Number of unique reads (excl. duplicate marked reads),2111268421,80.11
#> TUMOR MAPPING/ALIGNING SUMMARY,,Reads with mate sequenced,2635326658,100.00We can use the DragenMap class to parse, tidy, and write these files in one call via its run() method:
outdir_map <- file.path(tempdir(), "map_out")
DragenMap$new(indir_map)$run(
output_dir = outdir_map,
format = "parquet",
input_id = "run1"
)
list.files(outdir_map, pattern = "\\.parquet$")
#> [1] "metadata_dragenmap.parquet" "sampleA_dragenmap_fraglenhist.parquet"
#> [3] "sampleA_dragenmap_gcbias.parquet" "sampleA_dragenmap_gcmain.parquet"
#> [5] "sampleA_dragenmap_metrics.parquet" "sampleA_dragenmap_time.parquet"
#> [7] "sampleA_dragenmap_trimmer.parquet" "sampleA_dragenmap_umihist.parquet"
#> [9] "sampleA_dragenmap_umimain.parquet"Now read back one tidied table:
file.path(outdir_map, "sampleA_dragenmap_metrics.parquet") |>
arrow::read_parquet() |>
str()
#> tibble [4 × 131] (S3: tbl_df/tbl/data.frame)
#> $ input_id : chr [1:4] "run1" "run1" "run1" "run1"
#> $ section : chr [1:4] "TUMOR" "NORMAL" "TUMOR" "NORMAL"
#> $ rg : chr [1:4] "Total" "Total" "CAAGCTAG+CGCTATGT.4.260813_A01052_0316_AHTYNTDSXF" "CAGTAGGC+ATTCGTCA.3.260813_A01052_0316_AHTYNTDSXF"
#> $ reads_tot_input : num [1:4] 2.64e+09 9.39e+08 NA NA
#> $ reads_tot_input_pct : num [1:4] 100 100 NA NA
#> $ reads_num_dupmarked : num [1:4] 5.24e+08 1.31e+08 5.24e+08 1.31e+08
#> $ reads_num_dupmarked_pct : num [1:4] 19.9 13.9 19.9 13.9
#> $ reads_num_dupmarked_mate_reads_removed : num [1:4] NA NA NA NA
#> $ reads_num_uniq : num [1:4] 2.11e+09 8.08e+08 2.11e+09 8.08e+08
#> $ reads_num_uniq_pct : num [1:4] 80.1 86.1 80.1 86.1
#> $ reads_w_mate_seq : num [1:4] 2.64e+09 9.39e+08 2.64e+09 9.39e+08
#> $ reads_w_mate_seq_pct : num [1:4] 100 100 100 100
#> $ reads_wo_mate_seq : num [1:4] 0 0 0 0
#> $ reads_wo_mate_seq_pct : num [1:4] 0 0 0 0
#> $ reads_qcfail : num [1:4] 0 0 0 0
#> $ reads_qcfail_pct : num [1:4] 0 0 0 0
#> $ reads_mapped : num [1:4] 2.57e+09 9.19e+08 2.57e+09 9.19e+08
#> $ reads_mapped_pct : num [1:4] 97.6 97.9 97.6 97.9
#> $ reads_mapped_r1 : num [1:4] 1.29e+09 4.61e+08 1.29e+09 4.61e+08
#> $ reads_mapped_r1_pct : num [1:4] 98.1 98.2 98.1 98.2
#> $ reads_mapped_r2 : num [1:4] 1.28e+09 4.58e+08 1.28e+09 4.58e+08
#> $ reads_mapped_r2_pct : num [1:4] 97.2 97.6 97.2 97.6
#> $ reads_mapped_pai : num [1:4] 0 0 0 0
#> $ reads_mapped_pai_pct : num [1:4] 0 0 0 0
#> $ reads_mapped_nrd : num [1:4] 45892511 16234720 45892511 16234720
#> $ reads_mapped_nrd_pct : num [1:4] 1.74 1.73 1.74 1.73
#> $ reads_mapped_pai_nrd : num [1:4] 45892511 16234720 45892511 16234720
#> $ reads_mapped_pai_nrd_pct : num [1:4] 1.74 1.73 1.74 1.73
#> $ reads_mapped_rna_rrna_filt : num [1:4] NA NA NA NA
#> $ reads_mapped_rna_chrm_excl : num [1:4] NA NA NA NA
#> $ reads_mapped_incl_refext_filt_excl : num [1:4] 2.62e+09 9.35e+08 2.62e+09 9.35e+08
#> $ reads_mapped_incl_refext_filt_excl_pct : num [1:4] 99.4 99.6 99.4 99.6
#> $ reads_unmapped : num [1:4] 62134158 19917093 62134158 19917093
#> $ reads_unmapped_pct : num [1:4] 2.36 2.12 2.36 2.12
#> $ reads_unmapped_minus_refext : num [1:4] 16241647 3682373 16241647 3682373
#> $ reads_unmapped_minus_refext_pct : num [1:4] 0.62 0.39 0.62 0.39
#> $ reads_unmapped_minus_filt : num [1:4] 62134158 19917093 62134158 19917093
#> $ reads_unmapped_minus_filt_pct : num [1:4] 2.36 2.12 2.36 2.12
#> $ reads_unmapped_minus_excl : num [1:4] 62134158 19917093 62134158 19917093
#> $ reads_unmapped_minus_excl_pct : num [1:4] 2.36 2.12 2.36 2.12
#> $ reads_unmapped_minus_refext_filt_excl : num [1:4] 16241647 3682373 16241647 3682373
#> $ reads_unmapped_minus_refext_filt_excl_pct: num [1:4] 0.62 0.39 0.62 0.39
#> $ reads_singleton : num [1:4] 14289628 3295781 14289628 3295781
#> $ reads_singleton_pct : num [1:4] 0.54 0.35 0.54 0.35
#> $ reads_paired : num [1:4] 2.56e+09 9.16e+08 2.56e+09 9.16e+08
#> $ reads_paired_pct : num [1:4] 97.1 97.5 97.1 97.5
#> $ reads_paired_proper : num [1:4] 2.55e+09 9.12e+08 2.55e+09 9.12e+08
#> $ reads_paired_proper_pct : num [1:4] 96.7 97.1 96.7 97.1
#> $ reads_discordant : num [1:4] 11520728 3939608 11520728 3939608
#> $ reads_discordant_pct : num [1:4] 0.44 0.42 0.44 0.42
#> $ reads_paired_mapped_diff_chrom : num [1:4] 3992370 1220540 3992370 1220540
#> $ reads_paired_mapped_diff_chrom_pct : num [1:4] 0.16 0.13 0.16 0.13
#> $ reads_paired_mapped_diff_chrom_mapq10 : num [1:4] 2933157 846318 2933157 846318
#> $ reads_paired_mapped_diff_chrom_mapq10_pct: num [1:4] 0.11 0.09 0.11 0.09
#> $ reads_map_multiloc : num [1:4] 1.10e+08 3.84e+07 1.10e+08 3.84e+07
#> $ reads_map_multiloc_pct : num [1:4] 4.16 4.09 4.16 4.09
#> $ reads_mapq_40_inf : num [1:4] 2.40e+09 8.57e+08 2.40e+09 8.57e+08
#> $ reads_mapq_40_inf_pct : num [1:4] 90.9 91.3 90.9 91.3
#> $ reads_mapq_30_40 : num [1:4] 7124864 2469532 7124864 2469532
#> $ reads_mapq_30_40_pct : num [1:4] 0.27 0.26 0.27 0.26
#> $ reads_mapq_20_30 : num [1:4] 17744796 6147077 17744796 6147077
#> $ reads_mapq_20_30_pct : num [1:4] 0.67 0.65 0.67 0.65
#> $ reads_mapq_10_20 : num [1:4] 28433795 10003292 28433795 10003292
#> $ reads_mapq_10_20_pct : num [1:4] 1.08 1.07 1.08 1.07
#> $ reads_mapq_0_10 : num [1:4] 1.24e+08 4.32e+07 1.24e+08 4.32e+07
#> $ reads_mapq_0_10_pct : num [1:4] 4.7 4.6 4.7 4.6
#> $ reads_mapq_na_unmapped : num [1:4] 62134158 19917093 62134158 19917093
#> $ reads_mapq_na_unmapped_pct : num [1:4] 2.36 2.12 2.36 2.12
#> $ reads_indel_r1 : num [1:4] 40110680 14217155 40110680 14217155
#> $ reads_indel_r1_pct : num [1:4] 3.1 3.08 3.1 3.08
#> $ reads_indel_r2 : num [1:4] 37398760 13618249 37398760 13618249
#> $ reads_indel_r2_pct : num [1:4] 2.92 2.97 2.92 2.97
#> $ reads_splicejunc : num [1:4] NA NA NA NA
#> $ bases_tot : num [1:4] 3.98e+11 1.42e+11 3.98e+11 1.42e+11
#> $ bases_tot_r1 : num [1:4] 1.99e+11 7.09e+10 1.99e+11 7.09e+10
#> $ bases_tot_r2 : num [1:4] 1.99e+11 7.09e+10 1.99e+11 7.09e+10
#> $ bases_mapped : num [1:4] 3.89e+11 1.39e+11 3.89e+11 1.39e+11
#> $ bases_mapped_r1 : num [1:4] 1.95e+11 6.96e+10 1.95e+11 6.96e+10
#> $ bases_mapped_r2 : num [1:4] 1.93e+11 6.92e+10 1.93e+11 6.92e+10
#> $ bases_softclip : num [1:4] 4.57e+09 1.28e+09 4.57e+09 1.28e+09
#> $ bases_softclip_pct : num [1:4] 1.18 0.93 1.18 0.93
#> $ bases_softclip_r1 : num [1:4] 1.49e+09 4.82e+08 1.49e+09 4.82e+08
#> $ bases_softclip_r1_pct : num [1:4] 0.76 0.69 0.76 0.69
#> $ bases_softclip_r2 : num [1:4] 3.08e+09 8.03e+08 3.08e+09 8.03e+08
#> $ bases_softclip_r2_pct : num [1:4] 1.59 1.16 1.59 1.16
#> $ bases_hardclip : num [1:4] 0 0 0 0
#> $ bases_hardclip_pct : num [1:4] 0 0 0 0
#> $ bases_hardclip_r1 : num [1:4] 0 0 0 0
#> $ bases_hardclip_r1_pct : num [1:4] 0 0 0 0
#> $ bases_hardclip_r2 : num [1:4] 0 0 0 0
#> $ bases_hardclip_r2_pct : num [1:4] 0 0 0 0
#> $ bases_mismatched_r1 : num [1:4] 8.57e+08 3.16e+08 8.57e+08 3.16e+08
#> $ bases_mismatched_r1_pct : num [1:4] 0.44 0.45 0.44 0.45
#> $ bases_mismatched_r2 : num [1:4] 1.83e+09 4.77e+08 1.83e+09 4.77e+08
#> $ bases_mismatched_r2_pct : num [1:4] 0.94 0.69 0.94 0.69
#> $ bases_mismatched_r1_noindels : num [1:4] 7.35e+08 2.72e+08 7.35e+08 2.72e+08
#> $ bases_mismatched_r1_noindels_pct : num [1:4] 0.38 0.39 0.38 0.39
#> $ bases_mismatched_r2_noindels : num [1:4] 1.72e+09 4.37e+08 1.72e+09 4.37e+08
#> $ bases_mismatched_r2_noindels_pct : num [1:4] 0.89 0.63 0.89 0.63
#> [list output truncated]Full DRAGEN run
A whole DRAGEN results directory can be processed with the Dragen workflow class. Tools whose files are absent contribute nothing, so the same call works across the germline, somatic tumor-normal, and RNA pipelines:
View input files
indir_d <- system.file("extdata", package = "tidydragen")
dir_tree(indir_d)
/home/runner/miniconda3/envs/bump_env/lib/R/library/tidydragen/extdata
├── dragencov
│ ├── sampleA.exon_coverage_metrics.csv
│ ├── sampleA.exon_coverage_metrics.csv.dvc
│ ├── sampleA.qc-coverage-region-umccr_cov_report.bed
│ ├── sampleA.qc-coverage-region-umccr_cov_report.bed.dvc
│ ├── sampleA.qc-coverage-region-umccr_read_cov_report.bed
│ ├── sampleA.qc-coverage-region-umccr_read_cov_report.bed.dvc
│ ├── sampleA.target_bed_coverage_metrics.csv
│ ├── sampleA.target_bed_coverage_metrics.csv.dvc
│ ├── sampleA.wgs_contig_mean_cov.csv
│ ├── sampleA.wgs_contig_mean_cov.csv.dvc
│ ├── sampleA.wgs_contig_mean_cov_normal.csv
│ ├── sampleA.wgs_contig_mean_cov_normal.csv.dvc
│ ├── sampleA.wgs_contig_mean_cov_tumor.csv
│ ├── sampleA.wgs_contig_mean_cov_tumor.csv.dvc
│ ├── sampleA.wgs_coverage_metrics.csv
│ ├── sampleA.wgs_coverage_metrics.csv.dvc
│ ├── sampleA.wgs_fine_hist.csv
│ └── sampleA.wgs_fine_hist.csv.dvc
├── dragenfqc
│ ├── sampleA.fastqc_metrics.csv
│ └── sampleA.fastqc_metrics.csv.dvc
├── dragenmap
│ ├── sampleA.fragment_length_hist.csv
│ ├── sampleA.fragment_length_hist.csv.dvc
│ ├── sampleA.gc_metrics.csv
│ ├── sampleA.gc_metrics.csv.dvc
│ ├── sampleA.mapping_metrics.csv
│ ├── sampleA.mapping_metrics.csv.dvc
│ ├── sampleA.time_metrics.csv
│ ├── sampleA.time_metrics.csv.dvc
│ ├── sampleA.trimmer_metrics.csv
│ ├── sampleA.trimmer_metrics.csv.dvc
│ ├── sampleA.umi_metrics.csv
│ └── sampleA.umi_metrics.csv.dvc
├── dragenrna
│ ├── sampleA.fusion_metrics.csv
│ ├── sampleA.fusion_metrics.csv.dvc
│ ├── sampleA.quant_metrics.csv
│ └── sampleA.quant_metrics.csv.dvc
├── dragentso
│ ├── sampleA.exon_cov_report.tsv
│ ├── sampleA.exon_cov_report.tsv.dvc
│ ├── sampleA.gene_cov_report.tsv
│ ├── sampleA.gene_cov_report.tsv.dvc
│ ├── sampleA.tmb.msaf.csv
│ ├── sampleA.tmb.msaf.csv.dvc
│ ├── sampleA.tmb.trace.tsv
│ ├── sampleA.tmb.trace.tsv.dvc
│ ├── sampleA_CombinedVariantOutput.tsv
│ ├── sampleA_CombinedVariantOutput.tsv.dvc
│ ├── sampleA_Fusions.csv
│ ├── sampleA_Fusions.csv.dvc
│ ├── sampleA_SampleAnalysisResults.json
│ └── sampleA_SampleAnalysisResults.json.dvc
└── dragenvar
├── sampleA.allele_transition_noise_metrics.csv
├── sampleA.allele_transition_noise_metrics.csv.dvc
├── sampleA.cnv_metrics.csv
├── sampleA.cnv_metrics.csv.dvc
├── sampleA.contamination.json
├── sampleA.contamination.json.dvc
├── sampleA.gvcf_metrics.csv
├── sampleA.gvcf_metrics.csv.dvc
├── sampleA.hrdscore.csv
├── sampleA.hrdscore.csv.dvc
├── sampleA.microsat_output.json
├── sampleA.microsat_output.json.dvc
├── sampleA.ploidy.vcf.gz
├── sampleA.ploidy.vcf.gz.dvc
├── sampleA.ploidy_estimation_metrics.csv
├── sampleA.ploidy_estimation_metrics.csv.dvc
├── sampleA.sv_metrics.csv
├── sampleA.sv_metrics.csv.dvc
├── sampleA.tmb.metrics.csv
├── sampleA.tmb.metrics.csv.dvc
├── sampleA.vc_hethom_ratio_metrics.csv
├── sampleA.vc_hethom_ratio_metrics.csv.dvc
├── sampleA.vc_metrics.csv
├── sampleA.vc_metrics.csv.dvc
├── sampleB.cnv_metrics.csv
├── sampleB.cnv_metrics.csv.dvc
├── sampleB.ploidy_estimation_metrics.csv
├── sampleB.ploidy_estimation_metrics.csv.dvc
├── sampleB.vc_metrics.csv
└── sampleB.vc_metrics.csv.dvcWe can parse, tidy, and write the results into e.g. Parquet format as follows:
outdir_d <- file.path(tempdir(), "dragen_out_parquet")
d <- Dragen$new(indir_d)
res <- d$run(
output_dir = outdir_d,
format = "parquet",
input_id = "run1",
output_id = "out1",
prefix_include = TRUE
)
res # shows summary of Dragen object
#> #--- Workflow Dragen ---#
#>
#> |var |value |
#> |:-------------|:----------------------------------------------------------------------|
#> |name |Dragen |
#> |path |/home/runner/miniconda3/envs/bump_env/lib/R/library/tidydragen/extdata |
#> |ntools |6 |
#> |files_total |80 |
#> |files_matched |40 |
#> |tidied |true |
#> |written |true |
list.files(outdir_d, pattern = "\\.parquet$") |> sort() |> str()
#> chr [1:64] "metadata.parquet" "sampleA_dragenfqc_posbasecontent.parquet" ...Results can also be written to a PostgreSQL database with format = "db" and a DBI connection (see the PostgreSQL article).
Three optional columns can be prepended to every written table to support downstream tracing and joining. All are opt-in and off by default, but highly recommended for any multi-sample or multi-run pipeline:
| Column | Purpose | User-supplied or auto-generated? |
|---|---|---|
input_id |
identifies the sample or input run | user |
output_id |
identifies the tidydragen processing run | user or auto (ULID) |
input_prefix |
filename prefix (e.g. sample name) | auto |
Installation
Using {remotes} directly from GitHub:
install.packages("remotes")
remotes::install_github("tidywf/tidydragen") # latest main commit
remotes::install_github("tidywf/tidydragen@v0.0.0.9001") # specific versionAlternatively:
- conda package: https://anaconda.org/tidywf/r-tidydragen
- Docker image: https://github.com/tidywf/tidydragen/pkgs/container/tidydragen
For more details see: https://tidywf.github.io/tidydragen/articles/installation
CLI
A tidydragen.R command line interface is available for convenience.
- If you’re using the conda package, the
tidydragen.Rcommand will already be available inside the activated conda environment. - If you’re not using the conda package, you need to export the
tidydragen/inst/cli/directory to yourPATHin order to usetidydragen.R.
td_cli=$(Rscript -e 'x = system.file("cli", package = "tidydragen"); cat(x, "\n")' | xargs)
export PATH="${td_cli}:${PATH}"$ tidydragen.R --version
tidydragen 0.0.0.9001
#-----------------------------------#
$ tidydragen.R --help
usage: tidydragen.R [-h] [-v] {tidy,list} ...
✨ DRAGEN Output Tidying ✨
positional arguments:
{tidy,list} sub-command help
tidy Tidy Workflow Outputs
list List Parsable Workflow Outputs
options:
-h, --help show this help message and exit
-v, --version show program's version number and exit
'
#-----------------------------------#
#------- Tidy ----------------------#
$ tidydragen.R tidy --help
usage: tidydragen.R tidy [-h] -d IN_DIR [-o OUTPUT_DIR] [-f FORMAT]
[--input_id INPUT_ID] [--output_id OUTPUT_ID |
--ulid] [--dbname DBNAME] [--dbuser DBUSER]
[--include INCLUDE] [--exclude EXCLUDE]
[--prefix_include] [-q]
options:
-h, --help show this help message and exit
-d, --in_dir IN_DIR Input directory.
-o, --output_dir OUTPUT_DIR
Output directory.
-f, --format FORMAT Format of output [def: parquet] (parquet, db, tsv,
csv, rds)
--input_id INPUT_ID Input ID for this run.
--output_id OUTPUT_ID
Output ID for this run.
--ulid Generate a ULID as output ID.
--dbname DBNAME Database name.
--dbuser DBUSER Database user.
--include INCLUDE Include only these files (comma sep tool_parsers).
--exclude EXCLUDE Exclude only these files (comma sep tool_parsers).
--prefix_include Include input prefix column in output tables.
-q, --quiet Shush all the logs.
#-----------------------------------#
#------- List ----------------------#
$ tidydragen.R list --help
usage: tidydragen.R list [-h] -d IN_DIR [-f FORMAT] [-m MAX] [-q]
options:
-h, --help show this help message and exit
-d, --in_dir IN_DIR Input directory.
-f, --format FORMAT Format of list output [def: pretty] (tsv, pretty)
-m, --max MAX Max rows to show.
-q, --quiet Shush all the logs.