Tidy WiGiTS Outputs

Contents

• tidywigits
• Documentation
• Quick Start
  • Single tool
  • Full WiGiTS
• Installation
• CLI

tidywigits

Parses and tidies WiGiTS/hmftools outputs into versioned, tidy tables, written to Parquet, PostgreSQL, TSV, CSV or RDS (plus a metadata.parquet per run). Built on nemo.

Documentation

• Installation: https://tidywf.github.io/tidywigits/articles/installation
• Quickstart: https://tidywf.github.io/tidywigits/articles/quickstart
• Files supported: https://tidywf.github.io/tidywigits/articles/schema_table
• Output naming: https://tidywf.github.io/tidywigits/articles/output_naming
• Structure: https://tidywf.github.io/tidywigits/articles/structure
• Changelog: https://tidywf.github.io/tidywigits/articles/NEWS
• R6: https://tidywf.github.io/nemo/articles/structure

Quickstart

Single tool

Raw PURPLE purple.qc (key-value rows):

indir_ppl <- system.file("extdata/oa/purple", package = "tidywigits")
writeLines(readLines(file.path(indir_ppl, "sample1.purple.qc")))
#> QCStatus PASS
#> Method   NORMAL
#> CopyNumberSegments   472
#> UnsupportedCopyNumberSegments    5
#> Purity   1.0000
#> AmberGender  FEMALE
#> CobaltGender FEMALE
#> DeletedGenes 0
#> Contamination    0.0
#> GermlineAberrations  NONE
#> AmberMeanDepth   79
#> LohPercent   0.0270
#> TincLevel    0.0000
#> ChimerismPercentage  0.0000

Parse, tidy and write with Purple$run():

outdir_ppl <- file.path(tempdir(), "ppl_out")
# fmt: skip
ppl <- Purple$new(indir_ppl)
ppl$run(
  output_dir = outdir_ppl,
  format = "parquet",
  input_id = "run1"
)
list.files(outdir_ppl, pattern = "\\.parquet$")
#>  [1] "metadata_purple.parquet"                        
#>  [2] "sample1_2_purple_cnvgenetsv.parquet"            
#>  [3] "sample1_2_purple_qc.parquet"                    
#>  [4] "sample1_3_purple_cnvgenetsv.parquet"            
#>  [5] "sample1_germline_2_purple_drivercatalog.parquet"
#>  [6] "sample1_germline_purple_drivercatalog.parquet"  
#>  [7] "sample1_purple_chromarm.parquet"                
#>  [8] "sample1_purple_cnvgenetsv.parquet"              
#>  [9] "sample1_purple_cnvsomtsv.parquet"               
#> [10] "sample1_purple_germampdel.parquet"              
#> [11] "sample1_purple_germdeltsv.parquet"              
#> [12] "sample1_purple_purityrange.parquet"             
#> [13] "sample1_purple_puritytsv.parquet"               
#> [14] "sample1_purple_qc.parquet"                      
#> [15] "sample1_purple_somclonality.parquet"            
#> [16] "sample1_purple_somhist.parquet"                 
#> [17] "sample1_somatic_2_purple_drivercatalog.parquet" 
#> [18] "sample1_somatic_purple_drivercatalog.parquet"   
#> [19] "version_2_purple_version.parquet"               
#> [20] "version_purple_version.parquet"

Tidy output:

qc_file <- list.files(
  outdir_ppl,
  pattern = "sample1_purple_qc.parquet",
  full.names = TRUE
)
arrow::read_parquet(qc_file) |> str()
#> tibble [1 × 15] (S3: tbl_df/tbl/data.frame)
#>  $ input_id               : chr "run1"
#>  $ qc_status              : chr "PASS"
#>  $ method                 : chr "NORMAL"
#>  $ cn_segments            : int 472
#>  $ cn_segments_unsupported: int 5
#>  $ purity                 : num 1
#>  $ gender_amber           : chr "FEMALE"
#>  $ gender_cobalt          : chr "FEMALE"
#>  $ deleted_genes          : int 0
#>  $ contamination          : num 0
#>  $ germline_aberrations   : chr "NONE"
#>  $ mean_depth_amber       : num 79
#>  $ loh_percent            : num 0.027
#>  $ tinc_level             : num 0
#>  $ chimerism_percent      : num 0
#>  - attr(*, "file_version")= chr "latest"

Full WiGiTS

Wigits runs all tools over a parent results directory:

View input files

indir_w <- system.file("extdata/oa", package = "tidywigits")
dir_tree(indir_w, invert = TRUE, glob = "*.dvc")
/home/runner/miniconda3/envs/bump_env/lib/R/library/tidywigits/extdata/oa
├── alignments
│   ├── sample1.duplicate_freq.tsv
│   ├── sample1.md.metrics
│   └── sample1.redux.duplicate_freq.tsv
├── amber
│   ├── amber.version
│   ├── sample1.amber.baf.pcf
│   ├── sample1.amber.contamination.tsv
│   ├── sample1.amber.homozygousregion.tsv
│   ├── sample1.amber.qc
│   └── v4.2
│       └── sample1.amber.baf.pcf
├── bamtools
│   ├── sample1.bam_metric.coverage.tsv
│   ├── sample1.bam_metric.exon_coverage.tsv
│   ├── sample1.bam_metric.flag_counts.tsv
│   ├── sample1.bam_metric.frag_length.tsv
│   ├── sample1.bam_metric.gene_coverage.tsv
│   ├── sample1.bam_metric.partition_stats.tsv
│   ├── sample1.bam_metric.summary.tsv
│   └── v1.4.2
│       ├── sample1.bam_metric.exon_medians.tsv
│       ├── sample1.bam_metric.gene_coverage.tsv
│       ├── sample1.bam_metric.summary.tsv
│       └── sample1.wgsmetrics
├── chord
│   ├── sample1.chord.mutation_contexts.tsv
│   └── sample1.chord.prediction.tsv
├── cider
│   ├── sample1.cider.alignment_match.tsv.gz
│   ├── sample1.cider.locus_stats.tsv
│   ├── sample1.cider.vdj.tsv.gz
│   └── v1.0.0
│       ├── sample1.cider.blastn_match.tsv.gz
│       └── sample1.cider.vdj.tsv.gz
├── cobalt
│   ├── cobalt.version
│   ├── sample1.cobalt.gc.median.tsv
│   ├── sample1.cobalt.ratio.median.tsv
│   ├── sample1.cobalt.ratio.pcf
│   └── v2.0
│       └── sample1.cobalt.ratio.pcf
├── cuppa
│   ├── sample1.cup.data.csv
│   ├── sample1.cuppa.pred_summ.tsv
│   ├── sample1.cuppa.vis_data.tsv
│   ├── sample1.cuppa_data.tsv.gz
│   └── v1.4
│       └── sample1.cup.data.csv
├── esvee
│   ├── sample1.esvee.alignment.tsv
│   ├── sample1.esvee.assembly.tsv
│   ├── sample1.esvee.breakend.tsv
│   ├── sample1.esvee.phased_assembly.tsv
│   ├── sample1.esvee.prep.disc_stats.tsv
│   ├── sample1.esvee.prep.fragment_length.tsv
│   ├── sample1.esvee.prep.junction.tsv
│   └── v1.0
│       ├── sample1.esvee.breakend.tsv
│       └── sample1.esvee.prep.junction.tsv
├── flagstats
│   └── sample1.flagstat
├── isofox
│   ├── sample1.isf.alt_splice_junc.csv
│   ├── sample1.isf.alt_splice_junc.tsv
│   ├── sample1.isf.alt_splice_junc_unfiltered.tsv
│   ├── sample1.isf.fusions.csv
│   ├── sample1.isf.fusions.tsv
│   ├── sample1.isf.gene_collection.csv
│   ├── sample1.isf.gene_collection.tsv
│   ├── sample1.isf.gene_data.csv
│   ├── sample1.isf.gene_data.tsv
│   ├── sample1.isf.pass_fusions.csv
│   ├── sample1.isf.pass_fusions.tsv
│   ├── sample1.isf.retained_intron.csv
│   ├── sample1.isf.summary.csv
│   ├── sample1.isf.summary.tsv
│   ├── sample1.isf.transcript_data.csv
│   └── sample1.isf.transcript_data.tsv
├── lilac
│   ├── sample1.lilac.qc.tsv
│   ├── sample1.lilac.tsv
│   └── v1.7.1
│       ├── sample1.lilac.qc.tsv
│       └── sample1.lilac.tsv
├── linx
│   ├── germline_annotations
│   │   ├── linx.version
│   │   ├── sample1.linx.germline.breakend.tsv
│   │   ├── sample1.linx.germline.clusters.tsv
│   │   ├── sample1.linx.germline.disruption.tsv
│   │   ├── sample1.linx.germline.driver.catalog.tsv
│   │   ├── sample1.linx.germline.links.tsv
│   │   └── sample1.linx.germline.svs.tsv
│   ├── somatic_annotations
│   │   ├── linx.version
│   │   ├── sample1.linx.breakend.tsv
│   │   ├── sample1.linx.clusters.tsv
│   │   ├── sample1.linx.driver.catalog.tsv
│   │   ├── sample1.linx.drivers.tsv
│   │   ├── sample1.linx.fusion.tsv
│   │   ├── sample1.linx.links.tsv
│   │   ├── sample1.linx.neoepitope.tsv
│   │   ├── sample1.linx.svs.tsv
│   │   ├── sample1.linx.vis_copy_number.tsv
│   │   ├── sample1.linx.vis_fusion.tsv
│   │   ├── sample1.linx.vis_gene_exon.tsv
│   │   ├── sample1.linx.vis_protein_domain.tsv
│   │   ├── sample1.linx.vis_segments.tsv
│   │   └── sample1.linx.vis_sv_data.tsv
│   ├── v1.25
│   │   ├── germline_ann
│   │   │   ├── linx.version
│   │   │   └── sample1.linx.germline.breakend.tsv
│   │   └── somatic_ann
│   │       ├── linx.version
│   │       ├── sample1.linx.breakend.tsv
│   │       ├── sample1.linx.vis_copy_number.tsv
│   │       ├── sample1.linx.vis_fusion.tsv
│   │       ├── sample1.linx.vis_gene_exon.tsv
│   │       ├── sample1.linx.vis_protein_domain.tsv
│   │       ├── sample1.linx.vis_segments.tsv
│   │       └── sample1.linx.vis_sv_data.tsv
│   └── v2.1
│       ├── germline_ann
│       │   ├── sample1.linx.germline.breakend.tsv
│       │   ├── sample1.linx.germline.driver.catalog.tsv
│       │   └── sample1.linx.germline.svs.tsv
│       └── somatic_ann
│           ├── sample1.linx.breakend.tsv
│           ├── sample1.linx.driver.catalog.tsv
│           ├── sample1.linx.fusion.tsv
│           └── sample1.linx.svs.tsv
├── neo
│   ├── sample1.isf.neoepitope.tsv
│   ├── sample1.neo.neo_data.tsv
│   ├── sample1.neo.neoepitope.tsv
│   └── sample1.neo.peptide_scores.tsv
├── peach
│   ├── sample1.peach.events.tsv
│   ├── sample1.peach.gene.events.tsv
│   ├── sample1.peach.haplotypes.all.tsv
│   ├── sample1.peach.haplotypes.best.tsv
│   └── sample1.peach.qc.tsv
├── purple
│   ├── purple.version
│   ├── sample1.purple.chromosome_arm.tsv
│   ├── sample1.purple.cnv.gene.tsv
│   ├── sample1.purple.cnv.somatic.tsv
│   ├── sample1.purple.driver.catalog.germline.tsv
│   ├── sample1.purple.driver.catalog.somatic.tsv
│   ├── sample1.purple.germline_amp_del.tsv
│   ├── sample1.purple.purity.range.tsv
│   ├── sample1.purple.purity.tsv
│   ├── sample1.purple.qc
│   ├── sample1.purple.somatic.clonality.tsv
│   ├── sample1.purple.somatic.hist.tsv
│   ├── v4.0
│   │   ├── purple.version
│   │   ├── sample1.purple.cnv.gene.tsv
│   │   └── sample1.purple.qc
│   └── v4.2
│       ├── sample1.purple.cnv.gene.tsv
│       ├── sample1.purple.driver.catalog.germline.tsv
│       ├── sample1.purple.driver.catalog.somatic.tsv
│       └── sample1.purple.germline.deletion.tsv
├── qsee
│   ├── sample1.qsee.status.tsv.gz
│   └── sample1.qsee.vis.data.tsv.gz
├── sage
│   ├── germline
│   │   ├── sample1.sage.bqr.tsv
│   │   ├── sample2.sage.bqr.tsv
│   │   ├── sample2.sage.exon.medians.tsv
│   │   └── sample2.sage.gene.coverage.tsv
│   ├── somatic
│   │   ├── sample1.sage.bqr.tsv
│   │   ├── sample1.sage.exon.medians.tsv
│   │   ├── sample1.sage.gene.coverage.tsv
│   │   └── sample2.sage.bqr.tsv
│   └── v3.4.4
│       └── sample1.sage.bqr.tsv
├── sigs
│   ├── sample1.sig.allocation.tsv
│   └── sample1.sig.snv_counts.csv
├── teal
│   ├── sample1.teal.breakend.tsv.gz
│   └── sample1.teal.tellength.tsv
├── virusbreakend
│   └── sample1.virusbreakend.vcf.summary.tsv
└── virusinterpreter
    └── sample1.virus.annotated.tsv

Parquet:

outdir_w <- file.path(tempdir(), "wigits_out_parquet")
w <- Wigits$new(indir_w)
res <- w$run(
  output_dir = outdir_w,
  format = "parquet",
  input_id = "run1",
  output_id = "out1",
  prefix_include = TRUE
)
res # shows summary of Wigits object
#> #--- Workflow Wigits ---#
#> 
#> |var           |value                                                                     |
#> |:-------------|:-------------------------------------------------------------------------|
#> |name          |Wigits                                                                    |
#> |path          |/home/runner/miniconda3/envs/bump_env/lib/R/library/tidywigits/extdata/oa |
#> |ntools        |20                                                                        |
#> |files_total   |302                                                                       |
#> |files_matched |150                                                                       |
#> |tidied        |true                                                                      |
#> |written       |true                                                                      |
list.files(outdir_w, pattern = "\\.parquet$") |> str()
#>  chr [1:161] "metadata.parquet" "sample1_2_alignments_dupfreq.parquet" ...

PostgreSQL:

w2 <- Wigits$new(indir_w)
dbconn <- DBI::dbConnect(
  drv = RPostgres::Postgres(),
  dbname = "tidywigits",
  user = "me"
)
res <- w2$run(
  format = "db",
  input_id = "run2",
  output_id = "out2",
  prefix_include = TRUE,
  dbconn = dbconn
)

Note: VCF support is WIP.

Optional provenance columns (recommended for multi-sample/multi-run use):

Column	Purpose	User-supplied or auto-generated?
input_id	identifies the sample or input run	user
output_id	identifies the tidywigits processing run	user or auto (ULID)
input_prefix	filename prefix (e.g. sample name)	auto

Installation

From GitHub:

install.packages("remotes")
remotes::install_github("tidywf/tidywigits") # latest main commit
remotes::install_github("tidywf/tidywigits@v0.2.2") # specific version

• conda: https://anaconda.org/tidywf/r-tidywigits
• Docker: https://github.com/tidywf/tidywigits/pkgs/container/tidywigits
• more: https://tidywf.github.io/tidywigits/articles/installation

CLI

tidywigits.R is on PATH in the conda env. Otherwise:

tw_cli=$(Rscript -e 'x = system.file("cli", package = "tidywigits"); cat(x, "\n")' | xargs)
export PATH="${tw_cli}:${PATH}"

$ tidywigits.R --version
tidywigits 0.2.2

#-----------------------------------#
$ tidywigits.R --help
usage: tidywigits.R [-h] [-v] {tidy,list,sync} ...

✨ WiGiTS Output Tidying ✨

positional arguments:
  {tidy,list,sync}  sub-command help
    tidy            Tidy Workflow Outputs
    list            List Parsable Workflow Outputs
    sync            Sync Parsable Workflow Outputs From AWS S3

options:
  -h, --help        show this help message and exit
  -v, --version     show program's version number and exit
'
#-----------------------------------#
#------- Tidy ----------------------#
$ tidywigits.R tidy --help
usage: tidywigits.R tidy [-h] -d IN_DIR [-o OUTPUT_DIR] [-f FORMAT]
                         [--input_id INPUT_ID] [--output_id OUTPUT_ID |
                         --ulid] [--dbname DBNAME] [--dbuser DBUSER]
                         [--dbhost DBHOST] [--dbport DBPORT]
                         [--include INCLUDE] [--exclude EXCLUDE]
                         [--prefix_include] [-q]

options:
  -h, --help            show this help message and exit
  -d, --in_dir IN_DIR   Input directory.
  -o, --output_dir OUTPUT_DIR
                        Output directory.
  -f, --format FORMAT   Format of output [def: parquet] (parquet, db, tsv,
                        csv, rds)
  --input_id INPUT_ID   Input ID for this run.
  --output_id OUTPUT_ID
                        Output ID for this run.
  --ulid                Generate a ULID as output ID.
  --dbname DBNAME       Database name.
  --dbuser DBUSER       Database user.
  --dbhost DBHOST       Database host (default: driver/env default, e.g.
                        PGHOST).
  --dbport DBPORT       Database port (default: driver/env default, e.g.
                        PGPORT).
  --include INCLUDE     Include only these files (comma sep tool_parsers).
  --exclude EXCLUDE     Exclude only these files (comma sep tool_parsers).
  --prefix_include      Include input prefix column in output tables.
  -q, --quiet           Shush all the logs.

#-----------------------------------#
#------- List ----------------------#
$ tidywigits.R list --help
usage: tidywigits.R list [-h] -d IN_DIR [-f FORMAT] [-m MAX] [-q]

options:
  -h, --help           show this help message and exit
  -d, --in_dir IN_DIR  Input directory.
  -f, --format FORMAT  Format of list output [def: pretty] (tsv, pretty)
  -m, --max MAX        Max rows to show.
  -q, --quiet          Shush all the logs.

#-----------------------------------#
#------- Sync ----------------------#
$ tidywigits.R s3sync --help
usage: tidywigits.R [-h] [-v] {tidy,list,sync} ...
tidywigits.R: error: argument subparser_name: invalid choice: 's3sync', maybe you meant 'sync'? (choose from 'tidy', 'list', 'sync')