From 8c30b016f4067daf79378098f466031084e410fd Mon Sep 17 00:00:00 2001 From: nsheff Date: Fri, 25 Sep 2026 17:56:31 -0400 Subject: [PATCH 1/2] Add site docs from pepspec; drop MkDocs --- .readthedocs.yaml | 15 - docs/README.md | 10 +- docs/changelog.md | 39 +- docs/code/feature1_append.md | 143 +++++++ docs/code/feature2_imply.md | 180 ++++++++ docs/code/feature3_derived.md | 175 ++++++++ docs/code/feature4_subsample_table.md | 390 ++++++++++++++++++ docs/code/feature5_amend.md | 328 +++++++++++++++ docs/code/python-api.md | 52 +++ docs/code/tutorial.md | 59 +++ docs/docs.yml | 30 ++ docs/img/peppy_logo.svg | 178 ++++++++ docs/models.md | 2 +- .../notebooks}/feature1_append.ipynb | 0 .../notebooks}/feature2_imply.ipynb | 0 .../notebooks}/feature3_derived.ipynb | 15 +- .../notebooks}/feature4_subsample_table.ipynb | 0 .../notebooks}/feature5_amend.ipynb | 0 .../notebooks}/tutorial.ipynb | 0 docs_jupyter/build/.gitignore | 2 - mkdocs.yml | 31 -- requirements/requirements-doc.txt | 6 - 22 files changed, 1584 insertions(+), 71 deletions(-) delete mode 100644 .readthedocs.yaml create mode 100644 docs/code/feature1_append.md create mode 100644 docs/code/feature2_imply.md create mode 100644 docs/code/feature3_derived.md create mode 100644 docs/code/feature4_subsample_table.md create mode 100644 docs/code/feature5_amend.md create mode 100644 docs/code/python-api.md create mode 100644 docs/code/tutorial.md create mode 100644 docs/docs.yml create mode 100644 docs/img/peppy_logo.svg rename {docs_jupyter => docs/notebooks}/feature1_append.ipynb (100%) rename {docs_jupyter => docs/notebooks}/feature2_imply.ipynb (100%) rename {docs_jupyter => docs/notebooks}/feature3_derived.ipynb (77%) rename {docs_jupyter => docs/notebooks}/feature4_subsample_table.ipynb (100%) rename {docs_jupyter => docs/notebooks}/feature5_amend.ipynb (100%) rename {docs_jupyter => docs/notebooks}/tutorial.ipynb (100%) delete mode 100644 docs_jupyter/build/.gitignore delete mode 100644 mkdocs.yml delete mode 100644 requirements/requirements-doc.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml deleted file mode 100644 index 3c789e4c..00000000 --- a/.readthedocs.yaml +++ /dev/null @@ -1,15 +0,0 @@ -version: 2 - -build: - os: ubuntu-22.04 - tools: - python: "3.10" - -mkdocs: - configuration: mkdocs.yml - fail_on_warning: false - -# Optionally declare the Python requirements required to build your docs -python: - install: - - requirements: requirements/requirements-doc.txt diff --git a/docs/README.md b/docs/README.md index 5c7dab21..1a180544 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,4 +1,12 @@ -# peppy +# peppy + +

+ + + + + +

## Introduction diff --git a/docs/changelog.md b/docs/changelog.md index 4db79aac..5878798e 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -2,6 +2,41 @@ This project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html) and [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) format. +## [0.50.0a1] -- 2025-12-22 +### Added +- Merged [`eido`](https://github.com/pepkit/eido) ([#492](https://github.com/pepkit/peppy/issues/492)) and [`pephubclient`](https://github.com/pepkit/pephubclient) into `peppy`. +- Command line interface based on `typer` for `eido` and `pephubclient`. +- Pytest for Windows. + +### Fixed +- Fixed incorrect path expanding from config file URL in Windows. +- Ambiguity ([#498](https://github.com/pepkit/peppy/issues/498)) and redundancy ([#499](https://github.com/pepkit/peppy/issues/499)) in error messages. + +## [0.40.7] -- 2024-09-30 +### Fixed +- Fixed incorrect sample table index in subtables + +## [0.40.6] -- 2024-09-10 +### Changed +- [#493](https://github.com/pepkit/peppy/issues/493) + +## [0.40.5] -- 2024-07-25 +### Fixed +- Fixed bug in initialization of peppy object from csv + +## [0.40.4] -- 2024-07-17 +### Changed +- minor change, [PR #490](https://github.com/pepkit/peppy/pull/490) + +## [0.40.3] -- 2024-07-17 +### Fixed +- [#393](https://github.com/pepkit/peppy/issues/393) +- [#480](https://github.com/pepkit/peppy/issues/480) +- [#369](https://github.com/pepkit/peppy/issues/369) +- [#399](https://github.com/pepkit/peppy/issues/399) +- [#476](https://github.com/pepkit/peppy/issues/476) +- [#471](https://github.com/pepkit/peppy/issues/471) + ## [0.40.2] -- 2024-05-28 ### Added - added `sample_name` property to samples object. @@ -110,12 +145,12 @@ _Due to the changes mentioned above, a few functionalities may be disabled. For ### Fixed - Performance issues during sample parsing. Two list comprehensions were combined to speed up this functionality. -- `KeyError` is thrown when attempting to access the `pep_version` of a `peppy.Project` instance instatiated from a sample table (`csv`) +- `KeyError` is thrown when attempting to access the `pep_version` of a `peppy.Project` instance instantiated from a sample table (`csv`) ### Added - Implementation of `__eq__` for the `peppy.Project` class such that two instances of the class can be compared using python's equality operators (`==`, `!=`). -- New `from_dict` function that lets a user instatiate a new `peppy.Project` object using an in-memory representation of a PEP (a `dict`). This supports database storage of PEPs. +- New `from_dict` function that lets a user instantiate a new `peppy.Project` object using an in-memory representation of a PEP (a `dict`). This supports database storage of PEPs. - New `extended` flag for the `to_dict` method on `peppy.Project` objects. This creates a **richer** dictionary representation of PEPs. - Better sample parsing diff --git a/docs/code/feature1_append.md b/docs/code/feature1_append.md new file mode 100644 index 00000000..9531728e --- /dev/null +++ b/docs/code/feature1_append.md @@ -0,0 +1,143 @@ +# Learn append sample modifier in `peppy` + +This vignette will show you how and why to use the append functionality of the `pepr` package. + + - basic information about this concept on the [specification website](http://pep.databio.org/en/2.0.0/specification/#sample_modifiersappend). + +## Problem/Goal +The example below demonstrates how to use the constant attributes to **define the samples attributes in the `read_type` column** of the `sample_table.csv` file. This functionality is extremely useful when there are many samples that are characterized by identical values of certain attribute (here: value `SINGLE` in `read_type` attribute). Please consider the example below for reference: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_append/" +sample_table_ori = examples_dir + "sample_table_pre.csv" +%cat $sample_table_ori | column -t -s, | cat +``` + + sample_name organism time read_type + + pig_0h pig 0 SINGLE + + pig_1h pig 1 SINGLE + + frog_0h frog 0 SINGLE + + frog_1h frog 1 SINGLE + + +## Solution +As the name suggests the attributes in the specified attributes (here: `read_type`) can be defined as constant ones. The way how this process is carried out is indicated explicitly in the `project_config.yaml` file (presented below). The name of the column is determined in the `sample_modifiers.append` key-value pair. Note that definition of more than one constant attribute is possible. + + + +```python +project_config_file = examples_dir + "project_config.yaml" +%cat $project_config_file +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + + sample_modifiers: + append: + read_type: SINGLE + + + +Let's introduce a few modifications to the original `sample_table.csv` file to use the `sample_modifiers.append` section of the config. Simply skip the attributes that are set constant and let the `pepr` do the work for you. + + +```python +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name organism time + + pig_0h pig 0 + + pig_1h pig 1 + + frog_0h frog 0 + + frog_1h frog 1 + + + +## Code +Import `peppy` and read in the project metadata by specifying the path to the `project_config.yaml`: + + +```python +from peppy import Project +p = Project(project_config_file) +``` + +And inspect it: + + +```python +print(p) +p.sample_table +``` + + Project 'example_append' (/Users/mstolarczyk/Uczelnia/UVA/code/peppy/tests/data/example_peps-cfg2/example_append/project_config.yaml) + 4 samples: pig_0h, pig_1h, frog_0h, frog_1h + Sections: pep_version, sample_table, sample_modifiers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
organismread_typesample_nametime
sample_name
pig_0hpigSINGLEpig_0h0
pig_1hpigSINGLEpig_1h1
frog_0hfrogSINGLEfrog_0h0
frog_1hfrogSINGLEfrog_1h1
+ + + +As you can see, the resulting samples are annotated the same way as if they were read from the original annotations file with attributes in the last column manually determined. diff --git a/docs/code/feature2_imply.md b/docs/code/feature2_imply.md new file mode 100644 index 00000000..b24fb944 --- /dev/null +++ b/docs/code/feature2_imply.md @@ -0,0 +1,180 @@ +# Learn implied sample modifier in `peppy` + +This vignette will show you how and why to use the implied attributes functionality of the `peppy` package. + + - basic information about the PEP concept on the [project website](http://pep.databio.org/en/2.0.0) + + - broader theoretical description in the implied attributes [documentation section](http://pep.databio.org/en/2.0.0/specification/#sample_modifiersimply). + +## Problem/Goal +The example below demonstrates how and why to use implied attributes functionality to **save your time and effort** in case multiple sample attributes need to be defined for many samples and they **follow certain patterns**. Please consider the example below for reference: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_imply/" +sample_table_ori = examples_dir + "sample_table_pre.csv" +%cat $sample_table_ori | column -t -s, | cat +``` + + sample_name organism time file_path genome genome_size + frog_0h frog 0 data/lab/project/frog_0h.fastq + frog_1h frog 1 data/lab/project/frog_1h.fastq + human_1h human 1 data/lab/project/human_1h.fastq hg38 hs + human_0h human 0 data/lab/project/human_0h.fastq hg38 hs + mouse_1h mouse 1 data/lab/project/mouse_1h.fastq mm10 mm + mouse_0h mouse 0 data/lab/project/mouse_1h.fastq mm10 mm + + +## Solution +Noticeably, the samples with attributes `human` and `mouse` (in the `organism` column) follow two distinct patterns here. They have additional attributes in attributes `genome` and `genome_size` in the `sample_table.csv` file. Consequently you can use implied attributes to add those attributes to the sample annotations (set global, species-level attributes at the project level instead of duplicating that information for every sample that belongs to a species). The way how this process is carried out is indicated explicitly in the `project_config.yaml` file (presented below). + + +```python +project_config_file = examples_dir + "project_config.yaml" +%cat $project_config_file +``` + + pep_version: '2.0.0' + sample_table: sample_table.csv + looper: + output_dir: $HOME/hello_looper_results + + sample_modifiers: + imply: + - if: + organism: human + then: + genome: hg38 + macs_genome_size: hs + - if: + organism: mouse + then: + genome: mm10 + macs_genome_size: mm + +Consequently, you can design `sample_modifiers.imply` - a multi-level key-value section in the `project_config.yaml` file. Note that the keys must match the column names and attributes in the `sample_annotations.csv` file. + +Let's introduce a few modifications to the original `sample_table.csv` file to use the `sample_modifiers.imply` section of the config. Simply skip the attributes that will be implied and let the `peppy` do the work for you. + + +```python +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name organism time file_path + frog_0h frog 0 data/lab/project/frog_0h.fastq + frog_1h frog 1 data/lab/project/frog_1h.fastq + human_1h human 1 data/lab/project/human_1h.fastq + human_0h human 0 data/lab/project/human_0h.fastq + mouse_1h mouse 1 data/lab/project/mouse_1h.fastq + mouse_0h mouse 0 data/lab/project/mouse_1h.fastq + + +## Code +Load `peppy` and read in the project metadata by specifying the path to the `project_config.yaml`: + + +```python +from peppy import Project +p = Project(project_config_file) +``` + +And inspect it: + + +```python +print(p) +p.sample_table +``` + + Project 'example_imply' (/Users/mstolarczyk/Uczelnia/UVA/code/peppy/tests/data/example_peps-cfg2/example_imply/project_config.yaml) + 6 samples: frog_0h, frog_1h, human_1h, human_0h, mouse_1h, mouse_0h + Sections: pep_version, sample_table, looper, sample_modifiers + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
file_pathorganismsample_nametimegenomemacs_genome_size
sample_name
frog_0hdata/lab/project/frog_0h.fastqfrogfrog_0h0NaNNaN
frog_1hdata/lab/project/frog_1h.fastqfrogfrog_1h1NaNNaN
human_1hdata/lab/project/human_1h.fastqhumanhuman_1h1hg38hs
human_0hdata/lab/project/human_0h.fastqhumanhuman_0h0hg38hs
mouse_1hdata/lab/project/mouse_1h.fastqmousemouse_1h1mm10mm
mouse_0hdata/lab/project/mouse_1h.fastqmousemouse_0h0mm10mm
+ + + +As you can see, the resulting samples are annotated the same way as if they were read from the original annotations file with attributes in the two last columns manually determined. diff --git a/docs/code/feature3_derived.md b/docs/code/feature3_derived.md new file mode 100644 index 00000000..eebc8a6a --- /dev/null +++ b/docs/code/feature3_derived.md @@ -0,0 +1,175 @@ +# Learn derived attributes in `peppy` + +This vignette will show you how and why to use the derived attributes functionality of the `peppy` package. + + - basic information about the PEP concept on the [project website](https://pepkit.github.io/). + + - broader theoretical description in the derived attributes [documentation section](http://pep.databio.org/en/2.0.0/specification/#sample_modifiersderive). + +## What are derived attributes? + +Derived attributes allow you to construct sample attributes dynamically using patterns and variables from other columns in your sample table. This is especially useful for building file paths without having to repeat directory structures for every sample. + +## Quick Start - Simple Example + +Let's say you have samples with names like `sample1`, `sample2`, etc., and your data files follow a pattern like `/data/{sample_name}.fastq`. Instead of typing out the full path for each sample, you can use derived attributes. + +**Your sample table (sample_table.csv):** +```csv +sample_name +sample1 +sample2 +sample3 +``` + +**Your project config (project_config.yaml):** +```yaml +pep_version: "2.0.0" +sample_table: sample_table.csv +sample_modifiers: + derive: + attributes: [file_path] + sources: + my_source: /data/{sample_name}.fastq +``` + +**In your sample table, add the file_path column:** +```csv +sample_name,file_path +sample1,my_source +sample2,my_source +sample3,my_source +``` + +When peppy loads this PEP, it will automatically expand `my_source` using the pattern from your config, replacing `{sample_name}` with the actual sample name. The result: +- `sample1` → `/data/sample1.fastq` +- `sample2` → `/data/sample2.fastq` +- `sample3` → `/data/sample3.fastq` + +That's it! The `my_source` identifier gets replaced with the pattern defined in your config, and any variables in curly braces (like `{sample_name}`) get replaced with values from that sample's row. + +## Advanced Example - Multiple Sources + +The example below demonstrates how to use derived attributes in a more complex scenario with **multiple data sources and multiple variables**. Please consider the example below for reference: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_derive/" +sample_table_pre = examples_dir + "sample_table_pre.csv" +%cat $sample_table_pre | column -t -s, | cat +``` + + sample_name protocol organism time file_path + pig_0h RRBS pig 0 data/lab/project/pig_0h.fastq + pig_1h RRBS pig 1 data/lab/project/pig_1h.fastq + frog_0h RRBS frog 0 data/lab/project/frog_0h.fastq + frog_1h RRBS frog 1 data/lab/project/frog_1h.fastq + + +## Solution +As the name suggests the attributes in the specified attributes (here: `file_path`) can be derived from other ones. The way how this process is carried out is indicated explicitly in the `project_config.yaml` file (presented below). The name of the column is determined in the `sample_modifiers.derive.attributes` key-value pair, whereas the pattern for the attributes construction - in the `sample_modifiers.derive.sources` one. Note that the second level key (here: `source`) has to exactly match the attributes in the `file_path` column of the modified `sample_annotation.csv` (presented below). + + +```python +project_config = examples_dir + "project_config.yaml" +%cat $project_config | column -t -s, | cat +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + output_dir: "$HOME/hello_looper_results" + sample_modifiers: + derive: + attributes: [file_path] + sources: + source1: $HOME/data/lab/project/{organism}_{time}h.fastq + source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq + + +Let's introduce a few modifications to the original `sample_annotation.csv` file to map the appropriate data sources from the `project_config.yaml` with attributes in the derived column - `[file_path]`: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_derive/" +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol organism time file_path + pig_0h RRBS pig 0 source1 + pig_1h RRBS pig 1 source1 + frog_0h RRBS frog 0 source1 + frog_1h RRBS frog 1 source1 + + +## Code +Import `peppy` and read in the project metadata by specifying the path to the `project_config.yaml`: + + +```python +from peppy import Project +p = Project(project_config) +p.sample_table +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
file_pathorganismprotocolsample_nametime
sample_name
pig_0h/Users/mstolarczyk/data/lab/project/pig_0h.fastqpigRRBSpig_0h0
pig_1h/Users/mstolarczyk/data/lab/project/pig_1h.fastqpigRRBSpig_1h1
frog_0h/Users/mstolarczyk/data/lab/project/frog_0h.fastqfrogRRBSfrog_0h0
frog_1h/Users/mstolarczyk/data/lab/project/frog_1h.fastqfrogRRBSfrog_1h1
+ + + +As you can see, the resulting samples are annotated the same way as if they were read from the original, unwieldy, annotations file. diff --git a/docs/code/feature4_subsample_table.md b/docs/code/feature4_subsample_table.md new file mode 100644 index 00000000..2660f449 --- /dev/null +++ b/docs/code/feature4_subsample_table.md @@ -0,0 +1,390 @@ +# Learn sample subannotations in `peppy` + +This vignette will show you how and why to use the subsample table functionality of the `peppy` package. + + - basic information about the PEP concept visit the [project website](http://pep.databio.org/en/2.0.0/). + + - broader theoretical description in the subsample table [documentation section](http://pep.databio.org/en/2.0.0/specification/#project-attribute-subsample_table). + +## Problem/Goal + +This series of examples below demonstrates how and why to use sample subannoatation functionality in multiple cases to **provide multiple input files of the same type for a single sample**. + +## Solutions + +### Example 1: basic sample subannotation table + +This example demonstrates how the sample subannotation functionality is used. In this example, 2 samples have multiple input files that need merging (`frog_1` and `frog_2`), while 1 sample (`frog_3`) does not. Therefore, `frog_3` specifies its file in the `sample_table.csv` file, while the others leave that field blank and instead specify several files in the `subsample_table.csv` file. + +This example is made up of these components: + +* Project config file: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_subtable1/" +project_config = examples_dir + "project_config.yaml" +%cat $project_config +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + subsample_table: subsample_table.csv + output_dir: $HOME/example_results + + +* Sample table: + + +```python +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol file + frog_1 anySampleType multi + frog_2 anySampleType multi + frog_3 anySampleType multi + + +* Subsample table: + + +```python +subsample_table = examples_dir + "subsample_table.csv" +%cat $subsample_table | column -t -s, | cat +``` + + column: line too long + sample_name subsample_name file + frog_1 sub_a data/frog1a_data.txt + frog_1 sub_b data/frog1b_data.txt + frog_1 sub_c data/frog1c_data.txt + frog_2 sub_a data/frog2a_data.txt + + +Let's load the project config, create the Project object and see if multiple files are present + + +```python +from peppy import Project +p = Project(project_config) +samples = p.sample_table +samples +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
fileprotocolsample_namesubsample_name
sample_name
frog_1[data/frog1a_data.txt, data/frog1b_data.txt, d...anySampleTypefrog_1[sub_a, sub_b, sub_c]
frog_2[data/frog2a_data.txt, data/frog2b_data.txt]anySampleTypefrog_2[sub_a, sub_b]
frog_3multianySampleTypefrog_3NaN
+ + + +### Example 2: subannotations and derived attributes + +This example uses a `subsample_table.csv` file and a derived attributes to point to files. This is a rather complex example. Notice we must include the `file_id` column in the `sample_table.csv` file, and leave it blank; this is then populated by just some of the samples (`frog_1` and `frog_2`) in the `subsample_table.csv`, but is left empty for the samples that are not merged. + +This example is made up of these components: + +* Project config file: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_subtable2/" +project_config = examples_dir + "project_config.yaml" +%cat $project_config +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + subsample_table: subsample_table.csv + output_dir: $HOME/hello_looper_results + pipeline_interfaces: [../pipeline/pipeline_interface.yaml] + + sample_modifiers: + derive: + attributes: [file] + sources: + local_files: "../data/{identifier}{file_id}_data.txt" + local_files_unmerged: "../data/{identifier}_data.txt" + +* Sample table: + + +```python +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + column: line too long + sample_name protocol identifier file + frog_1 anySampleType frog1 local_files + frog_2 anySampleType frog2 local_files + frog_3 anySampleType frog3 local_files_unmerged + + +* Subsample table: + + +```python +subsample_table = examples_dir + "subsample_table.csv" +%cat $subsample_table | column -t -s, | cat +``` + + column: line too long + sample_name file_id subsample_name + frog_1 a a + frog_1 b b + frog_1 c c + frog_2 a a + + +Let's load the project config, create the Project object and see if multiple files are present + + +```python +p = Project(project_config) +samples = p.sample_table +samples +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
filefile_ididentifierprotocolsample_namesubsample_name
sample_name
frog_1[../data/frog1a_data.txt, ../data/frog1b_data....[a, b, c]frog1anySampleTypefrog_1[a, b, c]
frog_2[../data/frog2a_data.txt, ../data/frog2b_data....[a, b]frog2anySampleTypefrog_2[a, b]
frog_3../data/frog3_data.txtNaNfrog3anySampleTypefrog_3NaN
frog_4../data/frog4_data.txtNaNfrog4anySampleTypefrog_4NaN
+ + + +### Example 3: subannotations and expansion characters + +This example gives the exact same results as Example 2, but in this case, uses a wildcard for `frog_2` instead of including it in the `subsample_table.csv` file. Since we can't use a wildcard and a subannotation for the same sample, this necessitates specifying a second data source class (`local_files_unmerged`) that uses an asterisk (`*`). The outcome is the same (`file` columns match). + +This example is made up of these components: + +* Project config file: + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_subtable3/" +# need to cd to the example dir so that the glob works as expected +%cd $examples_dir +project_config = "project_config.yaml" +%cat $project_config +``` + + /Users/mstolarczyk/Uczelnia/UVA/code/peppy/tests/data/example_peps-cfg2/example_subtable3 + pep_version: "2.0.0" + sample_table: sample_table.csv + subsample_table: subsample_table.csv + output_dir: $HOME/hello_looper_results + pipeline_interfaces: [../pipeline/pipeline_interface.yaml] + + sample_modifiers: + derive: + attributes: [file] + sources: + local_files: "../data/{identifier}{file_id}_data.txt" + local_files_unmerged: "../data/{identifier}*_data.txt" + + + +* Sample table: + + +```python +%cat sample_table.csv | column -t -s, | cat +``` + + sample_name protocol identifier file file_id + frog_1 anySampleType frog1 local_files + frog_2 anySampleType frog2 local_files_unmerged + frog_3 anySampleType frog3 local_files_unmerged + frog_4 anySampleType frog4 local_files_unmerged + + +* Subsample table: + + +```python +%cat subsample_table.csv | column -t -s, | cat +``` + + sample_name file_id + frog_1 a + frog_1 b + frog_1 c + + +Let's load the project config, create the Project object and see if multiple files are present + + +```python +p = Project(project_config) +samples = p.sample_table +samples +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
filefile_ididentifierprotocolsample_namesubsample_name
sample_name
frog_1[../data/frog1a_data.txt, ../data/frog1b_data....[a, b, c]frog1anySampleTypefrog_1[0, 1, 2]
frog_2[../data/frog2_data.txt, ../data/frog2a_data.t...NaNfrog2anySampleTypefrog_2NaN
frog_3../data/frog3_data.txtNaNfrog3anySampleTypefrog_3NaN
frog_4../data/frog4_data.txtNaNfrog4anySampleTypefrog_4NaN
+ + diff --git a/docs/code/feature5_amend.md b/docs/code/feature5_amend.md new file mode 100644 index 00000000..28462759 --- /dev/null +++ b/docs/code/feature5_amend.md @@ -0,0 +1,328 @@ +# Learn amendments in `peppy` + +This vignette will show you how and why to use the amendments functionality of the `peppy` package. + + - basic information about the PEP concept on the [project website](http://pep.databio.org/en/2.0.0/). + + - broader theoretical description in the amendments [documentation section](http://pep.databio.org/en/2.0.0/specification/#project-attribute-amendments). + + +## Problem/Goal + +The example below demonstrates how and why to use amendments project attribute to, e.g. **define numerous similar projects in a single project config file**. This functionality is extremely convenient when one has to define projects with small settings discreptancies, like different attributes in the annotation sheet. For example libraries `ABCD` and `EFGH` instead of the original `RRBS`. + + +```python +examples_dir = "../tests/data/example_peps-cfg2/example_amendments1/" +sample_table = examples_dir + "sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol organism time file_path + + pig_0h RRBS pig 0 source1 + + pig_1h RRBS pig 1 source1 + + frog_0h RRBS frog 0 source1 + + frog_1h RRBS frog 1 source1 + + +## Solution + +This can be achieved by using amendments section of `project_config.yaml` file (presented below). The attributes specified in the lowest levels of this section (here: `sample_table`) overwrite the original ones. Consequently, a completely new set of settings is determined with just this value changed. Moreover, multiple amendments can be defined in a single config file *and* activated at the same time. Based on the file presented below, two subprojects will be defined: `newLib` and `newLib2`. + + +```python +project_config_file = examples_dir + "project_config.yaml" +%cat $project_config_file +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + output_dir: $HOME/hello_looper_results + + sample_modifiers: + derive: + attributes: [file_path] + sources: + source1: /data/lab/project/{organism}_{time}h.fastq + source2: /path/from/collaborator/weirdNamingScheme_{external_id}.fastq + project_modifiers: + amend: + newLib: + sample_table: sample_table_newLib.csv + newLib2: + sample_table: sample_table_newLib2.csv + + +Obviously, the amendments functionality can be combined with other `peppy` package options, e.g. imply and derive sample modifiers. The derive modifier is used in the example considered here (`derive` key in the `sample_modifiers` section of the config file). + + +Files `sample_table_newLib.csv` and `sample_table_newLib2.csv` introduce different the `library` attributes. They are used in the subprojects `newLib` and `newLib2`, respectively + + +```python +sample_table = examples_dir + "sample_table_newLib.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol organism time file_path + pig_0h ABCD pig 0 source1 + pig_1h ABCD pig 1 source1 + frog_0h ABCD frog 0 source1 + frog_1h ABCD frog 1 source1 + + + +```python +sample_table = examples_dir + "sample_table_newLib2.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol organism time file_path + pig_0h EFGH pig 0 source1 + pig_1h EFGH pig 1 source1 + frog_0h EFGH frog 0 source1 + frog_1h EFGH frog 1 source1 + + +## Code + +Import `peppy` and read in the project metadata by specifying the path to the `project_config.yaml` + + +```python +from peppy import Project +p = Project(project_config_file) +``` + +An appropriate message is displayed, which informs you what are the names of the amendments that you have defined in the `project_config.yaml` file. Nonetheless, just the main project is "active". + +Let's inspect it: + + +```python +p.sample_table +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
file_pathorganismprotocolsample_nametime
sample_name
pig_0h/data/lab/project/pig_0h.fastqpigRRBSpig_0h0
pig_1h/data/lab/project/pig_1h.fastqpigRRBSpig_1h1
frog_0h/data/lab/project/frog_0h.fastqfrogRRBSfrog_0h0
frog_1h/data/lab/project/frog_1h.fastqfrogRRBSfrog_1h1
+ + + +The column `file_path` was derived and the `library` column holds the original attributes: `RRBS` for each sample. + +To "activate" any of the amendments just pass the names of the desired amendments to the `amendments` argument in the `Project` object constructor. + +In case you don't remember the subproject names run the `listAmendments()` metohods on the `Project` object, just like that: + + +```python +p.list_amendments +``` + + + + + ['newLib', 'newLib2'] + + + + +```python +p_new_lib = Project(project_config_file, amendments = "newLib") +``` + +Let's inspect it: + + +```python +p_new_lib.sample_table +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
file_pathorganismprotocolsample_nametime
sample_name
pig_0h/data/lab/project/pig_0h.fastqpigABCDpig_0h0
pig_1h/data/lab/project/pig_1h.fastqpigABCDpig_1h1
frog_0h/data/lab/project/frog_0h.fastqfrogABCDfrog_0h0
frog_1h/data/lab/project/frog_1h.fastqfrogABCDfrog_1h1
+ + + +As you can see, the `library` column consists of new attributes (`ABCD`), which were defined in the `sample_table_newLib.csv` file. + +Amendments can be also activated interactively, after `Project` object has been crated. Let's activate the second amendment this way: + + +```python +p_new_lib2 = p.activate_amendments("newLib2") +p_new_lib2.sample_table +``` + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
file_pathorganismprotocolsample_nametime
sample_name
pig_0h/data/lab/project/pig_0h.fastqpigEFGHpig_0h0
pig_1h/data/lab/project/pig_1h.fastqpigEFGHpig_1h1
frog_0h/data/lab/project/frog_0h.fastqfrogEFGHfrog_0h0
frog_1h/data/lab/project/frog_1h.fastqfrogEFGHfrog_1h1
+ + diff --git a/docs/code/python-api.md b/docs/code/python-api.md new file mode 100644 index 00000000..12976153 --- /dev/null +++ b/docs/code/python-api.md @@ -0,0 +1,52 @@ +# Package `peppy` Documentation + +## Package Overview + +The `peppy` package provides a Python interface for working with Portable Encapsulated Projects (PEPs). A PEP is a standardized format for organizing metadata for biological samples and sample-intensive data. + +### Key Features + +- **Project Management**: Create and manage collections of samples with metadata +- **Sample Access**: Retrieve individual samples and their attributes +- **Amendments**: Activate different project configurations +- **Validation**: Validate projects against schemas + +### Installation + +```bash +pip install peppy +``` + +### Quick Example + +```python +from peppy import Project + +# Initialize with a project config file +prj = Project(cfg="ngs.yaml") + +# Access samples +samples = prj.samples +``` + +## API Reference + +### Project Class + +The main class for working with PEPs: + +::: peppy.Project + options: + docstring_style: google + show_source: true + show_signature: true + merge_init_into_class: true + +### Sample Class + +::: peppy.Sample + options: + docstring_style: google + show_source: true + show_signature: true + merge_init_into_class: true diff --git a/docs/code/tutorial.md b/docs/code/tutorial.md new file mode 100644 index 00000000..8cedf886 --- /dev/null +++ b/docs/code/tutorial.md @@ -0,0 +1,59 @@ +# Basic PEP example + +This vignette will show you a simple example PEP-formatted project, and how to read it into python using the `peppy` package. This example comes from the [example_peps repository](https://github.com/pepkit/example_peps) in the [example_basic](https://github.com/pepkit/example_peps/tree/master/example_basic) folder. + + +Start by importing `peppy`, and then let's take a look at the configuration file that defines our project: + + +```python +import peppy +examples_dir = "../tests/data/example_peps-cfg2/" +``` + + +```python +project_config_file = examples_dir + "example_basic/project_config.yaml" +%cat $project_config_file +``` + + pep_version: "2.0.0" + sample_table: sample_table.csv + +It's a basic YAML-formatted file with couple of sections: +- `pep_version` (required; indicates with PEP spec version this config subscribes to) +- `sample_table` (suggested; a pointer to sample annotation sheet. It is stored in the same folder as `project_config.yaml`) + +Now, let's now glance at that annotation file: + + +```python +sample_table = examples_dir + "example_basic/sample_table.csv" +%cat $sample_table | column -t -s, | cat +``` + + sample_name protocol file + frog_1 anySampleType data/frog1_data.txt + frog_2 anySampleType data/frog2_data.txt + + +This `sample_table.csv` file is a basic CSV file, with rows corresponding to samples, and columns corresponding to sample attributes. Let's read this simple example project into Python using `peppy`: + + +```python +project = peppy.Project(examples_dir + "example_basic/project_config.yaml") +``` + +Now, we have access to all the project metadata in easy-to-use form using python objects. We can browse the samples in the project like this: + + +```python +project.samples[0].file +``` + + + + + 'data/frog1_data.txt' + + diff --git a/docs/docs.yml b/docs/docs.yml new file mode 100644 index 00000000..22f799c0 --- /dev/null +++ b/docs/docs.yml @@ -0,0 +1,30 @@ +site_name: Peppy +nav: + - Peppy: README.md + - Getting started: + - Installing and Hello World: hello-world.md + - How-to guides: + - How to initialize a Project: initialize.md + - How to use peppy: code/tutorial.md + - How to use subsample table: code/feature4_subsample_table.md + - How to use amendments: code/feature5_amend.md + - How to use append sample modifier: code/feature1_append.md + - How to use imply sample modifier: code/feature2_imply.md + - How to validate a PEP: validating.md + - Reference: + - Project models: models.md + - How to cite: /citations/ + - API: code/python-api.md + - Support: support.md + - Contributing: contributing.md + - Changelog: changelog.md + +renderers: + - hub-notebooks: + include: [notebooks/*.ipynb] + execute: false + - hub-python-api: + package_root: .. + packages: [peppy] + include: [code/python-api.md] + docstring_style: google diff --git a/docs/img/peppy_logo.svg b/docs/img/peppy_logo.svg new file mode 100644 index 00000000..cf21780c --- /dev/null +++ b/docs/img/peppy_logo.svg @@ -0,0 +1,178 @@ + + + +image/svg+xmlPeppy diff --git a/docs/models.md b/docs/models.md index 5f4628d5..bc053ca0 100644 --- a/docs/models.md +++ b/docs/models.md @@ -26,7 +26,7 @@ and used. ## Extending sample objects -By default we use *generic* models (see [API docs](autodoc_build/peppy.md) for more) that can be used in many contexts +By default we use *generic* models (see [Peppy API docs](code/python-api.md) for more) that can be used in many contexts via Python import, or by object serialization and deserialization via YAML. Since these models provide useful methods to store, update, and read attributes in the objects created from them diff --git a/docs_jupyter/feature1_append.ipynb b/docs/notebooks/feature1_append.ipynb similarity index 100% rename from docs_jupyter/feature1_append.ipynb rename to docs/notebooks/feature1_append.ipynb diff --git a/docs_jupyter/feature2_imply.ipynb b/docs/notebooks/feature2_imply.ipynb similarity index 100% rename from docs_jupyter/feature2_imply.ipynb rename to docs/notebooks/feature2_imply.ipynb diff --git a/docs_jupyter/feature3_derived.ipynb b/docs/notebooks/feature3_derived.ipynb similarity index 77% rename from docs_jupyter/feature3_derived.ipynb rename to docs/notebooks/feature3_derived.ipynb index 9bcc1123..fb44bfdd 100644 --- a/docs_jupyter/feature3_derived.ipynb +++ b/docs/notebooks/feature3_derived.ipynb @@ -3,18 +3,7 @@ { "cell_type": "markdown", "metadata": {}, - "source": [ - "# Learn derived attributes in `peppy`\n", - "\n", - "This vignette will show you how and why to use the derived attributes functionality of the `peppy` package. \n", - "\n", - " - basic information about the PEP concept on the [project website](https://pepkit.github.io/).\n", - " \n", - " - broader theoretical description in the derived attributes [documentation section](http://pep.databio.org/en/2.0.0/specification/#sample_modifiersderive).\n", - "\n", - "## Problem/Goal\n", - "The example below demonstrates how to use the derived attributes to **flexibly define the samples attributes the `file_path` column** of the `sample_table.csv` file to match the file names in your project. Please consider the example below for reference:" - ] + "source": "# Learn derived attributes in `peppy`\n\nThis vignette will show you how and why to use the derived attributes functionality of the `peppy` package. \n\n - basic information about the PEP concept on the [project website](https://pepkit.github.io/).\n \n - broader theoretical description in the derived attributes [documentation section](http://pep.databio.org/en/2.0.0/specification/#sample_modifiersderive).\n\n## What are derived attributes?\n\nDerived attributes allow you to construct sample attributes dynamically using patterns and variables from other columns in your sample table. This is especially useful for building file paths without having to repeat directory structures for every sample.\n\n## Quick Start - Simple Example\n\nLet's say you have samples with names like `sample1`, `sample2`, etc., and your data files follow a pattern like `/data/{sample_name}.fastq`. Instead of typing out the full path for each sample, you can use derived attributes.\n\n**Your sample table (sample_table.csv):**\n```csv\nsample_name\nsample1\nsample2\nsample3\n```\n\n**Your project config (project_config.yaml):**\n```yaml\npep_version: \"2.0.0\"\nsample_table: sample_table.csv\nsample_modifiers:\n derive:\n attributes: [file_path]\n sources:\n my_source: /data/{sample_name}.fastq\n```\n\n**In your sample table, add the file_path column:**\n```csv\nsample_name,file_path\nsample1,my_source\nsample2,my_source\nsample3,my_source\n```\n\nWhen peppy loads this PEP, it will automatically expand `my_source` using the pattern from your config, replacing `{sample_name}` with the actual sample name. The result:\n- `sample1` → `/data/sample1.fastq`\n- `sample2` → `/data/sample2.fastq`\n- `sample3` → `/data/sample3.fastq`\n\nThat's it! The `my_source` identifier gets replaced with the pattern defined in your config, and any variables in curly braces (like `{sample_name}`) get replaced with values from that sample's row.\n\n## Advanced Example - Multiple Sources\n\nThe example below demonstrates how to use derived attributes in a more complex scenario with **multiple data sources and multiple variables**. Please consider the example below for reference:" }, { "cell_type": "code", @@ -249,4 +238,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} +} \ No newline at end of file diff --git a/docs_jupyter/feature4_subsample_table.ipynb b/docs/notebooks/feature4_subsample_table.ipynb similarity index 100% rename from docs_jupyter/feature4_subsample_table.ipynb rename to docs/notebooks/feature4_subsample_table.ipynb diff --git a/docs_jupyter/feature5_amend.ipynb b/docs/notebooks/feature5_amend.ipynb similarity index 100% rename from docs_jupyter/feature5_amend.ipynb rename to docs/notebooks/feature5_amend.ipynb diff --git a/docs_jupyter/tutorial.ipynb b/docs/notebooks/tutorial.ipynb similarity index 100% rename from docs_jupyter/tutorial.ipynb rename to docs/notebooks/tutorial.ipynb diff --git a/docs_jupyter/build/.gitignore b/docs_jupyter/build/.gitignore deleted file mode 100644 index d6b7ef32..00000000 --- a/docs_jupyter/build/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -* -!.gitignore diff --git a/mkdocs.yml b/mkdocs.yml deleted file mode 100644 index c2989c63..00000000 --- a/mkdocs.yml +++ /dev/null @@ -1,31 +0,0 @@ -site_name: Peppy -site_url: http://peppy.databio.org/ -repo_url: http://github.com/pepkit/peppy -pypi_name: peppy - -nav: - - Getting started: - - Introduction: README.md - - Installing and Hello World: hello-world.md - - How-to Guides: - - How to initialize a Project: initialize.md - - How to use peppy: tutorial.md - - How to use subsample table: feature4_subsample_table.md - - How to use amendments: feature5_amend.md - - How to use append sample modifier: feature1_append.md - - How to use imply sample modifier: feature2_imply.md - - How to validate a PEP: validating.md - - Reference: - - API: autodoc_build/peppy.md - - Support: support.md - - Contributing: contributing.md - - Changelog: changelog.md - -theme: databio - -plugins: - - databio: - autodoc_build: "docs/autodoc_build" - autodoc_package: "peppy" - no_top_level: true - - search diff --git a/requirements/requirements-doc.txt b/requirements/requirements-doc.txt deleted file mode 100644 index 74c8d06a..00000000 --- a/requirements/requirements-doc.txt +++ /dev/null @@ -1,6 +0,0 @@ -https://github.com/databio/mkdocs-databio/archive/master.zip -markdown-include -mkdocs>=1.0 -peppy -pydoc-markdown -ubiquerg From 5da62ea49bbef734f9a2460bdba6904e4d7912b7 Mon Sep 17 00:00:00 2001 From: nsheff Date: Fri, 25 Sep 2026 17:56:31 -0400 Subject: [PATCH 2/2] Notify doc-hubs on docs changes --- .github/workflows/notify-docs.yml | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 .github/workflows/notify-docs.yml diff --git a/.github/workflows/notify-docs.yml b/.github/workflows/notify-docs.yml new file mode 100644 index 00000000..22cfb547 --- /dev/null +++ b/.github/workflows/notify-docs.yml @@ -0,0 +1,25 @@ +name: Notify pepkit docs + +# Tells databio/doc-hubs that this repo's docs changed, so pep.databio.org +# rebuilds. All the logic is in the reusable workflow; this file only names +# which site and which spoke. See databio/doc-hubs README, "Add a spoke". + +on: + push: + # Must equal this spoke's `ref` in hubs/pepkit/spokes.toml — the hub builds + # that ref, so notifying from any other branch would rebuild the same bytes. + branches: [master] + paths: + - 'docs/**' # the spoke's docs tree, docs.yml included + - 'peppy/**' # the package the hub renders API pages from + - '.github/workflows/notify-docs.yml' # so a notifier fix proves itself + workflow_dispatch: + +jobs: + notify: + uses: databio/actions/.github/workflows/notify-doc-hub.yml@master + with: + site: pepkit + spoke: peppy + secrets: + DOCS_DISPATCH_TOKEN: ${{ secrets.DOCS_DISPATCH_TOKEN }}