Skip to content

Data Pipeline

For class documentation, function signatures, and common pitfalls, see the Data Pipeline API Reference.

Quick Start

The simplest pipeline has one data source: a fetch function to get data, a clean function to process it, and file paths for caching.

dataset.py
from pathlib import Path
import polars as pl
from .dataset_utils import DataSource, DatasetConfig, DatasetManager
from .config import PROCESSED_DATA_DIR, RAW_DATA_DIR


def fetch_data() -> pl.DataFrame:
    """Fetch data from your source."""
    return pl.DataFrame({
        "date": pl.date_range(start="2024-01-01", end="2024-01-10", eager=True),
        "value": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
    })


def clean_data(df: pl.DataFrame) -> pl.DataFrame:
    """Clean and preprocess your data."""
    return df.filter(pl.col("value") > 0)


def write_dataframe(df: pl.DataFrame, file_path: Path) -> Path:
    """Write a DataFrame to parquet or CSV based on file extension."""
    file_path = Path(file_path)
    if file_path.suffix == ".csv":
        df.write_csv(file_path)
    elif file_path.suffix == ".parquet":
        df.write_parquet(file_path)
    else:
        raise ValueError(f"Unsupported file extension: {file_path.suffix}")
    return file_path


def read_dataframe(file_path: Path) -> pl.DataFrame:
    """Read a DataFrame from parquet or CSV based on file extension."""
    file_path = Path(file_path)
    if file_path.suffix == ".csv":
        return pl.read_csv(file_path)
    elif file_path.suffix == ".parquet":
        return pl.read_parquet(file_path)
    else:
        raise ValueError(f"Unsupported file extension: {file_path.suffix}")


# Define a single data source
source = DataSource(
    name="my_source",
    fetch_fn=fetch_data,
    raw_file=RAW_DATA_DIR / "raw.csv",
    processed_file=PROCESSED_DATA_DIR / "processed.csv",
    clean_fn=clean_data,
)

# Create pipeline configuration with shared I/O functions
config = DatasetConfig(
    sources=[source],
    read_fn=read_dataframe,
    write_fn=write_dataframe,
)

Use it in a notebook:

notebook.ipynb
from project.dataset import pipeline

# Load dataset -- single output source returns a DataFrame
df = pipeline.load_dataset()

# Force fresh data fetch
df = pipeline.load_dataset(fetch_override=True)

# Force reprocessing
df = pipeline.load_dataset(process_override=True)

# Check dataset status
pipeline.show_status()

Or from the command line:

Terminal
dataset load                   # Fetch and process dataset
dataset load --fetch-override  # Force fresh data fetch
dataset load --process-override  # Force reprocessing
dataset status                 # Check dataset status
dataset graph                  # Show pipeline dependency graph

Working with Multiple Sources

When your pipeline has multiple independent data sources, define each as its own DataSource. If all sources have output=True (the default), the pipeline returns a dict keyed by source name.

dataset.py
def fetch_sales() -> pl.DataFrame:
    """Fetch sales transactions."""
    return pl.DataFrame({
        "date": ["2024-01-01", "2024-01-02", "2024-01-03"],
        "revenue": [100.0, 150.0, 200.0],
    })


def fetch_inventory() -> pl.DataFrame:
    """Fetch current inventory levels."""
    return pl.DataFrame({
        "product": ["A", "B", "C"],
        "quantity": [50, 30, 80],
    })


sales = DataSource(
    name="sales",
    fetch_fn=fetch_sales,
    raw_file=RAW_DATA_DIR / "sales.parquet",
    processed_file=PROCESSED_DATA_DIR / "sales.parquet",
)

inventory = DataSource(
    name="inventory",
    fetch_fn=fetch_inventory,
    raw_file=RAW_DATA_DIR / "inventory.parquet",
    processed_file=PROCESSED_DATA_DIR / "inventory.parquet",
)

config = DatasetConfig(
    sources=[sales, inventory],
    read_fn=read_dataframe,
    write_fn=write_dataframe,
)
notebook.ipynb
from project.dataset import pipeline

# Multiple output sources return a dict keyed by source name
result = pipeline.load_dataset()
sales_df = result["sales"]
inventory_df = result["inventory"]

Derived Sources

A derived source computes its data from other sources using required_sources. The framework resolves dependencies automatically and passes them as a dict to fetch_fn.

dataset.py
def compute_daily_stats(sources: dict[str, pl.DataFrame]) -> pl.DataFrame:
    """Compute daily aggregate statistics from sales data."""
    sales_df = sources["sales"]
    return sales_df.group_by("date").agg(
        pl.col("revenue").sum().alias("total_revenue"),
        pl.col("revenue").mean().alias("avg_revenue"),
    )


sales = DataSource(
    name="sales",
    fetch_fn=fetch_sales,
    raw_file=RAW_DATA_DIR / "sales.parquet",
)

daily_stats = DataSource(
    name="daily_stats",
    fetch_fn=compute_daily_stats,  # receives {"sales": df}
    required_sources=["sales"],
    processed_file=PROCESSED_DATA_DIR / "daily_stats.parquet",
)

config = DatasetConfig(
    sources=[sales, daily_stats],
    read_fn=read_dataframe,
    write_fn=write_dataframe,
)

The framework processes sales first (because daily_stats depends on it), then passes the result into compute_daily_stats. Since both sources have output=True by default, the result is a dict with both DataFrames.

Merging Data

Merging is just a derived source that depends on multiple sources. There is no separate merge concept -- write the join logic in fetch_fn and list all dependencies in required_sources.

dataset.py
def fetch_sales() -> pl.DataFrame:
    """Fetch sales transactions."""
    return pl.DataFrame({
        "date": ["2024-01-01", "2024-01-02"],
        "product": ["A", "B"],
        "revenue": [100.0, 150.0],
    })


def fetch_inventory() -> pl.DataFrame:
    """Fetch current inventory levels."""
    return pl.DataFrame({
        "product": ["A", "B", "C"],
        "quantity": [50, 30, 80],
    })


def merge_sales_inventory(sources: dict[str, pl.DataFrame]) -> pl.DataFrame:
    """Join sales and inventory data on product."""
    return sources["sales"].join(sources["inventory"], on="product", how="left")


sales = DataSource(name="sales", fetch_fn=fetch_sales, output=False)
inventory = DataSource(name="inventory", fetch_fn=fetch_inventory, output=False)
combined = DataSource(
    name="combined",
    fetch_fn=merge_sales_inventory,  # receives {"sales": df, "inventory": df}
    required_sources=["sales", "inventory"],
    processed_file=PROCESSED_DATA_DIR / "combined.parquet",
)

config = DatasetConfig(
    sources=[sales, inventory, combined],
    read_fn=read_dataframe,
    write_fn=write_dataframe,
)
notebook.ipynb
from project.dataset import pipeline

# Single output source -- returns a DataFrame directly
df = pipeline.load_dataset()

Controlling Output

Use output=False to mark intermediate sources that should not appear in the final result. These sources are still processed so downstream derived sources can use them.

dataset.py
# Intermediate -- not included in final result
raw_events = DataSource(
    name="raw_events",
    fetch_fn=fetch_events,
    output=False,
)

# Intermediate -- not included in final result
raw_users = DataSource(
    name="raw_users",
    fetch_fn=fetch_users,
    output=False,
)

# Derived from raw_events -- included in final result
event_summary = DataSource(
    name="event_summary",
    fetch_fn=summarize_events,  # receives {"raw_events": df}
    required_sources=["raw_events"],
)

# Merges both -- included in final result
user_activity = DataSource(
    name="user_activity",
    fetch_fn=merge_user_events,  # receives {"raw_events": df, "raw_users": df}
    required_sources=["raw_events", "raw_users"],
)

config = DatasetConfig(
    sources=[raw_events, raw_users, event_summary, user_activity],
    read_fn=read_dataframe,
    write_fn=write_dataframe,
)
notebook.ipynb
from project.dataset import pipeline

# Multiple output sources -- returns a dict keyed by source name
result = pipeline.load_dataset()
# result is {"event_summary": <DataFrame>, "user_activity": <DataFrame>}
# raw_events and raw_users are not included

The return type depends on how many output sources you have:

  • One output source -- load_dataset() returns a single DataFrame.
  • Multiple output sources -- returns a dict[str, DataFrame] keyed by source name.
  • Sources with output=False are always excluded from the return value.

Cache your output sources

Output sources without processed_file are recomputed on every load_dataset() call. Set processed_file to cache results to disk and skip reprocessing on subsequent runs.

Function Reuse

Config-Level Functions

Define I/O functions once at the config level. They serve as fallbacks for any source that doesn't specify its own:

dataset.py
config = DatasetConfig(
    sources=[source1, source2],
    read_fn=read_dataframe,  # Used by all sources unless overridden
    write_fn=write_dataframe,  # Used by all sources unless overridden
)

Lambda Overrides

Create a flexible cleaning function and customize it per source with lambdas:

dataset.py
def clean_numeric_data(
    df: pl.DataFrame,
    value_col: str = "value",
    min_value: float = 0,
) -> pl.DataFrame:
    """Clean numeric data by removing values below threshold."""
    return df.filter(pl.col(value_col) > min_value)


# Use with default parameters
source1 = DataSource(
    name="default_data",
    fetch_fn=fetch_data1,
    clean_fn=clean_numeric_data,  # Uses defaults: value_col="value", min_value=0
)

# Override column name via lambda
source2 = DataSource(
    name="score_data",
    fetch_fn=fetch_data2,
    clean_fn=lambda df: clean_numeric_data(df, value_col="score"),
)

# Override both parameters
source3 = DataSource(
    name="rating_data",
    fetch_fn=fetch_data3,
    clean_fn=lambda df: clean_numeric_data(df, value_col="rating", min_value=1),
)

functools.partial

For cleaner syntax when you have many sources, use partial instead of lambdas:

dataset.py
from functools import partial

def clean_numeric_data(
    df: pl.DataFrame,
    value_col: str = "value",
    min_value: float = 0,
) -> pl.DataFrame:
    """Clean numeric data by removing values below threshold."""
    return df.filter(pl.col(value_col) > min_value)


# Create pre-configured cleaning functions
clean_scores = partial(clean_numeric_data, value_col="score")
clean_ratings = partial(clean_numeric_data, value_col="rating", min_value=1)

# Use in different sources
source1 = DataSource(
    name="score_data",
    fetch_fn=fetch_data2,
    clean_fn=clean_scores,
)

source2 = DataSource(
    name="rating_data",
    fetch_fn=fetch_data3,
    clean_fn=clean_ratings,
)

CLI Usage

The pipeline includes built-in CLI commands:

Terminal
# Fetch and process the full dataset
dataset load

# Force fresh data fetch (ignores raw_file caches)
dataset load --fetch-override

# Force reprocessing (ignores processed_file caches)
dataset load --process-override

# Show status of all sources (file existence, size, modification time)
dataset status

# Show pipeline dependency graph
dataset graph