Data Pipeline¶
For class documentation, function signatures, and common pitfalls, see the Data Pipeline API Reference.
Quick Start¶
The simplest pipeline has one data source: a fetch function to get data, a clean function to process it, and file paths for caching.
from pathlib import Path
import polars as pl
from .dataset_utils import DataSource, DatasetConfig, DatasetManager
from .config import PROCESSED_DATA_DIR, RAW_DATA_DIR
def fetch_data() -> pl.DataFrame:
"""Fetch data from your source."""
return pl.DataFrame({
"date": pl.date_range(start="2024-01-01", end="2024-01-10", eager=True),
"value": [1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
})
def clean_data(df: pl.DataFrame) -> pl.DataFrame:
"""Clean and preprocess your data."""
return df.filter(pl.col("value") > 0)
def write_dataframe(df: pl.DataFrame, file_path: Path) -> Path:
"""Write a DataFrame to parquet or CSV based on file extension."""
file_path = Path(file_path)
if file_path.suffix == ".csv":
df.write_csv(file_path)
elif file_path.suffix == ".parquet":
df.write_parquet(file_path)
else:
raise ValueError(f"Unsupported file extension: {file_path.suffix}")
return file_path
def read_dataframe(file_path: Path) -> pl.DataFrame:
"""Read a DataFrame from parquet or CSV based on file extension."""
file_path = Path(file_path)
if file_path.suffix == ".csv":
return pl.read_csv(file_path)
elif file_path.suffix == ".parquet":
return pl.read_parquet(file_path)
else:
raise ValueError(f"Unsupported file extension: {file_path.suffix}")
# Define a single data source
source = DataSource(
name="my_source",
fetch_fn=fetch_data,
raw_file=RAW_DATA_DIR / "raw.csv",
processed_file=PROCESSED_DATA_DIR / "processed.csv",
clean_fn=clean_data,
)
# Create pipeline configuration with shared I/O functions
config = DatasetConfig(
sources=[source],
read_fn=read_dataframe,
write_fn=write_dataframe,
)
Use it in a notebook:
from project.dataset import pipeline
# Load dataset -- single output source returns a DataFrame
df = pipeline.load_dataset()
# Force fresh data fetch
df = pipeline.load_dataset(fetch_override=True)
# Force reprocessing
df = pipeline.load_dataset(process_override=True)
# Check dataset status
pipeline.show_status()
Or from the command line:
dataset load # Fetch and process dataset
dataset load --fetch-override # Force fresh data fetch
dataset load --process-override # Force reprocessing
dataset status # Check dataset status
dataset graph # Show pipeline dependency graph
Working with Multiple Sources¶
When your pipeline has multiple independent data sources, define each as its own DataSource. If all sources have output=True (the default), the pipeline returns a dict keyed by source name.
def fetch_sales() -> pl.DataFrame:
"""Fetch sales transactions."""
return pl.DataFrame({
"date": ["2024-01-01", "2024-01-02", "2024-01-03"],
"revenue": [100.0, 150.0, 200.0],
})
def fetch_inventory() -> pl.DataFrame:
"""Fetch current inventory levels."""
return pl.DataFrame({
"product": ["A", "B", "C"],
"quantity": [50, 30, 80],
})
sales = DataSource(
name="sales",
fetch_fn=fetch_sales,
raw_file=RAW_DATA_DIR / "sales.parquet",
processed_file=PROCESSED_DATA_DIR / "sales.parquet",
)
inventory = DataSource(
name="inventory",
fetch_fn=fetch_inventory,
raw_file=RAW_DATA_DIR / "inventory.parquet",
processed_file=PROCESSED_DATA_DIR / "inventory.parquet",
)
config = DatasetConfig(
sources=[sales, inventory],
read_fn=read_dataframe,
write_fn=write_dataframe,
)
from project.dataset import pipeline
# Multiple output sources return a dict keyed by source name
result = pipeline.load_dataset()
sales_df = result["sales"]
inventory_df = result["inventory"]
Derived Sources¶
A derived source computes its data from other sources using required_sources. The framework resolves dependencies automatically and passes them as a dict to fetch_fn.
def compute_daily_stats(sources: dict[str, pl.DataFrame]) -> pl.DataFrame:
"""Compute daily aggregate statistics from sales data."""
sales_df = sources["sales"]
return sales_df.group_by("date").agg(
pl.col("revenue").sum().alias("total_revenue"),
pl.col("revenue").mean().alias("avg_revenue"),
)
sales = DataSource(
name="sales",
fetch_fn=fetch_sales,
raw_file=RAW_DATA_DIR / "sales.parquet",
)
daily_stats = DataSource(
name="daily_stats",
fetch_fn=compute_daily_stats, # receives {"sales": df}
required_sources=["sales"],
processed_file=PROCESSED_DATA_DIR / "daily_stats.parquet",
)
config = DatasetConfig(
sources=[sales, daily_stats],
read_fn=read_dataframe,
write_fn=write_dataframe,
)
The framework processes sales first (because daily_stats depends on it), then passes the result into compute_daily_stats. Since both sources have output=True by default, the result is a dict with both DataFrames.
Merging Data¶
Merging is just a derived source that depends on multiple sources. There is no separate merge concept -- write the join logic in fetch_fn and list all dependencies in required_sources.
def fetch_sales() -> pl.DataFrame:
"""Fetch sales transactions."""
return pl.DataFrame({
"date": ["2024-01-01", "2024-01-02"],
"product": ["A", "B"],
"revenue": [100.0, 150.0],
})
def fetch_inventory() -> pl.DataFrame:
"""Fetch current inventory levels."""
return pl.DataFrame({
"product": ["A", "B", "C"],
"quantity": [50, 30, 80],
})
def merge_sales_inventory(sources: dict[str, pl.DataFrame]) -> pl.DataFrame:
"""Join sales and inventory data on product."""
return sources["sales"].join(sources["inventory"], on="product", how="left")
sales = DataSource(name="sales", fetch_fn=fetch_sales, output=False)
inventory = DataSource(name="inventory", fetch_fn=fetch_inventory, output=False)
combined = DataSource(
name="combined",
fetch_fn=merge_sales_inventory, # receives {"sales": df, "inventory": df}
required_sources=["sales", "inventory"],
processed_file=PROCESSED_DATA_DIR / "combined.parquet",
)
config = DatasetConfig(
sources=[sales, inventory, combined],
read_fn=read_dataframe,
write_fn=write_dataframe,
)
from project.dataset import pipeline
# Single output source -- returns a DataFrame directly
df = pipeline.load_dataset()
Controlling Output¶
Use output=False to mark intermediate sources that should not appear in the final result. These sources are still processed so downstream derived sources can use them.
# Intermediate -- not included in final result
raw_events = DataSource(
name="raw_events",
fetch_fn=fetch_events,
output=False,
)
# Intermediate -- not included in final result
raw_users = DataSource(
name="raw_users",
fetch_fn=fetch_users,
output=False,
)
# Derived from raw_events -- included in final result
event_summary = DataSource(
name="event_summary",
fetch_fn=summarize_events, # receives {"raw_events": df}
required_sources=["raw_events"],
)
# Merges both -- included in final result
user_activity = DataSource(
name="user_activity",
fetch_fn=merge_user_events, # receives {"raw_events": df, "raw_users": df}
required_sources=["raw_events", "raw_users"],
)
config = DatasetConfig(
sources=[raw_events, raw_users, event_summary, user_activity],
read_fn=read_dataframe,
write_fn=write_dataframe,
)
from project.dataset import pipeline
# Multiple output sources -- returns a dict keyed by source name
result = pipeline.load_dataset()
# result is {"event_summary": <DataFrame>, "user_activity": <DataFrame>}
# raw_events and raw_users are not included
The return type depends on how many output sources you have:
- One output source --
load_dataset()returns a single DataFrame. - Multiple output sources -- returns a
dict[str, DataFrame]keyed by source name. - Sources with
output=Falseare always excluded from the return value.
Cache your output sources
Output sources without processed_file are recomputed on every load_dataset() call. Set processed_file to cache results to disk and skip reprocessing on subsequent runs.
Function Reuse¶
Config-Level Functions¶
Define I/O functions once at the config level. They serve as fallbacks for any source that doesn't specify its own:
config = DatasetConfig(
sources=[source1, source2],
read_fn=read_dataframe, # Used by all sources unless overridden
write_fn=write_dataframe, # Used by all sources unless overridden
)
Lambda Overrides¶
Create a flexible cleaning function and customize it per source with lambdas:
def clean_numeric_data(
df: pl.DataFrame,
value_col: str = "value",
min_value: float = 0,
) -> pl.DataFrame:
"""Clean numeric data by removing values below threshold."""
return df.filter(pl.col(value_col) > min_value)
# Use with default parameters
source1 = DataSource(
name="default_data",
fetch_fn=fetch_data1,
clean_fn=clean_numeric_data, # Uses defaults: value_col="value", min_value=0
)
# Override column name via lambda
source2 = DataSource(
name="score_data",
fetch_fn=fetch_data2,
clean_fn=lambda df: clean_numeric_data(df, value_col="score"),
)
# Override both parameters
source3 = DataSource(
name="rating_data",
fetch_fn=fetch_data3,
clean_fn=lambda df: clean_numeric_data(df, value_col="rating", min_value=1),
)
functools.partial¶
For cleaner syntax when you have many sources, use partial instead of lambdas:
from functools import partial
def clean_numeric_data(
df: pl.DataFrame,
value_col: str = "value",
min_value: float = 0,
) -> pl.DataFrame:
"""Clean numeric data by removing values below threshold."""
return df.filter(pl.col(value_col) > min_value)
# Create pre-configured cleaning functions
clean_scores = partial(clean_numeric_data, value_col="score")
clean_ratings = partial(clean_numeric_data, value_col="rating", min_value=1)
# Use in different sources
source1 = DataSource(
name="score_data",
fetch_fn=fetch_data2,
clean_fn=clean_scores,
)
source2 = DataSource(
name="rating_data",
fetch_fn=fetch_data3,
clean_fn=clean_ratings,
)
CLI Usage¶
The pipeline includes built-in CLI commands:
# Fetch and process the full dataset
dataset load
# Force fresh data fetch (ignores raw_file caches)
dataset load --fetch-override
# Force reprocessing (ignores processed_file caches)
dataset load --process-override
# Show status of all sources (file existence, size, modification time)
dataset status
# Show pipeline dependency graph
dataset graph