Pandera¶
Description:
- Validate dataframes against a schema: column types, nullability and value checks, with clear error reports when the data does not match.
Resources:
Example 1¶
Define a schema with DataFrameSchema, Column and Check, and validate a dataframe against it
In [ ]:
Copied!
import pandas as pd
import pandera.pandas as pa
df = pd.DataFrame(
{
"Name": [
"Braund, Mr. Owen Harris",
"Allen, Mr. William Henry",
"Bonnell, Miss. Elizabeth",
],
"Age": [22, 35, 58],
"Sex": ["male", "male", "female"],
}
)
schema = pa.DataFrameSchema(
{
"Name": pa.Column(str),
"Age": pa.Column(int, pa.Check.in_range(0, 120)),
"Sex": pa.Column(str, pa.Check.isin(["male", "female"])),
}
)
# validate() returns the dataframe unchanged when it matches the schema
schema.validate(df)
import pandas as pd
import pandera.pandas as pa
df = pd.DataFrame(
{
"Name": [
"Braund, Mr. Owen Harris",
"Allen, Mr. William Henry",
"Bonnell, Miss. Elizabeth",
],
"Age": [22, 35, 58],
"Sex": ["male", "male", "female"],
}
)
schema = pa.DataFrameSchema(
{
"Name": pa.Column(str),
"Age": pa.Column(int, pa.Check.in_range(0, 120)),
"Sex": pa.Column(str, pa.Check.isin(["male", "female"])),
}
)
# validate() returns the dataframe unchanged when it matches the schema
schema.validate(df)
Example 2¶
Download a remote csv, then validate it lazily so that every failing check is reported at once
In [ ]:
Copied!
import pandas as pd
import pandera.pandas as pa
from fsspec import filesystem
from pathlib import Path
tmp = "/tmp/ganymede_demo"
Path(tmp).mkdir(parents=True, exist_ok=True)
url = "https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv"
filesystem("http").get(url, f"{tmp}/titanic.csv")
titanic = pd.read_csv(f"{tmp}/titanic.csv")
schema = pa.DataFrameSchema(
{
"PassengerId": pa.Column(int, unique=True),
"Survived": pa.Column(int, pa.Check.isin([0, 1])),
"Pclass": pa.Column(int, pa.Check.isin([1, 2, 3])),
# Age has missing values, so this column must allow nulls
"Age": pa.Column(float, pa.Check.ge(0), nullable=True),
# Deliberately too strict: some fares are above 100, so this check fails
"Fare": pa.Column(float, pa.Check.le(100)),
}
)
try:
schema.validate(titanic, lazy=True)
except pa.errors.SchemaErrors as err:
print(err.failure_cases.head())
import pandas as pd
import pandera.pandas as pa
from fsspec import filesystem
from pathlib import Path
tmp = "/tmp/ganymede_demo"
Path(tmp).mkdir(parents=True, exist_ok=True)
url = "https://raw.githubusercontent.com/pandas-dev/pandas/main/doc/data/titanic.csv"
filesystem("http").get(url, f"{tmp}/titanic.csv")
titanic = pd.read_csv(f"{tmp}/titanic.csv")
schema = pa.DataFrameSchema(
{
"PassengerId": pa.Column(int, unique=True),
"Survived": pa.Column(int, pa.Check.isin([0, 1])),
"Pclass": pa.Column(int, pa.Check.isin([1, 2, 3])),
# Age has missing values, so this column must allow nulls
"Age": pa.Column(float, pa.Check.ge(0), nullable=True),
# Deliberately too strict: some fares are above 100, so this check fails
"Fare": pa.Column(float, pa.Check.le(100)),
}
)
try:
schema.validate(titanic, lazy=True)
except pa.errors.SchemaErrors as err:
print(err.failure_cases.head())
Example 3¶
Write the same kind of schema as a class with DataFrameModel, and use it to check a function's output
In [ ]:
Copied!
import pandas as pd
import pandera.pandas as pa
from pandera.typing import DataFrame, Series
class Passengers(pa.DataFrameModel):
Name: Series[str]
Age: Series[int] = pa.Field(ge=0, le=120)
Sex: Series[str] = pa.Field(isin=["male", "female"])
@pa.check_types
def adults(df: DataFrame[Passengers]) -> DataFrame[Passengers]:
return df[df["Age"] >= 18]
people = pd.DataFrame(
{
"Name": ["Ada", "Bo", "Cy"],
"Age": [36, 12, 58],
"Sex": ["female", "male", "male"],
}
)
adults(people)
import pandas as pd
import pandera.pandas as pa
from pandera.typing import DataFrame, Series
class Passengers(pa.DataFrameModel):
Name: Series[str]
Age: Series[int] = pa.Field(ge=0, le=120)
Sex: Series[str] = pa.Field(isin=["male", "female"])
@pa.check_types
def adults(df: DataFrame[Passengers]) -> DataFrame[Passengers]:
return df[df["Age"] >= 18]
people = pd.DataFrame(
{
"Name": ["Ada", "Bo", "Cy"],
"Age": [36, 12, 58],
"Sex": ["female", "male", "male"],
}
)
adults(people)