Quickstart

pip install zeroeval
export ZEROEVAL_API_KEY="sk_ze_..."

import zeroeval as ze
ze.init()  # reads ZEROEVAL_API_KEY

import zeroeval as ze

ds = ze.Dataset(
    "capital-cities-demo",
    data=[
        {"question": "Capital of France?", "answer": "Paris"},
        {"question": "Capital of Germany?", "answer": "Berlin"},
        {"question": "Capital of Spain?", "answer": "Madrid"},
    ],
    description="Simple geography eval set",
)
ds.push()

import zeroeval as ze

@ze.task(outputs=["prediction"])
def predict_capital(row):
    # Replace with your model/provider call.
    return {"prediction": row.answer}

@ze.evaluation(mode="row", outputs=["exact_match"])
def exact_match(row, answer_col, prediction_col):
    return {"exact_match": int(answer_col == prediction_col)}

@ze.evaluation(mode="column", outputs=["accuracy"])
def accuracy(exact_match_col):
    total = len(exact_match_col)
    return {"accuracy": (sum(exact_match_col) / total) if total else 0.0}

run = ds.eval(predict_capital, workers=8)
run = run.score(
    [exact_match, accuracy],
    column_map={
        "exact_match": {
            "answer_col": "answer",
            "prediction_col": "prediction",
        },
        "accuracy": {"exact_match_col": "exact_match"},
    },
)

print("eval_id:", run.id)
print("metrics:", run.metrics)
print("health:", run.health)