Running Evals

import zeroeval as ze

ze.init()
dataset = ze.Dataset.pull("capital-cities")

@ze.task(outputs=["prediction"])
def predict(row):
    return {"prediction": row.answer}

run = dataset.eval(predict, workers=8)
print(run.id)
print(run.health)

print(type(run).__name__)  # Eval
print(run.id)              # backend eval ID
print(run.rows[0])         # task outputs for the first row
print(run.metrics)         # aggregate metrics after scoring

@ze.task(outputs=["prediction"])
def predict(row):
    value = call_model(row.question)
    return {"prediction": value}

@ze.task(outputs=["prediction"])
def predict(row):
    ze.emit_signal("phase", "retrieve")
    docs = retrieve(row.question)
    ze.emit_signal("retrieved_doc_count", len(docs))
    return {"prediction": call_model(row.question, docs)}

run = dataset.eval(
    predict,
    execution=ze.ExecutionConfig(
        workers=12,
        timeout_s=30,
        retry=ze.RetryPolicy(max_attempts=3),
    ),
)

run = dataset.eval(
    predict,
    checkpoint=ze.CheckpointConfig(
        enabled=True,
        flush_every_rows=50,
        flush_every_seconds=10.0,
    ),
)

@ze.task(outputs=["prediction"])
def predict(row):
    ze.emit_signal("phase", "retrieve")
    docs = retrieve(row.question)
    ze.emit_signal("retrieved_doc_count", len(docs))
    ze.emit_signal("retrieval_hit", bool(docs))
    return {"prediction": call_model(row.question, docs)}

run = dataset.eval(
    predict,
    parameters={
        "model": "gpt-4o-mini",
        "experiment": "capital-baseline",
        "dataset_version": dataset.version_number,
    },
)

print(run.health["status"])
print(run.health["success_count"])
print(run.health["error_count"])
print(run.health["retry_success_rate"])