The Affine environment provides three types of reasoning tasks for evaluating language models
1.1K
The Affine environment provides three types of reasoning tasks for evaluating language models:
Each task type evaluates different aspects of logical reasoning and problem-solving capabilities.
import affinetes as af
import asyncio
import os
async def main():
# Load environment from Docker Hub
env = af.load_env(
image="affinefoundation/affine:v4",
env_vars={"CHUTES_API_KEY": os.getenv("CHUTES_API_KEY")}
)
# Evaluate on a SAT task
result = await env.evaluate(
task_type="sat",
model="deepseek-ai/DeepSeek-V3",
base_url="https://llm.chutes.ai/v1",
task_id=42 # Optional: for deterministic task generation
)
print(f"Score: {result['score']}")
print(f"Success: {result['success']}")
await env.cleanup()
asyncio.run(main())
import affinetes as af
import asyncio
import os
async def main():
# Build image from local environment directory
image_tag = af.build_image_from_env(
env_path="environments/affine",
image_tag="affine:latest"
)
# Load environment
env = af.load_env(
image=image_tag,
env_vars={"CHUTES_API_KEY": os.getenv("CHUTES_API_KEY")}
)
# Evaluate on different task types
# SAT task
result = await env.evaluate(
task_type="sat",
model="deepseek-ai/DeepSeek-V3",
base_url="https://llm.chutes.ai/v1",
task_id=42
)
# ABD task
result = await env.evaluate(
task_type="abd",
model="deepseek-ai/DeepSeek-V3",
task_id=100
)
# DED task
result = await env.evaluate(
task_type="ded",
model="deepseek-ai/DeepSeek-V3",
task_id=200
)
await env.cleanup()
asyncio.run(main())
Content type
Image
Digest
sha256:63a814ca6…
Size
1.9 GB
Last updated
9 months ago
docker pull affinefoundation/affine-env:v4