he Code (CDE) environment evaluates language models on code generation tasks
2.0K
The Code (CDE) environment evaluates language models on code generation tasks from the Intellect-3-RL dataset. Models are presented with programming problems and must generate correct Python code that passes all test cases.
import affinetes as af
import asyncio
import os
async def main():
# Load environment from Docker Hub
env = af.load_env(
image="affinefoundation/cde:pi",
env_vars={"CHUTES_API_KEY": os.getenv("CHUTES_API_KEY")}
)
# Evaluate on a code generation task
result = await env.evaluate(
model="deepseek-ai/DeepSeek-V3",
base_url="https://llm.chutes.ai/v1",
task_id=42, # Optional: index into dataset
)
print(f"Score: {result['score']}")
print(f"Success: {result['success']}")
await env.cleanup()
asyncio.run(main())
import affinetes as af
import asyncio
import os
async def main():
# Build image from local environment directory
image_tag = af.build_image_from_env(
env_path="environments/primeintellect/cde",
image_tag="cde:pi"
)
# Load environment
env = af.load_env(
image=image_tag,
env_vars={"CHUTES_API_KEY": os.getenv("CHUTES_API_KEY")}
)
# Evaluate on a code generation task
result = await env.evaluate(
model="deepseek-ai/DeepSeek-V3",
base_url="https://llm.chutes.ai/v1",
task_id=42
)
print(f"Score: {result['score']}")
await env.cleanup()
asyncio.run(main())
Content type
Image
Digest
sha256:2cc008cbd…
Size
1.9 GB
Last updated
8 months ago
docker pull affinefoundation/cde:print