Get rid of get file path by using the data.json convention to store the challenge information (#67)

Signed-off-by: Merwane Hamadi <merwanehamadi@gmail.com>
This commit is contained in:
merwanehamadi
2023-07-07 13:58:17 -07:00
committed by GitHub
parent 6ef32a9b1f
commit e61523e59e
24 changed files with 4 additions and 67 deletions

View File

@@ -0,0 +1,22 @@
{
"name": "debug_simple_typo_without_guidance",
"category": ["code"],
"task": "Make test.py run without errors.",
"dependencies": [],
"ground": {
"answer": "[0, 1] [2, 5] [0, 3]",
"should_contain": ["[0, 1]", "[2, 5]", "[0, 3]"],
"should_not_contain": [],
"files": ["test.py"],
"type": "execute_python_code"
},
"mock": {
"mock_func": null,
"mock_task": null
},
"info": {
"difficulty": "basic",
"description": "Tests ability for the agent to debug python code with a simple typo in it, using a very broad prompt without guidance",
"side_effects": ["tests if there is in fact an LLM attached"]
}
}