Add basic memory challenge (#57)

This commit is contained in:
merwanehamadi
2023-07-05 20:32:28 -07:00
committed by GitHub
parent bfc7dfdb29
commit 74fc969dd6
24 changed files with 121 additions and 38 deletions

View File

@@ -0,0 +1 @@
Hello World!

View File

@@ -4,8 +4,8 @@
"task": "Write the string 'random string' before any existing text to the file called file_to_check.txt",
"dependencies": ["basic_write_file"],
"ground": {
"answer": "random string: this is how we're doing",
"should_contain": ["random string: this is how we're doing"],
"answer": "random string: Hello World!",
"should_contain": ["random string: Hello World!"],
"files": ["file_to_check.txt"]
},
"mock": {

View File

@@ -3,19 +3,12 @@ from typing import Any, Dict
import pytest
from agbenchmark.challenge import Challenge
from agbenchmark.tests.basic_abilities.basic_challenge import BasicChallenge
class TestReadFile(BasicChallenge):
"""Testing if LLM can read a file"""
@pytest.fixture(scope="module", autouse=True)
def setup_module(self, workspace: str) -> None:
Challenge.write_to_file(
workspace, self.data.ground.files[0], "this is how we're doing"
)
def get_file_path(self) -> str: # all tests must implement this method
return os.path.join(os.path.dirname(__file__), "r_file_data.json")

View File

@@ -1,5 +1,4 @@
import os
from pathlib import Path
from typing import Any, Dict
import pytest
@@ -17,8 +16,7 @@ class TestWriteFile(BasicChallenge):
def test_method(self, config: Dict[str, Any]) -> None:
self.setup_challenge(config)
workspace = Path(os.getcwd()) / config["workspace"]
files_contents = self.open_files(workspace, self.data.ground.files)
files_contents = self.open_files(config["workspace"], self.data.ground.files)
scores = []
for file_content in files_contents: