diff --git a/reports/beebot/folder13_08-01-08-13/report.json b/reports/beebot/folder13_08-01-08-13/report.json new file mode 100644 index 00000000..7f499d63 --- /dev/null +++ b/reports/beebot/folder13_08-01-08-13/report.json @@ -0,0 +1,13 @@ +{ + "command": "agbenchmark start", + "completion_time": "2023-08-01-08:14", + "benchmark_start_time": "2023-08-01-08:13", + "metrics": { + "run_time": "61.54 seconds", + "highest_difficulty": "No successful tests" + }, + "tests": {}, + "config": { + "workspace": "workspace" + } +} \ No newline at end of file diff --git a/reports/beebot/success_rate.json b/reports/beebot/success_rate.json index 2721335a..8ffae660 100644 --- a/reports/beebot/success_rate.json +++ b/reports/beebot/success_rate.json @@ -1,12 +1,54 @@ { + "TestAdaptLink": [ + false + ], + "TestAdaptSimpleTypoWithGuidance": [ + false + ], + "TestAdaptTeslaRevenue": [ + false + ], + "TestBasicContentGen": [ + false + ], + "TestBasicMemory": [ + true + ], "TestBasicRetrieval": [ false, false ], + "TestDebugMultipleTypo": [ + false + ], + "TestDebugSimpleTypoWithGuidance": [ + true + ], + "TestDebugSimpleTypoWithoutGuidance": [ + true + ], + "TestFunctionCodeGeneration": [ + false + ], "TestGoalDivergence": [ false, false ], + "TestGoalLoss_Hard": [ + false + ], + "TestGoalLoss_Medium": [ + false + ], + "TestGoalLoss_Simple": [ + false + ], + "TestGoalLoss_advanced": [ + false + ], + "TestInstructionFollowing": [ + false + ], "TestPlanCreation": [ true, true @@ -15,72 +57,10 @@ true, true ], - "TestSearch": [ - true, - true - ], - "TestWriteFile": [ - true, - true - ], - "TestBasicContentGen": [ - false - ], - "TestReturnCode_Simple": [ - true - ], - "TestDebugSimpleTypoWithGuidance": [ - true - ], - "TestBasicMemory": [ - true - ], - "TestAdaptLink": [ - false - ], - "TestRevenueRetrieval_1.2": [ - false - ], - "TestRevenueRetrieval_1.1": [ - false - ], - "TestRevenueRetrieval_1.0": [ - false - ], - "TestReturnCode_Write": [ - true - ], - "TestDebugSimpleTypoWithoutGuidance": [ - true - ], - "TestAdaptSimpleTypoWithGuidance": [ - false - ], - "TestInstructionFollowing": [ - false - ], - "TestGoalLoss_Simple": [ - false - ], - "TestGoalLoss_advanced": [ - false - ], - "TestGoalLoss_Hard": [ - false - ], - "TestGoalLoss_Medium": [ - false - ], "TestRememberMultipleIds": [ false ], - "TestFunctionCodeGeneration": [ - false - ], - "TestReturnCode_Modify": [ - false - ], - "TestDebugMultipleTypo": [ + "TestRememberMultiplePhrasesWithNoise": [ false ], "TestRememberMultipleWithNoise": [ @@ -89,16 +69,37 @@ "TestRetrieval3": [ false ], - "TestAdaptTeslaRevenue": [ + "TestReturnCode_Modify": [ false ], - "TestThreeSum": [ - false + "TestReturnCode_Simple": [ + true ], "TestReturnCode_Tests": [ false ], - "TestRememberMultiplePhrasesWithNoise": [ + "TestReturnCode_Write": [ + true + ], + "TestRevenueRetrieval_1.0": [ + false + ], + "TestRevenueRetrieval_1.1": [ + false + ], + "TestRevenueRetrieval_1.2": [ + false + ], + "TestSearch": [ + true, + true + ], + "TestThreeSum": [ + false + ], + "TestWriteFile": [ + true, + true, false ] } \ No newline at end of file