dokimos-dev/dokimos/create-experiment

Dataset dataset = Dataset.fromJson(Path.of("src/test/resources/datasets/my-dataset.json"));
 
Task task = example -> {
    String input = example.input();
    String output = callYourLLM(input);
    return Map.of("output", output);
};
 
List<Evaluator> evaluators = List.of(
        ExactMatchEvaluator.builder()
                .name("Exact Match")
                .threshold(1.0)
                .build()
);
 
ExperimentResult result = Experiment.builder()
        .name("My Evaluation")
        .description("Evaluating my LLM on QA tasks")
        .dataset(dataset)
        .task(task)
        .evaluators(evaluators)
        .build()
        .run();
 
System.out.println("Pass rate: " + result.passRate());

public class MyEvaluationTest {
 
    @ParameterizedTest(name = "[{index}] {0}")
    @DatasetSource("classpath:datasets/my-dataset.json")
    void testMyLLM(Example example) {
        String actualOutput = callYourLLM(example.input());
        EvalTestCase testCase = example.toTestCase(actualOutput);
 
        List<Evaluator> evaluators = List.of(
                ExactMatchEvaluator.builder().build()
        );
 
        assertEval(testCase, evaluators);
    }
}

Reporter reporter = DokimosServerReporter.builder()
        .baseUrl("http://localhost:8080")
        .build();
 
ExperimentResult result = Experiment.builder()
        .name("My Evaluation")
        .dataset(dataset)
        .task(task)
        .evaluators(evaluators)
        .reporter(reporter)
        .build()
        .run();