dokimos-dev/dokimos/evaluate-spring-ai

@SpringBootTest
class MyChatEvaluationTest {
 
    @Autowired
    private ChatClient.Builder chatClientBuilder;
 
    @Test
    void evaluateChatbot() {
        ChatClient chatClient = chatClientBuilder.build();
 
        Task task = example -> {
            String response = chatClient.prompt()
                    .user(example.input())
                    .call()
                    .content();
            return Map.of("output", response);
        };
 
        JudgeLM judge = SpringAiSupport.asJudge(chatClientBuilder);
 
        ExperimentResult result = Experiment.builder()
                .name("Chatbot Evaluation")
                .dataset(Dataset.fromJson(Path.of("src/test/resources/datasets/qa.json")))
                .task(task)
                .evaluator(LLMJudgeEvaluator.builder()
                        .name("answer-quality")
                        .judge(judge)
                        .criteria("Is the response helpful and accurate?")
                        .evaluationParams(List.of(
                                EvalTestCaseParam.INPUT,
                                EvalTestCaseParam.ACTUAL_OUTPUT,
                                EvalTestCaseParam.EXPECTED_OUTPUT))
                        .threshold(0.7)
                        .build())
                .build()
                .run();
    }
}

Task task = example -> {
    String input = example.input();
    ChatClient.ChatClientRequestSpec request = chatClient.prompt().user(input);
    request.advisors(new QuestionAnswerAdvisor(vectorStore));
 
    String response = request.call().content();
    List<Document> docs = vectorStore.similaritySearch(input);
    List<String> context = docs.stream().map(Document::getText).toList();
 
    return Map.of("output", response, "context", context);
};

EvaluationRequest request = new EvaluationRequest(userText, documents, responseContent);
EvalTestCase testCase = SpringAiSupport.toTestCase(request);
EvalResult result = evaluator.evaluate(testCase);
EvaluationResponse response = SpringAiSupport.toEvaluationResponse(result);

<dependency>
    <groupId>dev.dokimos</groupId>
    <artifactId>dokimos-spring-ai</artifactId>
    <version>${dokimos.version}</version>
</dependency>

AgentTrace trace = SpringAiSupport.toAgentTrace(assistantMessage, toolResponseMessages);
List<ToolDefinition> tools = SpringAiSupport.toToolDefinitions(toolDefinitions);
 
EvalTestCase testCase = trace.toTestCase(userMessage, tools);
var validity = ToolCallValidityEvaluator.builder().build().evaluate(testCase);