diff --git a/.idea/runConfigurations/BotAdmin.xml b/.idea/runConfigurations/BotAdmin.xml
index ae2753d4ca..bad8861fa8 100644
--- a/.idea/runConfigurations/BotAdmin.xml
+++ b/.idea/runConfigurations/BotAdmin.xml
@@ -4,6 +4,7 @@
+
diff --git a/bot/admin/server/src/main/kotlin/BotAdminService.kt b/bot/admin/server/src/main/kotlin/BotAdminService.kt
index 8be5638722..36a1ba988c 100644
--- a/bot/admin/server/src/main/kotlin/BotAdminService.kt
+++ b/bot/admin/server/src/main/kotlin/BotAdminService.kt
@@ -53,6 +53,7 @@ import ai.tock.bot.admin.dialog.DialogReportQueryResult
import ai.tock.bot.admin.dialog.DialogStatsQuery
import ai.tock.bot.admin.dialog.DialogStatsQueryResult
import ai.tock.bot.admin.dialog.RatingReportQueryResult
+import ai.tock.bot.admin.evaluation.EvaluationSampleDAO
import ai.tock.bot.admin.indicators.IndicatorDAO
import ai.tock.bot.admin.indicators.metric.MetricDAO
import ai.tock.bot.admin.kotlin.compiler.KotlinFile
@@ -156,6 +157,7 @@ object BotAdminService {
private val i18n: I18nDAO by injector.instance()
private val indicatorDAO: IndicatorDAO by injector.instance()
private val metricDAO: MetricDAO get() = injector.provide()
+ private val evaluationSampleDAO: EvaluationSampleDAO get() = injector.provide()
private class BotStoryDefinitionConfigurationDumpController(
override val targetNamespace: String,
@@ -1656,6 +1658,9 @@ object BotAdminService {
// delete Indicators and Metrics
indicatorDAO.deleteByApplicationName(app.namespace, app.name)
metricDAO.deleteByApplicationName(app.namespace, app.name)
+
+ // delete evaluation samples and their evaluations
+ evaluationSampleDAO.deleteByNamespaceAndBotId(app.namespace, app.name)
}
fun changeSupportedLocales(newApp: ApplicationDefinition) {
diff --git a/bot/admin/server/src/main/kotlin/BotAdminVerticle.kt b/bot/admin/server/src/main/kotlin/BotAdminVerticle.kt
index e4b4973848..059482c416 100644
--- a/bot/admin/server/src/main/kotlin/BotAdminVerticle.kt
+++ b/bot/admin/server/src/main/kotlin/BotAdminVerticle.kt
@@ -49,6 +49,7 @@ import ai.tock.bot.admin.story.dump.StoryDefinitionConfigurationDumpImport
import ai.tock.bot.admin.test.TestPlanService
import ai.tock.bot.admin.test.findTestService
import ai.tock.bot.admin.verticle.DialogVerticle
+import ai.tock.bot.admin.verticle.EvaluationVerticle
import ai.tock.bot.admin.verticle.GenAIVerticle
import ai.tock.bot.admin.verticle.IndicatorVerticle
import ai.tock.bot.connector.ConnectorType.Companion.rest
@@ -100,6 +101,7 @@ open class BotAdminVerticle : AdminVerticle() {
private val indicatorVerticle = IndicatorVerticle()
private val dialogVerticle = DialogVerticle()
private val aiVerticle = GenAIVerticle()
+ private val evaluationVerticle = EvaluationVerticle()
override val logger: KLogger = KotlinLogging.logger {}
@@ -162,6 +164,7 @@ open class BotAdminVerticle : AdminVerticle() {
indicatorVerticle.configure(this)
dialogVerticle.configure(this)
aiVerticle.configure(this)
+ evaluationVerticle.configure(this)
blockingJsonPost("/users/search", botUser) { context, query: UserSearchQuery ->
if (context.organization == query.namespace) {
diff --git a/bot/admin/server/src/main/kotlin/model/DialogsSearchQuery.kt b/bot/admin/server/src/main/kotlin/model/DialogsSearchQuery.kt
index 85ad27a47d..f72e1246fd 100644
--- a/bot/admin/server/src/main/kotlin/model/DialogsSearchQuery.kt
+++ b/bot/admin/server/src/main/kotlin/model/DialogsSearchQuery.kt
@@ -55,16 +55,17 @@ data class DialogsSearchQuery(
) : PaginatedQuery() {
fun toDialogReportQuery(): DialogReportQuery {
return DialogReportQuery(
- namespace,
- applicationName,
- language,
- start,
- size,
- playerId,
- text,
- dialogId,
- intentName,
- exactMatch,
+ namespace = namespace,
+ nlpModel = applicationName,
+ language = language,
+ start = start,
+ size = size,
+ playerId = playerId,
+ text = text,
+ dialogId = dialogId,
+ dialogIds = null,
+ intentName = intentName,
+ exactMatch = exactMatch,
connectorType = connectorType,
displayTests = displayTests,
obfuscated = true,
diff --git a/bot/admin/server/src/main/kotlin/model/evaluation/EvaluationRequests.kt b/bot/admin/server/src/main/kotlin/model/evaluation/EvaluationRequests.kt
new file mode 100644
index 0000000000..ba6ef2d942
--- /dev/null
+++ b/bot/admin/server/src/main/kotlin/model/evaluation/EvaluationRequests.kt
@@ -0,0 +1,169 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.model.evaluation
+
+import ai.tock.bot.admin.dialog.DialogReport
+import ai.tock.bot.admin.evaluation.ActionRef
+import ai.tock.bot.admin.evaluation.Evaluation
+import ai.tock.bot.admin.evaluation.EvaluationReason
+import ai.tock.bot.admin.evaluation.EvaluationSample
+import ai.tock.bot.admin.evaluation.EvaluationSampleStatus
+import ai.tock.bot.admin.evaluation.EvaluationStatus
+import ai.tock.bot.admin.evaluation.EvaluationsResult
+import java.time.Instant
+
+/**
+ * Request to create a new evaluation sample.
+ */
+data class CreateEvaluationSampleRequest(
+ val name: String?,
+ val description: String?,
+ val dialogActivityFrom: Instant,
+ val dialogActivityTo: Instant,
+ val requestedDialogCount: Int,
+ val allowTestDialogs: Boolean = false,
+)
+
+/**
+ * Query for paginated evaluation dialogs.
+ * Pagination is by dialog (dialogs are sorted by dialog id, ascending).
+ * POST body follows Tock convention for search endpoints.
+ *
+ * @param start Offset (index of first dialog), default 0
+ * @param size Number of dialogs per page, default 20
+ */
+data class EvaluationDialogsQuery(
+ val start: Int = 0,
+ val size: Int = 10,
+)
+
+/**
+ * Response for paginated evaluation dialogs.
+ * Contains action refs with evaluations for the requested page of dialogs,
+ * plus the corresponding dialog reports.
+ *
+ * @param start Requested offset
+ * @param size Number of action refs returned (one per evaluable action in the page)
+ * @param total Total number of dialogs in the sample
+ * @param actionRefs Action refs with their evaluations for the current page
+ * @param dialogs Found dialog reports and missing action refs
+ */
+data class EvaluationDialogsResponse(
+ val start: Int,
+ val size: Int,
+ /**
+ * exclusive. start + dialogs.size()
+ */
+ val end: Int,
+ val total: Long,
+ val actionRefs: List,
+ val dialogs: List,
+)
+
+/**
+ * An action ref with its associated evaluation.
+ */
+data class ActionRefWithEvaluation(
+ val dialogId: String,
+ val actionId: String,
+ val evaluation: Evaluation,
+)
+
+data class DialogEntry(
+ val dialogId: String,
+ val dialog: DialogReport?,
+)
+
+/**
+ * Result containing found and missing dialogs.
+ */
+data class DialogsResult(
+ val found: List,
+ val missing: List,
+)
+
+/**
+ * Request to evaluate an action (PATCH).
+ */
+data class EvaluateRequest(
+ val status: EvaluationStatus,
+ val reason: EvaluationReason? = null,
+)
+
+/**
+ * Request to change the status of an evaluation sample.
+ */
+data class ChangeStatusRequest(
+ val targetStatus: EvaluationSampleStatus,
+ val comment: String? = null,
+)
+
+/**
+ * DTO for evaluation sample with computed evaluationsResult.
+ */
+data class EvaluationSampleDTO(
+ val _id: String,
+ val botId: String,
+ val namespace: String,
+ val name: String?,
+ val description: String?,
+ val dialogActivityFrom: Instant,
+ val dialogActivityTo: Instant,
+ val requestedDialogCount: Int,
+ val dialogsCount: Int,
+ val totalDialogCount: Int,
+ val botActionCount: Int,
+ val allowTestDialogs: Boolean,
+ val status: EvaluationSampleStatus,
+ val createdBy: String,
+ val creationDate: Instant,
+ val statusChangedBy: String,
+ val statusChangeDate: Instant,
+ val statusComment: String?,
+ val lastUpdateDate: Instant,
+ val evaluationsResult: EvaluationsResult,
+) {
+ companion object {
+ fun from(
+ sample: EvaluationSample,
+ evaluationsResult: EvaluationsResult,
+ ): EvaluationSampleDTO {
+ return EvaluationSampleDTO(
+ _id = sample._id.toString(),
+ botId = sample.botId,
+ namespace = sample.namespace,
+ name = sample.name,
+ description = sample.description,
+ dialogActivityFrom = sample.dialogActivityFrom,
+ dialogActivityTo = sample.dialogActivityTo,
+ requestedDialogCount = sample.requestedDialogCount,
+ dialogsCount = sample.dialogsCount,
+ totalDialogCount = sample.totalDialogCount,
+ botActionCount = sample.botActionCount,
+ allowTestDialogs = sample.allowTestDialogs,
+ status = sample.status,
+ createdBy = sample.createdBy,
+ creationDate = sample.creationDate,
+ statusChangedBy = sample.statusChangedBy,
+ statusChangeDate = sample.statusChangeDate,
+ statusComment = sample.statusComment,
+ lastUpdateDate = sample.lastUpdateDate,
+ evaluationsResult = evaluationsResult,
+ )
+ }
+ }
+}
diff --git a/bot/admin/server/src/main/kotlin/service/EvaluationService.kt b/bot/admin/server/src/main/kotlin/service/EvaluationService.kt
new file mode 100644
index 0000000000..ccabec4a4c
--- /dev/null
+++ b/bot/admin/server/src/main/kotlin/service/EvaluationService.kt
@@ -0,0 +1,321 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.service
+
+import ai.tock.bot.admin.dialog.DialogReportDAO
+import ai.tock.bot.admin.dialog.DialogReportQuery
+import ai.tock.bot.admin.evaluation.ActionRef
+import ai.tock.bot.admin.evaluation.Evaluation
+import ai.tock.bot.admin.evaluation.EvaluationDAO
+import ai.tock.bot.admin.evaluation.EvaluationReason
+import ai.tock.bot.admin.evaluation.EvaluationSample
+import ai.tock.bot.admin.evaluation.EvaluationSampleDAO
+import ai.tock.bot.admin.evaluation.EvaluationSampleStatus
+import ai.tock.bot.admin.evaluation.EvaluationStatus
+import ai.tock.bot.admin.evaluation.EvaluationsResult
+import ai.tock.bot.admin.evaluation.Evaluator
+import ai.tock.bot.admin.model.evaluation.ActionRefWithEvaluation
+import ai.tock.bot.admin.model.evaluation.CreateEvaluationSampleRequest
+import ai.tock.bot.admin.model.evaluation.DialogEntry
+import ai.tock.bot.admin.model.evaluation.EvaluationDialogsResponse
+import ai.tock.bot.admin.model.evaluation.EvaluationSampleDTO
+import ai.tock.shared.exception.rest.BadRequestException
+import ai.tock.shared.exception.rest.UnprocessableEntityException
+import ai.tock.shared.injector
+import ai.tock.shared.provide
+import mu.KotlinLogging
+import org.litote.kmongo.toId
+import java.time.Instant
+import java.time.ZoneOffset
+
+private val logger = KotlinLogging.logger {}
+
+object EvaluationService {
+ private val evaluationSampleDAO: EvaluationSampleDAO get() = injector.provide()
+ private val evaluationDAO: EvaluationDAO get() = injector.provide()
+ private val dialogReportDAO: DialogReportDAO get() = injector.provide()
+
+ fun findSamplesByBotId(
+ namespace: String,
+ botId: String,
+ status: EvaluationSampleStatus? = null,
+ ): List {
+ val samples = evaluationSampleDAO.findByNamespaceAndBotId(namespace, botId, status)
+ return samples.map { sample ->
+ val statusCounts = evaluationDAO.countByStatus(sample._id)
+ val evaluationsResult = EvaluationsResult.fromStatusCounts(statusCounts)
+ EvaluationSampleDTO.from(sample, evaluationsResult)
+ }
+ }
+
+ fun findSampleById(sampleId: String): EvaluationSampleDTO? {
+ val sample = evaluationSampleDAO.findById(sampleId.toId()) ?: return null
+ val statusCounts = evaluationDAO.countByStatus(sample._id)
+ val evaluationsResult = EvaluationsResult.fromStatusCounts(statusCounts)
+ return EvaluationSampleDTO.from(sample, evaluationsResult)
+ }
+
+ fun createEvaluationSample(
+ namespace: String,
+ botId: String,
+ request: CreateEvaluationSampleRequest,
+ createdBy: String,
+ ): EvaluationSampleDTO {
+ logger.info { "Creating evaluation sample for bot $botId in namespace $namespace" }
+
+ val dialogQuery =
+ DialogReportQuery(
+ namespace = namespace,
+ nlpModel = botId,
+ dialogActivityFrom = request.dialogActivityFrom.atZone(ZoneOffset.UTC),
+ dialogActivityTo = request.dialogActivityTo.atZone(ZoneOffset.UTC),
+ displayTests = request.allowTestDialogs,
+ withAnnotations = false,
+ start = 0L,
+ size = request.requestedDialogCount,
+ random = true,
+ evaluableActionsOnly = true,
+ )
+
+ val dialogResult = dialogReportDAO.search(dialogQuery)
+ val totalDialogCount = dialogResult.total.toInt()
+
+ val actionRefs =
+ dialogResult.dialogs.flatMap { dialog ->
+ dialog.actions.map { action ->
+ ActionRef(
+ dialogId = dialog.id,
+ actionId = action.id,
+ )
+ }
+ }
+
+ if (actionRefs.isEmpty()) {
+ throw UnprocessableEntityException(
+ errorCode = 4221,
+ message = "No dialog matching query",
+ )
+ }
+
+ val selectedDialogs = actionRefs.map { it.dialogId }.distinct()
+ val now = Instant.now()
+ val sample =
+ EvaluationSample(
+ botId = botId,
+ namespace = namespace,
+ name = request.name,
+ description = request.description,
+ dialogActivityFrom = request.dialogActivityFrom,
+ dialogActivityTo = request.dialogActivityTo,
+ requestedDialogCount = request.requestedDialogCount,
+ dialogsCount = selectedDialogs.size,
+ totalDialogCount = totalDialogCount,
+ botActionCount = actionRefs.size,
+ allowTestDialogs = request.allowTestDialogs,
+ actionRefs = actionRefs,
+ status = EvaluationSampleStatus.IN_PROGRESS,
+ createdBy = createdBy,
+ creationDate = now,
+ statusChangedBy = createdBy,
+ statusChangeDate = now,
+ lastUpdateDate = now,
+ )
+
+ val savedSample = evaluationSampleDAO.save(sample)
+
+ val evaluations =
+ actionRefs.map { ref ->
+ Evaluation(
+ evaluationSampleId = savedSample._id,
+ dialogId = ref.dialogId,
+ actionId = ref.actionId,
+ status = EvaluationStatus.UNSET,
+ creationDate = now,
+ lastUpdateDate = now,
+ )
+ }
+ evaluationDAO.createAll(evaluations)
+
+ logger.info { "Created evaluation sample ${savedSample._id} with ${actionRefs.size} bot actions from ${selectedDialogs.size} dialogs" }
+
+ val evaluationsResult =
+ EvaluationsResult(
+ total = actionRefs.size,
+ evaluated = 0,
+ remaining = actionRefs.size,
+ positiveCount = 0,
+ negativeCount = 0,
+ )
+
+ return EvaluationSampleDTO.from(savedSample, evaluationsResult)
+ }
+
+ /**
+ * Returns paginated evaluation dialogs for a sample.
+ * Dialogs are sorted by dialog id (ascending). Each page contains action refs with their evaluations.
+ */
+ fun getEvaluationDialogs(
+ sampleId: String,
+ start: Int,
+ size: Int,
+ ): EvaluationDialogsResponse {
+ val sample =
+ evaluationSampleDAO.findById(sampleId.toId())
+ ?: run {
+ logger.error { "Evaluation sample not found for getEvaluationDialogs: sampleId=$sampleId" }
+ throw BadRequestException("Evaluation sample not found: $sampleId")
+ }
+ val groupedEvaluations =
+ evaluationDAO.findGroupedEvaluationsBySampleId(
+ sampleId = sample._id,
+ start = start,
+ size = size,
+ )
+
+ val pageDialogIds = groupedEvaluations.map { it._id }
+ val foundDialogs = dialogReportDAO.findByDialogByIds(pageDialogIds.toSet())
+ val dialogsById = foundDialogs.associateBy { it.id }
+ val evaluations = groupedEvaluations.flatMap { it.evaluations }
+ val actionRefsWithEvaluation =
+ evaluations.map { eval ->
+ ActionRefWithEvaluation(
+ dialogId = eval.dialogId.toString(),
+ actionId = eval.actionId.toString(),
+ evaluation = eval,
+ )
+ }
+
+ val dialogsOrdered =
+ pageDialogIds.map { dialogId ->
+ DialogEntry(
+ dialogId = dialogId.toString(),
+ dialog = dialogsById[dialogId],
+ )
+ }
+
+ return EvaluationDialogsResponse(
+ start = start,
+ size = actionRefsWithEvaluation.size,
+ end = start + dialogsOrdered.size,
+ total = sample.dialogsCount.toLong(),
+ actionRefs = actionRefsWithEvaluation,
+ dialogs = dialogsOrdered,
+ )
+ }
+
+ fun evaluate(
+ sampleId: String,
+ evaluationId: String,
+ status: EvaluationStatus,
+ reason: EvaluationReason?,
+ evaluatorId: String,
+ ): Evaluation? {
+ val sample =
+ evaluationSampleDAO.findById(sampleId.toId())
+ ?: run {
+ logger.error { "Evaluation sample not found: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Evaluation sample not found: $sampleId")
+ }
+
+ if (sample.status != EvaluationSampleStatus.IN_PROGRESS) {
+ logger.error { "Cannot evaluate: sample status is ${sample.status}, sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Cannot evaluate: sample status is ${sample.status}")
+ }
+
+ val evaluation =
+ evaluationDAO.findById(evaluationId.toId())
+ ?: run {
+ logger.error { "Evaluation not found: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Evaluation not found: $evaluationId")
+ }
+
+ if (evaluation.evaluationSampleId != sample._id) {
+ logger.error { "Evaluation does not belong to sample: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Evaluation $evaluationId does not belong to sample $sampleId")
+ }
+
+ if (status == EvaluationStatus.UNSET) {
+ logger.error { "Cannot set evaluation status to UNSET: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Cannot set evaluation status to UNSET")
+ }
+
+ if (status == EvaluationStatus.UP && reason != null) {
+ logger.error { "Reason must be null for UP evaluation: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Reason must be null for UP evaluation")
+ }
+
+ if (status == EvaluationStatus.DOWN && reason == null) {
+ logger.error { "Reason is required for DOWN evaluation: sampleId=$sampleId, evaluationId=$evaluationId" }
+ throw BadRequestException("Reason is required for DOWN evaluation")
+ }
+
+ val now = Instant.now()
+ val updated =
+ evaluation.copy(
+ status = status,
+ reason = reason,
+ evaluator = Evaluator(evaluatorId),
+ evaluationDate = now,
+ lastUpdateDate = now,
+ )
+
+ return evaluationDAO.update(updated)
+ }
+
+ fun changeStatus(
+ sampleId: String,
+ targetStatus: EvaluationSampleStatus,
+ changedBy: String,
+ comment: String?,
+ ): EvaluationSampleDTO? {
+ val sample =
+ evaluationSampleDAO.findById(sampleId.toId())
+ ?: run {
+ logger.error { "Evaluation sample not found for changeStatus: sampleId=$sampleId" }
+ throw BadRequestException("Evaluation sample not found: $sampleId")
+ }
+
+ if (sample.status != EvaluationSampleStatus.IN_PROGRESS) {
+ logger.error { "Cannot change status: sample is already ${sample.status}, sampleId=$sampleId" }
+ throw BadRequestException("Cannot change status: sample is already ${sample.status}")
+ }
+
+ if (targetStatus == EvaluationSampleStatus.IN_PROGRESS) {
+ logger.error { "Cannot change status to IN_PROGRESS: sampleId=$sampleId" }
+ throw BadRequestException("Cannot change status to IN_PROGRESS")
+ }
+
+ val updatedSample =
+ evaluationSampleDAO.updateStatus(
+ id = sample._id,
+ status = targetStatus,
+ changedBy = changedBy,
+ comment = comment,
+ ) ?: return null
+
+ val statusCounts = evaluationDAO.countByStatus(updatedSample._id)
+ val evaluationsResult = EvaluationsResult.fromStatusCounts(statusCounts)
+
+ return EvaluationSampleDTO.from(updatedSample, evaluationsResult)
+ }
+
+ fun deleteSample(sampleId: String): Boolean {
+ val id = sampleId.toId()
+ evaluationDAO.deleteByEvaluationSampleId(id)
+ return evaluationSampleDAO.delete(id)
+ }
+}
diff --git a/bot/admin/server/src/main/kotlin/verticle/AbstractNamespaceRetriever.kt b/bot/admin/server/src/main/kotlin/verticle/AbstractNamespaceRetriever.kt
new file mode 100644
index 0000000000..2c898818d6
--- /dev/null
+++ b/bot/admin/server/src/main/kotlin/verticle/AbstractNamespaceRetriever.kt
@@ -0,0 +1,41 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.verticle
+
+import ai.tock.nlp.front.client.FrontClient
+import ai.tock.nlp.front.shared.config.ApplicationDefinition
+import ai.tock.shared.exception.rest.NotFoundException
+import ai.tock.shared.security.TockUser
+import io.vertx.ext.web.RoutingContext
+
+abstract class AbstractNamespaceRetriever {
+ val front = FrontClient
+
+ /**
+ * Get the namespace from the context
+ * @param context : the vertx routing context
+ */
+ fun getNamespace(context: RoutingContext): String? = ((context.user() ?: context.session()?.get("tockUser")) as? TockUser)?.namespace
+
+ fun currentContextApp(context: RoutingContext): ApplicationDefinition? {
+ val botId = context.pathParam("botId")
+
+ return getNamespace(context)?.let { namespace ->
+ front.getApplicationByNamespaceAndName(namespace, botId)
+ } ?: throw NotFoundException(404, "Could not find $botId in namespace")
+ }
+}
diff --git a/bot/admin/server/src/main/kotlin/verticle/EvaluationVerticle.kt b/bot/admin/server/src/main/kotlin/verticle/EvaluationVerticle.kt
new file mode 100644
index 0000000000..3958067b56
--- /dev/null
+++ b/bot/admin/server/src/main/kotlin/verticle/EvaluationVerticle.kt
@@ -0,0 +1,117 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.verticle
+
+import ai.tock.bot.admin.model.evaluation.ChangeStatusRequest
+import ai.tock.bot.admin.model.evaluation.CreateEvaluationSampleRequest
+import ai.tock.bot.admin.model.evaluation.EvaluateRequest
+import ai.tock.bot.admin.model.evaluation.EvaluationDialogsQuery
+import ai.tock.bot.admin.service.EvaluationService
+import ai.tock.shared.security.TockUserRole.botUser
+import ai.tock.shared.vertx.WebVerticle
+import io.vertx.ext.web.RoutingContext
+
+/**
+ * Verticle handling evaluation sample endpoints.
+ */
+class EvaluationVerticle : AbstractNamespaceRetriever() {
+ companion object {
+ private const val PATH_SAMPLES = "/bots/:botId/evaluation-samples"
+ private const val PATH_SAMPLE = "/bots/:botId/evaluation-samples/:sampleId"
+ private const val PATH_ACTION_REFS = "/bots/:botId/evaluation-samples/:sampleId/action-refs"
+ private const val PATH_EVALUATION = "/bots/:botId/evaluation-samples/:sampleId/evaluations/:evaluationId"
+ private const val PATH_CHANGE_STATUS = "/bots/:botId/evaluation-samples/:sampleId/change-status"
+ }
+
+ fun configure(verticle: WebVerticle) {
+ with(verticle) {
+ // GET /bots/:botId/evaluation-samples - List all evaluation samples for a bot
+ blockingJsonGet(PATH_SAMPLES, botUser) { context ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val botId = context.pathParam("botId")
+ val namespace = context.organization
+ val statusParam = context.queryParam("status").firstOrNull()
+ val status =
+ statusParam?.let {
+ ai.tock.bot.admin.evaluation.EvaluationSampleStatus.valueOf(it)
+ }
+ EvaluationService.findSamplesByBotId(namespace, botId, status)
+ }
+ }
+
+ // POST /bots/:botId/evaluation-samples - Create a new evaluation sample (returns 201 Created)
+ blockingJsonPost(PATH_SAMPLES, botUser) { context, request: CreateEvaluationSampleRequest ->
+ val result =
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val botId = context.pathParam("botId")
+ val namespace = context.organization
+ val createdBy = context.userLogin
+ EvaluationService.createEvaluationSample(namespace, botId, request, createdBy)
+ }
+ context.response().statusCode = 201
+ result
+ }
+
+ // GET /bots/:botId/evaluation-samples/:sampleId - Get a specific evaluation sample
+ blockingJsonGet(PATH_SAMPLE, botUser) { context ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val sampleId = context.pathParam("sampleId")
+ EvaluationService.findSampleById(sampleId)
+ }
+ }
+
+ // DELETE /bots/:botId/evaluation-samples/:sampleId - Delete an evaluation sample (returns 204 No Content)
+ blockingDeleteEmptyResponse(PATH_SAMPLE, setOf(botUser)) { context ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val sampleId = context.pathParam("sampleId")
+ EvaluationService.deleteSample(sampleId)
+ }
+ }
+
+ // POST /bots/:botId/evaluation-samples/:sampleId/action-refs - Get paginated evaluation dialogs (POST with body)
+ // Returns dialogs sorted by creation date, with their action refs and evaluations
+ blockingJsonPost(PATH_ACTION_REFS, botUser) { context, query: EvaluationDialogsQuery ->
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val sampleId = context.pathParam("sampleId")
+ EvaluationService.getEvaluationDialogs(sampleId, query.start, query.size)
+ }
+ }
+
+ // PUT /bots/:botId/evaluation-samples/:sampleId/evaluations/:evaluationId - Evaluate an action
+ blockingJsonPut(
+ PATH_EVALUATION,
+ setOf(botUser),
+ ) { context: RoutingContext, request: EvaluateRequest ->
+ return@blockingJsonPut checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val sampleId = context.pathParam("sampleId")
+ val evaluationId = context.pathParam("evaluationId")
+ val evaluatorId = context.userLogin
+ EvaluationService.evaluate(sampleId, evaluationId, request.status, request.reason, evaluatorId)
+ }
+ }
+
+ // POST /bots/:botId/evaluation-samples/:sampleId/change-status - Change sample status
+ blockingJsonPost(PATH_CHANGE_STATUS, botUser) { context, request: ChangeStatusRequest ->
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) { app ->
+ val sampleId = context.pathParam("sampleId")
+ val changedBy = context.userLogin
+ EvaluationService.changeStatus(sampleId, request.targetStatus, changedBy, request.comment)
+ }
+ }
+ }
+ }
+}
diff --git a/bot/admin/server/src/main/kotlin/verticle/GenAIVerticle.kt b/bot/admin/server/src/main/kotlin/verticle/GenAIVerticle.kt
index b51fbb9269..ccf29d0898 100644
--- a/bot/admin/server/src/main/kotlin/verticle/GenAIVerticle.kt
+++ b/bot/admin/server/src/main/kotlin/verticle/GenAIVerticle.kt
@@ -31,10 +31,6 @@ import ai.tock.bot.admin.service.ObservabilityService
import ai.tock.bot.admin.service.RAGService
import ai.tock.bot.admin.service.SentenceGenerationService
import ai.tock.bot.admin.service.VectorStoreService
-import ai.tock.nlp.front.client.FrontClient
-import ai.tock.nlp.front.shared.config.ApplicationDefinition
-import ai.tock.shared.exception.rest.NotFoundException
-import ai.tock.shared.security.TockUser
import ai.tock.shared.security.TockUserRole.admin
import ai.tock.shared.security.TockUserRole.botUser
import ai.tock.shared.security.TockUserRole.nlpUser
@@ -44,7 +40,7 @@ import io.vertx.ext.web.RoutingContext
/**
* [GenAIVerticle] contains all the routes and actions associated with the AI tasks
*/
-class GenAIVerticle {
+class GenAIVerticle : AbstractNamespaceRetriever() {
companion object {
// Configuration
private const val PATH_CONFIG_RAG = "/gen-ai/bots/:botId/configuration/rag"
@@ -59,29 +55,14 @@ class GenAIVerticle {
private const val PATH_COMPLETION_PLAYGROUND = "/gen-ai/bots/:botId/completion/playground"
}
- private val front = FrontClient
-
fun configure(webVerticle: WebVerticle) {
with(webVerticle) {
- /**
- * lamdba calling database to retrieve application definition from request context
- * @return [ApplicationDefinition]
- */
- val currentContextApp: (RoutingContext) -> ApplicationDefinition? = { context ->
- val botId = context.pathParam("botId")
- getNamespace(context)?.let { namespace ->
- front.getApplicationByNamespaceAndName(
- namespace, botId,
- )
- } ?: throw NotFoundException(404, "Could not find $botId in namespace")
- }
-
// --------------------------------------- Config - RAG ----------------------------------------
blockingJsonPost(
PATH_CONFIG_RAG,
admin,
) { context: RoutingContext, request: BotRAGConfigurationDTO ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) {
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) {
logger.info { "Saving 'RAG' configuration..." }
BotRAGConfigurationDTO(
RAGService.saveRag(request),
@@ -93,7 +74,7 @@ class GenAIVerticle {
PATH_CONFIG_RAG,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'RAG' configuration..." }
RAGService.getRAGConfiguration(app.namespace, app.name)?.let { BotRAGConfigurationDTO(it) }
}
@@ -103,7 +84,7 @@ class GenAIVerticle {
PATH_CONFIG_RAG,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Deleting 'RAG' configuration..." }
RAGService.deleteConfig(app.namespace, app.name)
}
@@ -114,7 +95,7 @@ class GenAIVerticle {
PATH_CONFIG_SENTENCE_GENERATION,
admin,
) { context: RoutingContext, request: BotSentenceGenerationConfigurationDTO ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) {
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) {
logger.info { "Saving 'Sentence Generation' configuration..." }
BotSentenceGenerationConfigurationDTO(
SentenceGenerationService.saveSentenceGeneration(request),
@@ -126,7 +107,7 @@ class GenAIVerticle {
PATH_CONFIG_SENTENCE_GENERATION,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'Sentence Generation' configuration..." }
SentenceGenerationService.getSentenceGenerationConfiguration(app.namespace, app.name)
?.let { BotSentenceGenerationConfigurationDTO(it) }
@@ -137,7 +118,7 @@ class GenAIVerticle {
PATH_CONFIG_SENTENCE_GENERATION_INFO,
nlpUser,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'Sentence Generation' configuration info..." }
SentenceGenerationService.getSentenceGenerationConfiguration(app.namespace, app.name)
?.let { BotSentenceGenerationInfoDTO(it) } ?: BotSentenceGenerationInfoDTO()
@@ -148,7 +129,7 @@ class GenAIVerticle {
PATH_CONFIG_SENTENCE_GENERATION,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Deleting 'Sentence Generation' configuration..." }
SentenceGenerationService.deleteConfig(app.namespace, app.name)
}
@@ -159,7 +140,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_STORE,
admin,
) { context: RoutingContext, request: BotVectorStoreConfigurationDTO ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) {
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) {
logger.info { "Saving 'Vector Store' configuration..." }
BotVectorStoreConfigurationDTO(
VectorStoreService.saveVectorStore(request),
@@ -171,7 +152,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_STORE,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'Vector Store' configuration..." }
VectorStoreService.getVectorStoreConfiguration(app.namespace, app.name)
?.let { BotVectorStoreConfigurationDTO(it) }
@@ -182,7 +163,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_STORE,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Deleting 'Vector Store' configuration..." }
VectorStoreService.deleteConfig(app.namespace, app.name)
}
@@ -193,7 +174,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_OBSERVABILITY,
admin,
) { context: RoutingContext, request: BotObservabilityConfigurationDTO ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) {
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) {
logger.info { "Saving 'Observability' configuration..." }
BotObservabilityConfigurationDTO(
ObservabilityService.saveObservability(request),
@@ -205,7 +186,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_OBSERVABILITY,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'Observability' configuration..." }
ObservabilityService.getObservabilityConfiguration(app.namespace, app.name)
?.let { BotObservabilityConfigurationDTO(it) }
@@ -216,7 +197,7 @@ class GenAIVerticle {
PATH_CONFIG_VECTOR_OBSERVABILITY,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Deleting 'Observability' configuration..." }
ObservabilityService.deleteConfig(app.namespace, app.name)
}
@@ -227,7 +208,7 @@ class GenAIVerticle {
PATH_CONFIG_DOCUMENT_COMPRESSOR,
admin,
) { context: RoutingContext, request: BotDocumentCompressorConfigurationDTO ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) {
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) {
logger.info { "Saving 'Document Compressor' configuration..." }
BotDocumentCompressorConfigurationDTO(
DocumentCompressorService.saveDocumentCompressor(request),
@@ -239,7 +220,7 @@ class GenAIVerticle {
PATH_CONFIG_DOCUMENT_COMPRESSOR,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Retrieving 'Document Compressor' configuration..." }
DocumentCompressorService.getDocumentCompressorConfiguration(app.namespace, app.name)
?.let { BotDocumentCompressorConfigurationDTO(it) }
@@ -250,7 +231,7 @@ class GenAIVerticle {
PATH_CONFIG_DOCUMENT_COMPRESSOR,
admin,
) { context: RoutingContext ->
- checkNamespaceAndExecute(context, currentContextApp) { app ->
+ checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "Deleting 'Document Compressor' configuration..." }
DocumentCompressorService.deleteConfig(app.namespace, app.name)
}
@@ -261,7 +242,7 @@ class GenAIVerticle {
PATH_COMPLETION_SENTENCE_GENERATION,
botUser,
) { context: RoutingContext, request: SentenceGenerationRequest ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) { app ->
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "GEN AI - Generating sentences..." }
CompletionService.generateSentences(request, app.namespace, app.name)
}
@@ -272,7 +253,7 @@ class GenAIVerticle {
PATH_COMPLETION_PLAYGROUND,
admin,
) { context: RoutingContext, request: PlaygroundRequest ->
- return@blockingJsonPost checkNamespaceAndExecute(context, currentContextApp) { app ->
+ return@blockingJsonPost checkNamespaceAndExecute(context, ::currentContextApp) { app ->
logger.info { "GEN AI - Playground..." }
CompletionService.generate(request, app.namespace, app.name)
}
@@ -280,12 +261,6 @@ class GenAIVerticle {
}
}
- /**
- * Get the namespace from the context
- * @param context : the vertx routing context
- */
- private fun getNamespace(context: RoutingContext): String? = ((context.user() ?: context.session()?.get("tockUser")) as? TockUser)?.namespace
-
/**
* Merge namespace and botId on requested [MetricFilter]
* @param namespace the namespace
diff --git a/bot/admin/server/src/test/kotlin/model/evaluation/EvaluationModelTest.kt b/bot/admin/server/src/test/kotlin/model/evaluation/EvaluationModelTest.kt
new file mode 100644
index 0000000000..bb975ca962
--- /dev/null
+++ b/bot/admin/server/src/test/kotlin/model/evaluation/EvaluationModelTest.kt
@@ -0,0 +1,51 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.model.evaluation
+
+import ai.tock.shared.jackson.mapper
+import com.fasterxml.jackson.module.kotlin.readValue
+import org.junit.jupiter.api.Test
+import kotlin.test.assertEquals
+
+/**
+ * Tests for evaluation model serialization/deserialization.
+ * Vérifie EvaluationDialogsQuery (valeurs par défaut) et structure des réponses.
+ */
+class EvaluationModelTest {
+ @Test
+ fun `EvaluationDialogsQuery deserializes empty JSON with defaults`() {
+ val query: EvaluationDialogsQuery = mapper.readValue("{}")
+ assertEquals(0, query.start)
+ assertEquals(10, query.size)
+ }
+
+ @Test
+ fun `EvaluationDialogsQuery deserializes with explicit values`() {
+ val query: EvaluationDialogsQuery = mapper.readValue("""{"start":10,"size":5}""")
+ assertEquals(10, query.start)
+ assertEquals(5, query.size)
+ }
+
+ @Test
+ fun `EvaluationDialogsQuery serializes to JSON`() {
+ val query = EvaluationDialogsQuery(start = 0, size = 20)
+ val json = mapper.writeValueAsString(query)
+ val back: EvaluationDialogsQuery = mapper.readValue(json)
+ assertEquals(query.start, back.start)
+ assertEquals(query.size, back.size)
+ }
+}
diff --git a/bot/admin/server/src/test/kotlin/service/EvaluationServiceTest.kt b/bot/admin/server/src/test/kotlin/service/EvaluationServiceTest.kt
new file mode 100644
index 0000000000..aaf53dac20
--- /dev/null
+++ b/bot/admin/server/src/test/kotlin/service/EvaluationServiceTest.kt
@@ -0,0 +1,1282 @@
+/*
+ * Copyright (C) 2017/2025 SNCF Connect & Tech
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ai.tock.bot.admin.service
+
+import ai.tock.bot.admin.AbstractTest
+import ai.tock.bot.admin.dialog.ActionReport
+import ai.tock.bot.admin.dialog.DialogReport
+import ai.tock.bot.admin.dialog.DialogReportDAO
+import ai.tock.bot.admin.dialog.DialogReportQuery
+import ai.tock.bot.admin.dialog.DialogReportQueryResult
+import ai.tock.bot.admin.evaluation.ActionRef
+import ai.tock.bot.admin.evaluation.Evaluation
+import ai.tock.bot.admin.evaluation.EvaluationDAO
+import ai.tock.bot.admin.evaluation.EvaluationReason
+import ai.tock.bot.admin.evaluation.EvaluationSample
+import ai.tock.bot.admin.evaluation.EvaluationSampleDAO
+import ai.tock.bot.admin.evaluation.EvaluationSampleStatus
+import ai.tock.bot.admin.evaluation.EvaluationStatus
+import ai.tock.bot.admin.evaluation.GroupedEvaluations
+import ai.tock.bot.admin.model.evaluation.CreateEvaluationSampleRequest
+import ai.tock.bot.engine.action.Action
+import ai.tock.bot.engine.action.ActionMetadata
+import ai.tock.bot.engine.action.SendAttachment
+import ai.tock.bot.engine.dialog.Dialog
+import ai.tock.bot.engine.message.Attachment
+import ai.tock.bot.engine.message.Choice
+import ai.tock.bot.engine.message.Sentence
+import ai.tock.bot.engine.user.PlayerId
+import ai.tock.bot.engine.user.PlayerType
+import ai.tock.shared.exception.rest.BadRequestException
+import ai.tock.shared.exception.rest.UnprocessableEntityException
+import ai.tock.shared.tockInternalInjector
+import ai.tock.translator.UserInterfaceType
+import com.github.salomonbrys.kodein.Kodein
+import com.github.salomonbrys.kodein.KodeinInjector
+import com.github.salomonbrys.kodein.bind
+import com.github.salomonbrys.kodein.provider
+import io.mockk.clearMocks
+import io.mockk.every
+import io.mockk.mockk
+import io.mockk.slot
+import io.mockk.verify
+import org.junit.jupiter.api.BeforeEach
+import org.junit.jupiter.api.Test
+import org.junit.jupiter.api.assertThrows
+import org.litote.kmongo.Id
+import org.litote.kmongo.newId
+import org.litote.kmongo.toId
+import java.time.Instant
+import kotlin.test.assertEquals
+import kotlin.test.assertNotNull
+import kotlin.test.assertNull
+import kotlin.test.assertTrue
+
+class EvaluationServiceTest : AbstractTest() {
+ companion object {
+ private const val NAMESPACE = "testNamespace"
+ private const val BOT_ID = "testBotId"
+ private const val USER = "testUser"
+
+ val evaluationSampleDAO: EvaluationSampleDAO = mockk()
+ val evaluationDAO: EvaluationDAO = mockk()
+ val dialogReportDAO: DialogReportDAO = mockk()
+
+ init {
+ tockInternalInjector = KodeinInjector()
+ val module =
+ Kodein.Module(allowSilentOverride = true) {
+ bind() with provider { evaluationSampleDAO }
+ bind() with provider { evaluationDAO }
+ bind() with provider { dialogReportDAO }
+ }
+ tockInternalInjector.inject(
+ Kodein {
+ import(defaultModulesBinding())
+ import(module, allowOverride = true)
+ },
+ )
+ }
+ }
+
+ @BeforeEach
+ fun setup() {
+ clearMocks(evaluationSampleDAO, evaluationDAO, dialogReportDAO)
+ }
+
+ private fun createDialogReport(
+ dialogId: String,
+ botActions: Int = 2,
+ ): DialogReport {
+ val actions = mutableListOf()
+ for (i in 0 until botActions * 2) {
+ val isBot = i % 2 == 1
+ actions.add(
+ ActionReport(
+ playerId = PlayerId(if (isBot) BOT_ID else "user", if (isBot) PlayerType.bot else PlayerType.user),
+ recipientId = PlayerId(if (isBot) "user" else BOT_ID, if (isBot) PlayerType.user else PlayerType.bot),
+ date = Instant.now(),
+ message = Sentence(if (isBot) "Bot response $i" else "User message $i"),
+ connectorType = null,
+ userInterfaceType = UserInterfaceType.textChat,
+ intent = null,
+ applicationId = null,
+ metadata = ActionMetadata(),
+ ),
+ )
+ }
+ return DialogReport(
+ actions = actions,
+ id = dialogId.toId(),
+ userInterface = UserInterfaceType.textChat,
+ )
+ }
+
+ @Test
+ fun `findSampleById returns DTO when sample exists`() {
+ val sampleId = newId()
+ val sample =
+ EvaluationSample(
+ _id = sampleId,
+ botId = BOT_ID,
+ namespace = NAMESPACE,
+ name = "Test",
+ description = null,
+ dialogActivityFrom = Instant.now(),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ dialogsCount = 5,
+ totalDialogCount = 100,
+ botActionCount = 15,
+ allowTestDialogs = false,
+ actionRefs = emptyList(),
+ status = EvaluationSampleStatus.IN_PROGRESS,
+ createdBy = USER,
+ statusChangedBy = USER,
+ )
+
+ every { evaluationSampleDAO.findById(any>()) } returns sample
+ every { evaluationDAO.countByStatus(sampleId) } returns
+ mapOf(
+ EvaluationStatus.UNSET to 5L,
+ EvaluationStatus.UP to 10L,
+ EvaluationStatus.DOWN to 0L,
+ )
+
+ val result = EvaluationService.findSampleById(sampleId.toString())
+
+ assertNotNull(result)
+ assertEquals(sampleId.toString(), result._id)
+ assertEquals(BOT_ID, result.botId)
+ assertEquals(15, result.evaluationsResult.total)
+ }
+
+ @Test
+ fun `findSampleById returns null when sample not found`() {
+ every { evaluationSampleDAO.findById(any>()) } returns null
+
+ val result = EvaluationService.findSampleById("507f1f77bcf86cd799439011")
+
+ assertNull(result)
+ }
+
+ @Test
+ fun `findSamplesByBotId returns samples with computed evaluationsResult`() {
+ val sampleId = newId()
+ val sample =
+ EvaluationSample(
+ _id = sampleId,
+ botId = BOT_ID,
+ namespace = NAMESPACE,
+ name = "Test Sample",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ dialogsCount = 5,
+ totalDialogCount = 100,
+ botActionCount = 15,
+ allowTestDialogs = false,
+ actionRefs = emptyList(),
+ status = EvaluationSampleStatus.IN_PROGRESS,
+ createdBy = USER,
+ statusChangedBy = USER,
+ )
+
+ every { evaluationSampleDAO.findByNamespaceAndBotId(NAMESPACE, BOT_ID, null) } returns listOf(sample)
+ every { evaluationDAO.countByStatus(sampleId) } returns
+ mapOf(
+ EvaluationStatus.UNSET to 10L,
+ EvaluationStatus.UP to 3L,
+ EvaluationStatus.DOWN to 2L,
+ )
+
+ val result = EvaluationService.findSamplesByBotId(NAMESPACE, BOT_ID)
+
+ assertEquals(1, result.size)
+ val dto = result[0]
+ assertEquals(15, dto.evaluationsResult.total)
+ assertEquals(5, dto.evaluationsResult.evaluated)
+ assertEquals(10, dto.evaluationsResult.remaining)
+ assertEquals(3, dto.evaluationsResult.positiveCount)
+ assertEquals(2, dto.evaluationsResult.negativeCount)
+ }
+
+ @Test
+ fun `createEvaluationSample throws UnprocessableEntityException when no evaluable actions found`() {
+ every { dialogReportDAO.search(any()) } returns
+ DialogReportQueryResult(
+ total = 0,
+ dialogs = emptyList(),
+ )
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test Sample",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ assertThrows {
+ EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+ }
+
+ verify(exactly = 0) { evaluationSampleDAO.save(any()) }
+ verify(exactly = 0) { evaluationDAO.createAll(any()) }
+ }
+
+ @Test
+ fun `createEvaluationSample throws when no evaluable actions in returned dialogs`() {
+ val dialogWithEmptyActions =
+ DialogReport(
+ actions = emptyList(),
+ id = "dialog1".toId(),
+ userInterface = UserInterfaceType.textChat,
+ )
+
+ every { dialogReportDAO.search(any()) } returns
+ DialogReportQueryResult(
+ total = 1,
+ dialogs = listOf(dialogWithEmptyActions),
+ )
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ assertThrows {
+ EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+ }
+
+ verify(exactly = 0) { evaluationSampleDAO.save(any()) }
+ }
+
+ @Test
+ fun `createEvaluationSample passes evaluableActionsOnly to dialog query`() {
+ val dialog1 = createDialogReport("dialog1", 2)
+ val dialog2 = createDialogReport("dialog2", 3)
+ val querySlot = slot()
+ every { dialogReportDAO.search(capture(querySlot)) } returns
+ DialogReportQueryResult(
+ total = 2,
+ dialogs = listOf(dialog1, dialog2),
+ )
+ every { evaluationSampleDAO.save(any()) } answers { firstArg() }
+ every { evaluationDAO.createAll(any()) } returns Unit
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test Sample",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+
+ assertTrue(querySlot.captured.evaluableActionsOnly)
+ }
+
+ @Test
+ fun `createEvaluationSample creates sample and evaluations`() {
+ val dialog1 = createDialogReport("dialog1", 2)
+ val dialog2 = createDialogReport("dialog2", 3)
+
+ every { dialogReportDAO.search(any()) } returns
+ DialogReportQueryResult(
+ total = 2,
+ dialogs = listOf(dialog1, dialog2),
+ )
+
+ val sampleSlot = slot()
+ every { evaluationSampleDAO.save(capture(sampleSlot)) } answers { sampleSlot.captured }
+
+ val evaluationsSlot = slot>()
+ every { evaluationDAO.createAll(capture(evaluationsSlot)) } returns Unit
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test Sample",
+ description = "Test description",
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ val result = EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+
+ assertNotNull(result)
+ assertEquals("Test Sample", result.name)
+ assertEquals(BOT_ID, result.botId)
+ assertEquals(NAMESPACE, result.namespace)
+ assertEquals(2, result.dialogsCount)
+ assertTrue(result.botActionCount > 0)
+ assertEquals(EvaluationSampleStatus.IN_PROGRESS, result.status)
+
+ verify { evaluationSampleDAO.save(any()) }
+ verify { evaluationDAO.createAll(any()) }
+
+ val createdEvaluations = evaluationsSlot.captured
+ assertTrue(createdEvaluations.isNotEmpty())
+ assertTrue(createdEvaluations.all { it.status == EvaluationStatus.UNSET })
+ assertEquals(result.botActionCount.toLong(), createdEvaluations.size.toLong())
+ }
+
+ @Test
+ fun `createEvaluationSample includes Choice actions`() {
+ val dialogWithChoice =
+ DialogReport(
+ actions =
+ listOf(
+ ActionReport(
+ playerId = PlayerId(BOT_ID, PlayerType.bot),
+ recipientId = PlayerId("user", PlayerType.user),
+ date = Instant.now(),
+ message = Choice("test_intent", emptyMap()),
+ connectorType = null,
+ userInterfaceType = UserInterfaceType.textChat,
+ intent = null,
+ applicationId = null,
+ metadata = ActionMetadata(),
+ ),
+ ),
+ id = "dialog1".toId(),
+ userInterface = UserInterfaceType.textChat,
+ )
+
+ every { dialogReportDAO.search(any()) } returns
+ DialogReportQueryResult(
+ total = 1,
+ dialogs = listOf(dialogWithChoice),
+ )
+ every { evaluationSampleDAO.save(any()) } answers { firstArg() }
+ every { evaluationDAO.createAll(any()) } returns Unit
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ val result = EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+
+ assertNotNull(result)
+ assertEquals(1, result.botActionCount)
+ assertEquals(1, result.dialogsCount)
+ }
+
+ @Test
+ fun `createEvaluationSample includes Attachment and Location actions`() {
+ val dialogWithAttachment =
+ DialogReport(
+ actions =
+ listOf(
+ ActionReport(
+ playerId = PlayerId(BOT_ID, PlayerType.bot),
+ recipientId = PlayerId("user", PlayerType.user),
+ date = Instant.now(),
+ message = Attachment("https://example.com/file.pdf", SendAttachment.AttachmentType.file),
+ connectorType = null,
+ userInterfaceType = UserInterfaceType.textChat,
+ intent = null,
+ applicationId = null,
+ metadata = ActionMetadata(),
+ ),
+ ),
+ id = "dialog1".toId(),
+ userInterface = UserInterfaceType.textChat,
+ )
+
+ every { dialogReportDAO.search(any()) } returns
+ DialogReportQueryResult(
+ total = 1,
+ dialogs = listOf(dialogWithAttachment),
+ )
+ every { evaluationSampleDAO.save(any()) } answers { firstArg() }
+ every { evaluationDAO.createAll(any()) } returns Unit
+
+ val request =
+ CreateEvaluationSampleRequest(
+ name = "Test",
+ description = null,
+ dialogActivityFrom = Instant.now().minusSeconds(86400),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ allowTestDialogs = false,
+ )
+
+ val result = EvaluationService.createEvaluationSample(NAMESPACE, BOT_ID, request, USER)
+
+ assertNotNull(result)
+ assertEquals(1, result.botActionCount)
+ }
+
+ @Test
+ fun `evaluate updates evaluation with UP status`() {
+ val sampleId = newId()
+ val evaluationId = newId()
+
+ val sample =
+ EvaluationSample(
+ _id = sampleId,
+ botId = BOT_ID,
+ namespace = NAMESPACE,
+ name = "Test",
+ description = null,
+ dialogActivityFrom = Instant.now(),
+ dialogActivityTo = Instant.now(),
+ requestedDialogCount = 10,
+ dialogsCount = 5,
+ totalDialogCount = 100,
+ botActionCount = 15,
+ allowTestDialogs = false,
+ actionRefs = listOf(ActionRef("dialog1".toId