Add LLM Assistant Functionality for Scoring

build.gradle.kts

@@ -66,6 +66,8 @@ dependencies {
     implementation("org.apache.httpcomponents.client5:httpclient5:5.2.1")
     implementation("org.apache.tika:tika-core:2.8.0")
     implementation("org.flywaydb:flyway-core")
+    implementation("com.squareup.okhttp3:okhttp:4.9.2")
+    implementation("com.fasterxml.jackson.module:jackson-module-kotlin:2.12.+")
     implementation("io.github.oshai:kotlin-logging-jvm:5.1.0")
     testImplementation("org.springframework.security:spring-security-test")
     testImplementation("org.springframework.boot:spring-boot-starter-test") {

src/main/java/ch/uzh/ifi/access/config/ModelMapperConfig.java

@@ -5,16 +5,15 @@
 import ch.uzh.ifi.access.model.Submission;
 import ch.uzh.ifi.access.model.Task;
 import ch.uzh.ifi.access.model.constants.Visibility;
-import ch.uzh.ifi.access.model.dto.AssignmentDTO;
-import ch.uzh.ifi.access.model.dto.CourseDTO;
-import ch.uzh.ifi.access.model.dto.SubmissionDTO;
-import ch.uzh.ifi.access.model.dto.TaskDTO;
+import ch.uzh.ifi.access.model.dto.*;
 import org.apache.commons.lang3.ObjectUtils;
 import org.modelmapper.ModelMapper;
 import org.modelmapper.convention.MatchingStrategies;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import java.util.Objects;
+
 @Configuration
 public class ModelMapperConfig {
     // TODO: convert to Kotlin. This is very tricky, because ModelMapper is very Java-specific. Maybe replace it.
@@ -30,11 +29,46 @@ public ModelMapper modelMapper() {
         modelMapper.typeMap(AssignmentDTO.class, Assignment.class)
                 .addMappings(mapping -> mapping.skip(AssignmentDTO::getTasks, Assignment::setTasks));
         modelMapper.typeMap(TaskDTO.class, Task.class)
-                .addMappings(mapping -> mapping.skip(TaskDTO::getFiles, Task::setFiles));
+                .addMappings(mapper -> {
+                    mapper.skip(Task::setLlmSubmission);
+                    mapper.skip(Task::setLlmSolution);
+                    mapper.skip(Task::setLlmRubrics);
+                    mapper.skip(Task::setLlmCot);
+                    mapper.skip(Task::setLlmVoting);
+                    mapper.skip(Task::setLlmExamples);
+                    mapper.skip(Task::setLlmPrompt);
+                    mapper.skip(Task::setLlmPre);
+                    mapper.skip(Task::setLlmPost);
+                    mapper.skip(Task::setLlmModel);
+                    mapper.skip(Task::setLlmTemperature);
+                    mapper.skip(Task::setLlmMaxPoints);
+                });
         modelMapper.typeMap(SubmissionDTO.class, Submission.class)
                 .addMappings(mapping -> mapping.skip(Submission::setFiles));
         modelMapper.createTypeMap(String.class, Visibility.class)
                 .setConverter(context -> Visibility.valueOf(context.getSource().toUpperCase()));
+        modelMapper.addConverter((context) -> {
+            TaskDTO source = (TaskDTO) context.getSource();
+            Task destination = (Task) context.getDestination();
+
+            if (source.getLlm() != null) {
+                LLMConfigDTO llm = source.getLlm();
+                destination.setLlmSubmission(llm.getSubmission());
+                destination.setLlmSolution(llm.getSolution());
+                destination.setLlmRubrics(llm.getRubrics());
+                destination.setLlmCot(llm.getCot());
+                destination.setLlmVoting(llm.getVoting());
+                destination.setLlmExamples(llm.getExamples());
+                destination.setLlmPrompt(llm.getPrompt());
+                destination.setLlmPre(llm.getPre());
+                destination.setLlmPost(llm.getPost());
+                destination.setLlmModel(llm.getModel());
+                destination.setLlmTemperature(llm.getTemperature());
+                destination.setLlmMaxPoints(llm.getMaxPoints());
+            }
+
+            return destination;
+        }, TaskDTO.class, Task.class);
         return modelMapper;
     }
 }

src/main/kotlin/ch/uzh/ifi/access/config/SecurityConfig.kt

@@ -122,6 +122,11 @@ class SecurityConfig(private val env: Environment) {
         return Path.of(env.getProperty("WORKING_DIR", "/workspace/data"))
     }
 
+    @Bean
+    fun assistantServerUrl(): String {
+        return env.getProperty("ASSISTANT_SERVER_URL", "http://localhost:4000")
+    }
+
     @Bean
     fun accessRealm(): RealmResource {
         val keycloakClient = Keycloak.getInstance(

src/main/kotlin/ch/uzh/ifi/access/controller/CourseController.kt

@@ -17,6 +17,7 @@ import java.time.LocalDateTime
 import java.util.concurrent.Semaphore
 
 
+
 @RestController
 class CourseRootController(
     private val courseService: CourseService,

src/main/kotlin/ch/uzh/ifi/access/model/Task.kt

@@ -71,6 +71,42 @@ class Task {
     val attemptRefill: Int?
         get() = if (Objects.nonNull(attemptWindow)) Math.toIntExact(attemptWindow!!.toSeconds()) else null
 
+    @Column(nullable = false)
+    var llmSubmission: String? = null
+
+    @Column
+    var llmSolution: String? = null
+
+    @Column
+    var llmRubrics: String? = null
+
+    @Column(nullable = false)
+    var llmCot: Boolean = false
+
+    @Column(nullable = false)
+    var llmVoting: Int = 1
+
+    @Column
+    var llmExamples: String? = null
+
+    @Column
+    var llmPrompt: String? = null
+
+    @Column
+    var llmPre: String? = null
+
+    @Column
+    var llmPost: String? = null
+
+    @Column
+    var llmTemperature: Double? = 0.2
+
+    @Column
+    var llmModel: String? = null
+
+    @Column
+    var llmMaxPoints: Double? = null
+
     fun createFile(): TaskFile {
         val newTaskFile = TaskFile()
         files.add(newTaskFile)

src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantDTO.kt

@@ -0,0 +1,80 @@
+package ch.uzh.ifi.access.model.dto
+
+import com.fasterxml.jackson.annotation.JsonProperty
+import lombok.Data
+
+
+enum class LLMType {
+    gpt,
+    claude
+}
+
+@Data
+class RubricDTO (
+    @JsonProperty("id")
+    var id: String,
+
+    @JsonProperty("title")
+    var title: String,
+
+    @JsonProperty("points")
+    var points: Double
+)
+
+@Data
+class FewShotExampleDTO (
+    @JsonProperty("answer")
+    var answer: String,
+
+    @JsonProperty("points")
+    var points: String // Stringified JSON for flexibility
+)
+
+@Data
+class AssistantDTO (
+    @JsonProperty("question")
+    var question: String,
+
+    @JsonProperty("answer")
+    var answer: String,
+
+    @JsonProperty("rubrics")
+    var rubrics: List<RubricDTO>? = null,
+
+    @JsonProperty("modelSolution")
+    var modelSolution: String? = null,
+
+    @JsonProperty("maxPoints")
+    var maxPoints: Double? = 1.0,
+
+    @JsonProperty("minPoints")
+    var minPoints: Double? = 0.0,
+
+    @JsonProperty("pointStep")
+    var pointStep: Double? = 0.5,
+
+    @JsonProperty("chainOfThought")
+    var chainOfThought: Boolean? = true,
+
+    @JsonProperty("votingCount")
+    var votingCount: Int? = 1,
+
+    @JsonProperty("temperature")
+    var temperature: Double? = 0.2,
+
+    @JsonProperty("fewShotExamples")
+    var fewShotExamples: List<FewShotExampleDTO>? = null,
+
+    @JsonProperty("prePrompt")
+    var prePrompt: String? = null,
+
+    @JsonProperty("prompt")
+    var prompt: String? = null,
+
+    @JsonProperty("postPrompt")
+    var postPrompt: String? = null,
+
+    @JsonProperty("llmType")
+    var llmType: LLMType? = null
+
+)

src/main/kotlin/ch/uzh/ifi/access/model/dto/AssistantResponseDTO.kt

@@ -0,0 +1,45 @@
+package ch.uzh.ifi.access.model.dto
+
+import com.fasterxml.jackson.annotation.JsonIgnoreProperties
+import com.fasterxml.jackson.annotation.JsonProperty
+import lombok.Data
+
+enum class Status {
+    correct,
+    incorrect,
+    incomplete
+}
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Data
+class AssistantResponseDTO (
+    @JsonProperty("status")
+    var status: Status,
+
+    @JsonProperty("feedback")
+    var feedback: String,
+
+    @JsonProperty("hint")
+    var hint: String? = null,
+
+    @JsonProperty("points")
+    var points: Double,
+
+    @JsonProperty("votingResult")
+    var votingResult: String
+)
+
+@JsonIgnoreProperties(ignoreUnknown = true)
+@Data
+class AssistantEvaluationResponseDTO (
+    @JsonProperty("status")
+    var status: String,
+
+    @JsonProperty("result")
+    var result: AssistantResponseDTO?
+)
+
+class TaskIdDTO(
+    @JsonProperty("jobId")
+    var jobId: String
+)

src/main/kotlin/ch/uzh/ifi/access/model/dto/LLMConfigDTO.kt

@@ -0,0 +1,16 @@
+package ch.uzh.ifi.access.model.dto
+
+data class LLMConfigDTO(
+    var submission: String? = null,
+    var solution: String? = null,
+    var rubrics: String? = null,
+    var cot: Boolean = false,
+    var voting: Int = 1,
+    var examples: String? = null,
+    var prompt: String? = null,
+    var pre: String? = null,
+    var post: String? = null,
+    var temperature: Double? = null,
+    var model: String? = null,
+    var maxPoints: Double? = null
+) 

src/main/kotlin/ch/uzh/ifi/access/model/dto/TaskDTO.kt

@@ -11,5 +11,6 @@ class TaskDTO(
     var maxAttempts: Int? = null,
     var refill: Int? = null,
     var evaluator: TaskEvaluatorDTO? = null,
-    var files: TaskFilesDTO? = null
+    var files: TaskFilesDTO? = null,
+    var llm: LLMConfigDTO? = null,
 )