Refactor/bug fixes for merging -- DO NOT USE IN PRODUCTION YET

Author: JamesKane

app/models/api/haplogroups/TreeMergeModels.scala

@@ -212,6 +212,47 @@ object SplitOperation {
   implicit val format: OFormat[SplitOperation] = Json.format[SplitOperation]
 }
 
+/**
+ * Records an ambiguous or inconsistent placement during merge.
+ *
+ * Phylogenetics often involves uncertain placement. This flag alerts human
+ * curators to branches where SNP data is contradictory, possibly due to:
+ *   - Sequencing errors in source data
+ *   - Recurrent mutations (homoplasy)
+ *   - Missing intermediate nodes
+ *   - Nomenclature mismatches between sources
+ *
+ * @param nodeName The node with ambiguous placement
+ * @param ambiguityType Classification of the ambiguity
+ * @param description Human-readable explanation
+ * @param sharedVariants Variants that matched (overlap)
+ * @param conflictingVariants Variants that conflict (present in one but not expected)
+ * @param candidateMatches Other nodes that could have been matches
+ * @param resolution How the algorithm resolved the ambiguity
+ * @param confidence Score from 0.0 (very uncertain) to 1.0 (confident)
+ */
+case class PlacementAmbiguity(
+  nodeName: String,
+  ambiguityType: String,
+  description: String,
+  sharedVariants: List[String] = List.empty,
+  conflictingVariants: List[String] = List.empty,
+  candidateMatches: List[String] = List.empty,
+  resolution: String,
+  confidence: Double
+)
+
+object PlacementAmbiguity {
+  implicit val format: OFormat[PlacementAmbiguity] = Json.format[PlacementAmbiguity]
+
+  // Ambiguity type constants
+  val PARTIAL_MATCH = "PARTIAL_MATCH"           // Some SNPs match, others don't
+  val MULTIPLE_CANDIDATES = "MULTIPLE_CANDIDATES" // Multiple nodes could be the match
+  val RECURRENT_SNP = "RECURRENT_SNP"           // SNP appears in multiple lineages
+  val ORPHAN_PLACEMENT = "ORPHAN_PLACEMENT"     // Node placed without strong variant evidence
+  val NAME_VARIANT_MISMATCH = "NAME_VARIANT_MISMATCH" // Name matches but variants differ
+}
+
 /**
  * Result of a merge operation.
  */
@@ -221,6 +262,7 @@ case class TreeMergeResponse(
   statistics: MergeStatistics,
   conflicts: List[MergeConflict] = List.empty,
   splits: List[SplitOperation] = List.empty,
+  ambiguities: List[PlacementAmbiguity] = List.empty,
   errors: List[String] = List.empty
 )
 
@@ -243,6 +285,7 @@ case class MergePreviewResponse(
   statistics: MergeStatistics,
   conflicts: List[MergeConflict],
   splits: List[SplitOperation],
+  ambiguities: List[PlacementAmbiguity],
   newNodes: List[String],
   updatedNodes: List[String],
   unchangedNodes: List[String]

app/repositories/HaplogroupCoreRepository.scala

@@ -152,6 +152,38 @@ trait HaplogroupCoreRepository {
    * @return A Future containing an Option with the latest revision ID, or None if no relationships exist for the child.
    */
   def getLatestRelationshipRevisionId(childHaplogroupId: Int): Future[Option[Int]]
+
+  // === Bulk Operations for Merge ===
+
+  /**
+   * Create multiple haplogroups in a single batch operation.
+   *
+   * @param haplogroups The haplogroups to create
+   * @return A Future containing a map from haplogroup name to newly assigned ID
+   */
+  def createBatch(haplogroups: Seq[Haplogroup]): Future[Map[String, Int]]
+
+  /**
+   * Create multiple parent-child relationships in a single batch operation.
+   *
+   * @param relationships Sequence of (childId, parentId, source) tuples
+   * @return A Future containing the sequence of created relationship IDs
+   */
+  def createRelationshipsBatch(relationships: Seq[(Int, Int, String)]): Future[Seq[Int]]
+
+  /**
+   * Update provenance for multiple haplogroups in a single batch.
+   *
+   * @param updates Sequence of (haplogroupId, provenance) tuples
+   * @return A Future containing the count of updated records
+   */
+  def updateProvenanceBatch(updates: Seq[(Int, HaplogroupProvenance)]): Future[Int]
+
+  /**
+   * Get all active parent-child relationships for a haplogroup type.
+   * Returns (childId, parentId) tuples for building in-memory tree.
+   */
+  def getAllRelationships(haplogroupType: HaplogroupType): Future[Seq[(Int, Int)]]
 }
 
 class HaplogroupCoreRepositoryImpl @Inject()(
@@ -550,4 +582,60 @@ class HaplogroupCoreRepositoryImpl @Inject()(
       .result
     runQuery(query)
   }
+
+  // === Bulk Operations for Merge ===
+
+  override def createBatch(haplogroupsToCreate: Seq[Haplogroup]): Future[Map[String, Int]] = {
+    if (haplogroupsToCreate.isEmpty) return Future.successful(Map.empty)
+
+    runQuery(
+      (haplogroups returning haplogroups.map(h => (h.name, h.haplogroupId))) ++= haplogroupsToCreate
+    ).map(_.toMap)
+  }
+
+  override def createRelationshipsBatch(relationships: Seq[(Int, Int, String)]): Future[Seq[Int]] = {
+    if (relationships.isEmpty) return Future.successful(Seq.empty)
+
+    import models.domain.haplogroups.HaplogroupRelationship
+    val now = LocalDateTime.now()
+
+    val relationshipsToInsert = relationships.map { case (childId, parentId, source) =>
+      HaplogroupRelationship(
+        id = None,
+        childHaplogroupId = childId,
+        parentHaplogroupId = parentId,
+        revisionId = 1,
+        validFrom = now,
+        validUntil = None,
+        source = source
+      )
+    }
+
+    runQuery(
+      (haplogroupRelationships returning haplogroupRelationships.map(_.haplogroupRelationshipId)) ++= relationshipsToInsert
+    )
+  }
+
+  override def updateProvenanceBatch(updates: Seq[(Int, HaplogroupProvenance)]): Future[Int] = {
+    if (updates.isEmpty) return Future.successful(0)
+
+    // Use DBIO.sequence for batch updates
+    val updateActions = updates.map { case (id, provenance) =>
+      haplogroups
+        .filter(_.haplogroupId === id)
+        .map(_.provenance)
+        .update(Some(provenance))
+    }
+
+    runTransactionally(DBIO.sequence(updateActions)).map(_.sum)
+  }
+
+  override def getAllRelationships(haplogroupType: HaplogroupType): Future[Seq[(Int, Int)]] = {
+    val query = for {
+      rel <- activeRelationships
+      child <- activeHaplogroups if child.haplogroupId === rel.childHaplogroupId && child.haplogroupType === haplogroupType
+    } yield (rel.childHaplogroupId, rel.parentHaplogroupId)
+
+    runQuery(query.result)
+  }
 }

app/repositories/HaplogroupVariantRepository.scala

@@ -96,6 +96,32 @@ trait HaplogroupVariantRepository {
    * @return A Future containing a sequence of (haplogroupId, VariantV2) tuples.
    */
   def getVariantsForHaplogroups(haplogroupIds: Seq[Int]): Future[Seq[(Int, VariantV2)]]
+
+  /**
+   * Bulk associate variants with haplogroups in a single operation.
+   * Uses ON CONFLICT to handle duplicates gracefully.
+   *
+   * @param associations Sequence of (haplogroupId, variantId) tuples
+   * @return A Future containing the sequence of haplogroup_variant_ids created or found
+   */
+  def bulkAddVariantsToHaplogroups(associations: Seq[(Int, Int)]): Future[Seq[Int]]
+
+  /**
+   * Bulk remove variant associations from a haplogroup.
+   *
+   * @param haplogroupId The haplogroup to remove variants from
+   * @param variantIds   The variant IDs to remove
+   * @return A Future containing the number of associations removed
+   */
+  def bulkRemoveVariantsFromHaplogroup(haplogroupId: Int, variantIds: Seq[Int]): Future[Int]
+
+  /**
+   * Gets the variant IDs (not haplogroup_variant_ids) for a haplogroup.
+   *
+   * @param haplogroupId The haplogroup ID
+   * @return A Future containing the sequence of variant IDs
+   */
+  def getVariantIdsForHaplogroup(haplogroupId: Int): Future[Seq[Int]]
 }
 
 class HaplogroupVariantRepositoryImpl @Inject()(
@@ -298,4 +324,38 @@ class HaplogroupVariantRepositoryImpl @Inject()(
 
     runQuery(query.result)
   }
+
+  override def bulkAddVariantsToHaplogroups(associations: Seq[(Int, Int)]): Future[Seq[Int]] = {
+    if (associations.isEmpty) return Future.successful(Seq.empty)
+
+    // Build values clause for bulk insert
+    val valuesClause = associations.map { case (hgId, varId) =>
+      s"($hgId, $varId)"
+    }.mkString(", ")
+
+    val insertQuery = sql"""
+      INSERT INTO tree.haplogroup_variant (haplogroup_id, variant_id)
+      VALUES #$valuesClause
+      ON CONFLICT (haplogroup_id, variant_id) DO UPDATE
+        SET haplogroup_id = EXCLUDED.haplogroup_id
+      RETURNING haplogroup_variant_id
+    """.as[Int]
+
+    runQuery(insertQuery)
+  }
+
+  override def bulkRemoveVariantsFromHaplogroup(haplogroupId: Int, variantIds: Seq[Int]): Future[Int] = {
+    if (variantIds.isEmpty) return Future.successful(0)
+
+    val query = haplogroupVariants
+      .filter(hv => hv.haplogroupId === haplogroupId && hv.variantId.inSet(variantIds))
+      .delete
+
+    runQuery(query)
+  }
+
+  override def getVariantIdsForHaplogroup(haplogroupId: Int): Future[Seq[Int]] = {
+    val query = haplogroupVariants.filter(_.haplogroupId === haplogroupId).map(_.variantId)
+    runQuery(query.result)
+  }
 }

app/repositories/VariantV2Repository.scala

@@ -98,6 +98,13 @@ trait VariantV2Repository {
   def streamAll(): Future[Seq[VariantV2]]
   def findByIds(ids: Seq[Int]): Future[Seq[VariantV2]]
 
+  /**
+   * Bulk search for variants by a list of names (canonical names or aliases).
+   * Returns a map from search name (uppercased) to matching variants.
+   * Much more efficient than individual searchByName calls for large batches.
+   */
+  def searchByNames(names: Seq[String]): Future[Map[String, Seq[VariantV2]]]
+
   // === DU Naming Authority ===
 
   def nextDuName(): Future[String]
@@ -625,6 +632,44 @@ class VariantV2RepositoryImpl @Inject()(
   override def streamAll(): Future[Seq[VariantV2]] = db.run(variantsV2.result)
   override def findByIds(ids: Seq[Int]): Future[Seq[VariantV2]] = if (ids.isEmpty) Future.successful(Seq.empty) else db.run(variantsV2.filter(_.variantId.inSet(ids)).result)
 
+  override def searchByNames(names: Seq[String]): Future[Map[String, Seq[VariantV2]]] = {
+    if (names.isEmpty) return Future.successful(Map.empty)
+
+    // Variant names are typically uppercase (M269, L21, etc.)
+    val searchNames = names.flatMap(n => Seq(n, n.toUpperCase)).distinct
+    val batchSize = 2000 // PostgreSQL handles IN/ANY efficiently up to several thousand
+
+    val batches = searchNames.grouped(batchSize).toSeq
+
+    // Process batches sequentially
+    batches.foldLeft(Future.successful(Map.empty[String, Seq[VariantV2]])) { (accFuture, batch) =>
+      accFuture.flatMap { acc =>
+        val namesArray = batch.map(n => s"'${n.replace("'", "''")}'").mkString(",")
+        val query = sql"""
+          SELECT variant_id, canonical_name, mutation_type, naming_status,
+                 aliases, coordinates, defining_haplogroup_id, evidence,
+                 primers, notes, created_at, updated_at, annotations
+          FROM variant_v2
+          WHERE canonical_name = ANY(ARRAY[#$namesArray])
+        """.as[VariantV2](variantV2GetResult)
+
+        db.run(query).map { variants =>
+          val batchResults = variants.flatMap { v =>
+            v.canonicalName.map(cn => cn.toUpperCase -> v)
+          }.groupMap(_._1)(_._2)
+
+          // Merge into accumulator
+          batchResults.foldLeft(acc) { case (map, (name, vars)) =>
+            map.updatedWith(name) {
+              case Some(existing) => Some(existing ++ vars)
+              case None => Some(vars)
+            }
+          }
+        }
+      }
+    }
+  }
+
   // === DU Naming Authority ===
 
   private val DuNamePattern = "^DU[1-9][0-9]*$".r