Feature/store method checksum with coverage epmdj 11189

admin-app/src/main/resources/openapi.yml

@@ -1522,14 +1522,20 @@ components:
           type: string
         instanceId:
           type: string
+        commitSha:
+          type: string
+        buildVersion:
+          type: string
         coverage:
           type: array
           items:
-            $ref: '#/components/schemas/SingleClassCoveragePayload'
-    SingleClassCoveragePayload:
+            $ref: '#/components/schemas/SingleMethodCoveragePayload'
+    SingleMethodCoveragePayload:
       type: object
       properties:
-        classname:
+        signature:
+          type: string
+        bodyChecksum:
           type: string
         testId:
           type: string
@@ -1576,20 +1582,6 @@ components:
           type: integer
         bodyChecksum:
           type: string
-        annotations:
-          type: object
-          additionalProperties:
-            type: array
-            items:
-              type: string
-          nullable: true
-        classAnnotations:
-          type: object
-          additionalProperties:
-            type: array
-            items:
-              type: string
-          nullable: true
     AddTestsPayload:
       type: object
       properties:
@@ -1684,12 +1676,6 @@ components:
         classnamePattern:
           type: string
           nullable: true
-        annotationsPattern:
-          type: string
-          nullable: true
-        classAnnotationsPattern:
-          type: string
-          nullable: true
     # Auth Schemas
     LoginPayload:
       type: object

admin-etl/src/main/kotlin/com/epam/drill/admin/etl/impl/SqlDataLoader.kt

@@ -100,6 +100,6 @@ abstract class SqlDataLoader<T: EtlRow>(
             logger.error(e) { "Error during deleting data with loader $name for groupId $groupId: ${e.message ?: e.javaClass.simpleName}" }
             throw e
         }
-        logger.debug { "Loader [$name] deleted data for groupId $groupId in ${duration}ms" }
+        logger.debug { "Loader [$name] deleted data for groupId [$groupId] in ${duration}ms" }
     }
 }

admin-metrics/src/main/kotlin/com/epam/drill/admin/metrics/job/MetricsDataRetentionPolicyJob.kt

@@ -40,7 +40,7 @@ class MetricsDataRetentionPolicyJob(
                     if (groupId != null) mapOf(groupId to (it[groupId] ?: Instant.EPOCH)) else it
                 }.map { (groupId, initTimestamp) ->
                     async {
-                        logger.info { "Deleting all metrics data for groupId $groupId older than $initTimestamp..." }
+                        logger.info { "Deleting all metrics data for groupId [$groupId] older than $initTimestamp..." }
                         metricsRepository.deleteAllBuildDataCreatedBefore(groupId, initTimestamp)
                         metricsRepository.deleteAllTestDataCreatedBefore(groupId, initTimestamp)
                         metricsRepository.deleteAllDailyDataCreatedBefore(groupId, initTimestamp)

admin-metrics/src/main/resources/metrics/db/etl/build_methods_extractor.sql

@@ -1,29 +1,30 @@
 SELECT
     m.group_id,
     m.app_id,
-    m.build_id,
+    bm.build_id,
     m.signature,
     m.body_checksum,
     m.probes_count,
-    md5(m.signature||':'||m.body_checksum||':'||m.probes_count) as method_id,
-    m.name AS method_name,
-    m.classname AS class_name,
-    m.params AS method_params,
+    m.method_id,
+    m.method_name,
+    m.class_name,
+    m.method_params,
     m.return_type,
-    m.created_at AS created_at,
-    DATE_TRUNC('day', m.created_at) AS created_at_day
-FROM raw_data.methods m
-WHERE m.group_id = :group_id
-    AND m.created_at > :since_timestamp
-    AND m.created_at <= :until_timestamp
-    AND probes_count > 0
-    AND NOT EXISTS (SELECT 1
+    bm.created_at,
+    DATE_TRUNC('day',bm.created_at) AS created_at_day
+FROM raw_data.build_methods bm
+JOIN raw_data.methods m ON m.method_id = bm.method_id AND m.app_id = bm.app_id AND m.group_id = bm.group_id
+WHERE bm.group_id = :group_id
+    AND bm.created_at > :since_timestamp
+    AND bm.created_at <= :until_timestamp
+    AND m.probes_count > 0
+    AND NOT EXISTS (
+        SELECT 1
         FROM raw_data.method_ignore_rules r
         WHERE r.group_id = m.group_id
             AND r.app_id = m.app_id
-            AND (r.name_pattern IS NOT NULL AND m.name::text ~ r.name_pattern::text
-                OR r.classname_pattern IS NOT NULL AND m.classname::text ~ r.classname_pattern::text
-                OR r.annotations_pattern IS NOT NULL AND m.annotations::text ~ r.annotations_pattern::text
-                OR r.class_annotations_pattern IS NOT NULL AND m.class_annotations::text ~ r.class_annotations_pattern::text))
-ORDER BY m.created_at ASC, m.signature
+            AND (r.classname_pattern IS NOT NULL AND m.class_name::text ~ r.classname_pattern::text
+                OR r.name_pattern IS NOT NULL AND m.method_name::text ~ r.name_pattern::text)
+    )
+ORDER BY bm.created_at ASC, bm.group_id, bm.method_id
 LIMIT :limit

admin-metrics/src/main/resources/metrics/db/etl/coverage_extractor.sql

@@ -4,9 +4,9 @@ SELECT
     i.build_id,
     i.env_id AS app_env_id,
     CASE WHEN c.test_session_id = 'GLOBAL' THEN NULL ELSE c.test_session_id END AS test_session_id,
-    CASE WHEN c.test_id = 'TEST_CONTEXT_NONE' THEN NULL ELSE test_id END AS test_launch_id,
-    MD5(c.signature||':'||m.body_checksum||':'||m.probes_count) AS method_id,
-    c.signature,
+    CASE WHEN c.test_id = 'TEST_CONTEXT_NONE' THEN NULL ELSE c.test_id END AS test_launch_id,
+    c.method_id,
+    m.signature,
     b.branch,
     tl.test_definition_id,
     test_tag,
@@ -18,30 +18,23 @@ SELECT
     DATE_TRUNC('day', c.created_at) AS created_at_day,
     c.probes AS probes
 FROM raw_data.method_coverage c
-JOIN raw_data.methods m ON m.signature = c.signature
-	AND m.build_id = c.build_id
- 	AND m.probes_count = c.probes_count
-	AND m.app_id = c.app_id
- 	AND m.group_id = c.group_id
-	 AND NOT EXISTS (
+JOIN raw_data.methods m ON m.method_id = c.method_id AND m.app_id = c.app_id AND m.group_id = c.group_id
+    AND NOT EXISTS (
             SELECT 1
             FROM raw_data.method_ignore_rules r
             WHERE r.group_id = m.group_id
-		        AND r.app_id = m.app_id
-		        AND (r.classname_pattern IS NOT NULL AND m.classname::text ~ r.classname_pattern::text
-		            OR r.name_pattern IS NOT NULL AND m.name::text ~ r.name_pattern::text
-		            OR r.annotations_pattern IS NOT NULL AND m.annotations::text ~ r.annotations_pattern::text
-		            OR r.class_annotations_pattern IS NOT NULL AND m.class_annotations::text ~ r.class_annotations_pattern::text))
-JOIN raw_data.instances i ON i.id = c.instance_id
-	AND i.app_id = c.app_id
-	AND i.group_id = c.group_id
-JOIN raw_data.builds b ON b.group_id = c.group_id AND b.app_id = c.app_id AND b.id = i.build_id
+                AND r.app_id = m.app_id
+                AND (r.classname_pattern IS NOT NULL AND m.class_name::text ~ r.classname_pattern::text
+                    OR r.name_pattern IS NOT NULL AND m.method_name::text ~ r.name_pattern::text)
+        )
+JOIN raw_data.instances i ON i.id = c.instance_id AND i.app_id = c.app_id AND i.group_id = c.group_id
+JOIN raw_data.builds b ON b.group_id = c.group_id AND b.app_id = c.app_id AND b.id = c.build_id
 LEFT JOIN raw_data.test_sessions ts ON ts.id = c.test_session_id AND ts.group_id = c.group_id
 LEFT JOIN raw_data.test_launches tl ON tl.id = c.test_id AND tl.group_id = c.group_id
 LEFT JOIN raw_data.test_definitions td ON td.group_id = tl.group_id AND td.id = tl.test_definition_id
 LEFT JOIN LATERAL unnest(td.tags) AS test_tag ON TRUE
 WHERE c.created_at > :since_timestamp
     AND c.created_at <= :until_timestamp
     AND c.group_id = :group_id
-ORDER BY c.created_at, c.signature
+ORDER BY c.created_at, c.group_id, c.method_id
 LIMIT :limit

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/BuildDiffReportApiTest.kt

@@ -155,6 +155,7 @@ class BuildDiffReportApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/ChangesApiTest.kt

@@ -20,6 +20,7 @@ import com.epam.drill.admin.metrics.views.ChangeType
 import com.epam.drill.admin.test.*
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
 import com.epam.drill.admin.writer.rawdata.route.payload.SingleMethodPayload
+import com.epam.drill.admin.writer.rawdata.table.BuildMethodTable
 import com.epam.drill.admin.writer.rawdata.table.BuildTable
 import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable
 import com.epam.drill.admin.writer.rawdata.table.InstanceTable
@@ -179,6 +180,7 @@ class ChangesApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageApiTest.kt

@@ -20,6 +20,7 @@ import com.epam.drill.admin.test.DatabaseTests
 import com.epam.drill.admin.test.withTransaction
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
 import com.epam.drill.admin.writer.rawdata.route.payload.SingleMethodPayload
+import com.epam.drill.admin.writer.rawdata.table.BuildMethodTable
 import com.epam.drill.admin.writer.rawdata.table.BuildTable
 import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable
 import com.epam.drill.admin.writer.rawdata.table.InstanceTable
@@ -114,6 +115,7 @@ class CoverageApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/CoverageTreemapTest.kt

@@ -75,6 +75,7 @@ class CoverageTreemapTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/DataIngestClient.kt

@@ -80,6 +80,7 @@ suspend fun HttpClient.launchTest(
                         it.first.params,
                         it.first.returnType
                     ).joinToString(":"),
+                    bodyChecksum = it.first.bodyChecksum,
                     testId = testLaunchId,
                     testSessionId = session.id,
                     probes = it.second.map { probe -> probe != 0 }.toBooleanArray()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/ImpactedMethodsApiTest.kt

@@ -20,6 +20,7 @@ import com.epam.drill.admin.test.DatabaseTests
 import com.epam.drill.admin.test.withTransaction
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
 import com.epam.drill.admin.writer.rawdata.route.payload.TestDetails
+import com.epam.drill.admin.writer.rawdata.table.BuildMethodTable
 import com.epam.drill.admin.writer.rawdata.table.BuildTable
 import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable
 import com.epam.drill.admin.writer.rawdata.table.InstanceTable
@@ -111,6 +112,7 @@ class ImpactedMethodsApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/ImpactedTestsApiTest.kt

@@ -20,6 +20,7 @@ import com.epam.drill.admin.test.DatabaseTests
 import com.epam.drill.admin.test.withTransaction
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
 import com.epam.drill.admin.writer.rawdata.route.payload.TestDetails
+import com.epam.drill.admin.writer.rawdata.table.BuildMethodTable
 import com.epam.drill.admin.writer.rawdata.table.BuildTable
 import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable
 import com.epam.drill.admin.writer.rawdata.table.InstanceTable
@@ -209,6 +210,7 @@ class ImpactedTestsApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/RecommendedTestsApiTest.kt

@@ -212,6 +212,7 @@ class RecommendedTestsApiTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-metrics/src/test/kotlin/com/epam/drill/admin/metrics/TestImpactAnalysisTest.kt

@@ -19,6 +19,7 @@ import com.epam.drill.admin.metrics.config.MetricsDatabaseConfig
 import com.epam.drill.admin.test.DatabaseTests
 import com.epam.drill.admin.test.withTransaction
 import com.epam.drill.admin.writer.rawdata.config.RawDataWriterDatabaseConfig
+import com.epam.drill.admin.writer.rawdata.table.BuildMethodTable
 import com.epam.drill.admin.writer.rawdata.table.BuildTable
 import com.epam.drill.admin.writer.rawdata.table.MethodCoverageTable
 import com.epam.drill.admin.writer.rawdata.table.InstanceTable
@@ -107,6 +108,7 @@ class TestImpactAnalysisTest : DatabaseTests({
         MethodCoverageTable.deleteAll()
         InstanceTable.deleteAll()
         MethodTable.deleteAll()
+        BuildMethodTable.deleteAll()
         BuildTable.deleteAll()
         TestLaunchTable.deleteAll()
         TestSessionTable.deleteAll()

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/Coverage.kt

@@ -20,7 +20,7 @@ class Coverage(
     val appId: String,
     val instanceId: String,
     val buildId: String,
-    val signature: String,
+    val methodId: String,
     val testId: String?,
     val testSessionId: String?,
     val probes: BooleanArray

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/Method.kt

@@ -18,8 +18,8 @@ package com.epam.drill.admin.writer.rawdata.entity
 class Method(
     val groupId: String,
     val appId: String,
-    val id: String,
     val buildId: String,
+    val methodId: String,
     val classname: String,
     val name: String,
     val params: String,
@@ -28,6 +28,4 @@ class Method(
     val probesStartPos: Int,
     val bodyChecksum: String,
     val signature: String,
-    val annotations: Map<String, List<String>>?,
-    val classAnnotations: Map<String, List<String>>?
 )

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/entity/MethodIgnoreRule.kt

@@ -24,8 +24,6 @@ class MethodIgnoreRule(
     val appId: String,
     val namePattern: String? = null,
     val classnamePattern: String? = null,
-    val annotationsPattern: String? = null,
-    val classAnnotationsPattern: String? = null
 ) {
     init {
         validate()
@@ -40,19 +38,14 @@ class MethodIgnoreRule(
             throw InvalidParameters("Field 'appId' is required and must contain non-empty string")
         }
 
-        if (namePattern.isNullOrEmpty() && classnamePattern.isNullOrEmpty() &&
-            annotationsPattern.isNullOrEmpty() && classAnnotationsPattern.isNullOrEmpty()) {
+        if (namePattern.isNullOrEmpty() && classnamePattern.isNullOrEmpty()) {
             throw InvalidParameters("You must specify at least one of the following fields containing valid regex: " +
                     "'namePattern', " +
-                    "'classnamePattern', " +
-                    "'annotationsPattern', " +
-                    "'classAnnotationsPattern'")
+                    "'classnamePattern'")
         }
 
         namePattern?.let { validateRegex(it, "namePattern") }
         classnamePattern?.let { validateRegex(it, "classnamePattern") }
-        annotationsPattern?.let { validateRegex(it, "annotationsPattern") }
-        classAnnotationsPattern?.let { validateRegex(it, "classAnnotationsPattern") }
     }
 
     private fun validateRegex(pattern: String, patternName: String) {

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/CoverageRepositoryImpl.kt

@@ -34,7 +34,7 @@ class CoverageRepositoryImpl : CoverageRepository {
             this[MethodCoverageTable.appId] = it.appId
             this[MethodCoverageTable.instanceId] = it.instanceId
             this[MethodCoverageTable.buildId] = it.buildId
-            this[MethodCoverageTable.signature] = it.signature
+            this[MethodCoverageTable.methodId] = it.methodId
             this[MethodCoverageTable.testId] = it.testId
             this[MethodCoverageTable.testSessionId] = it.testSessionId
             this[MethodCoverageTable.probes] = it.probes

admin-writer/src/main/kotlin/com/epam/drill/admin/writer/rawdata/repository/impl/MethodIgnoreRuleRepositoryImpl.kt

@@ -31,8 +31,6 @@ class MethodIgnoreRuleRepositoryImpl : MethodIgnoreRuleRepository {
             it[appId] = rule.appId
             it[namePattern] = rule.namePattern
             it[classnamePattern] = rule.classnamePattern
-            it[annotationsPattern] = rule.annotationsPattern
-            it[classAnnotationsPattern] = rule.classAnnotationsPattern
         }
     }
 
@@ -44,8 +42,6 @@ class MethodIgnoreRuleRepositoryImpl : MethodIgnoreRuleRepository {
                 appId = it[MethodIgnoreRulesTable.appId],
                 namePattern = it[MethodIgnoreRulesTable.namePattern],
                 classnamePattern = it[MethodIgnoreRulesTable.classnamePattern],
-                annotationsPattern = it[MethodIgnoreRulesTable.annotationsPattern],
-                classAnnotationsPattern = it[MethodIgnoreRulesTable.classAnnotationsPattern]
             )
         }
     }