[WIP] Task #252259 feat: Created new Job for - When the household profile Observation is submitted, participant's profile should be updated

Documentation/Docker-setup/config.env

@@ -29,4 +29,4 @@ ELEVATE_DATA_CONTAINER_NAME=elevate-data
 
 # Kafka-Topics
 PROJECT_TOPIC=sl-improvement-project-submission-dev
-METABASE_TOPIC=sl-improvement-metabase-dev
+METABASE_TOPIC=sl-improvement-metabase-dev

Documentation/Docker-setup/docker-compose.yml

@@ -129,7 +129,7 @@ services:
 
   # Elevate Data Service
   elevate-data:
-    image: shikshalokamqa/elevate-data:latest
+    image: shikshalokamqa/elevate-data:v3.0.2
     restart: always
     container_name: elevate-data
     ports:
@@ -178,4 +178,4 @@ volumes:
   elevate_data:
     driver: local
   kafka_data:
-    driver: local
+    driver: local

jobs-core/src/main/resources/base-config.conf

@@ -27,4 +27,4 @@ task {
   restart-strategy.attempts = 3
   restart-strategy.delay = 30000 # in milli-seconds
   consumer.parallelism = 1
-}
+}

jobs-core/src/main/scala/org/shikshalokam/job/BaseJobConfig.scala

@@ -38,6 +38,13 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl
     properties.setProperty("bootstrap.servers", kafkaBrokerServers)
     properties.setProperty("group.id", groupId)
     properties.setProperty(ConsumerConfig.ISOLATION_LEVEL_CONFIG, "read_committed")
+
+    // Add timeout configurations to handle connection issues better
+    // Default timeout is 30 seconds, increase to 60 seconds for better reliability
+    // properties.setProperty(ConsumerConfig.REQUEST_TIMEOUT_MS_CONFIG, "60000")
+    // properties.setProperty(ConsumerConfig.SESSION_TIMEOUT_MS_CONFIG, "30000")
+    // properties.setProperty(ConsumerConfig.METADATA_MAX_AGE_CONFIG, "300000")
+
     kafkaAutoOffsetReset.map {
       properties.setProperty("auto.offset.reset", _)
     }
@@ -50,6 +57,11 @@ class BaseJobConfig(val config: Config, val jobName: String) extends Serializabl
     properties.put(ProducerConfig.LINGER_MS_CONFIG, new Integer(10))
     properties.put(ProducerConfig.BATCH_SIZE_CONFIG, new Integer(16384 * 4))
     properties.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, "snappy")
+
+    // Add timeout configurations for producer
+    // properties.setProperty(ProducerConfig.REQUEST_TIMEOUT_MS_CONFIG, "60000")
+    // properties.setProperty(ProducerConfig.DELIVERY_TIMEOUT_MS_CONFIG, "120000")
+
     properties
   }
 

metabase-jobs/project-dashboard-creator/src/main/resources/metabase-project-dashboard.conf

@@ -32,4 +32,4 @@ metabase {
   domainName = "http://localhost:3000/dashboard/"
   metabaseApiKey = "d3f4ult-api-key"
   evidenceBaseUrl = "https://elevate-api.sunbirdsaas.com/project/v1/cloud-services/files/download?file="
-}
+}

metabase-jobs/project-dashboard-creator/src/test/resources/test.conf

@@ -33,4 +33,4 @@ metabase {
   domainName = "http://localhost:3000/dashboard/"
   metabaseApiKey = "d3f4ult-api-key"
   evidenceBaseUrl = "https://TESTING="
-}
+}

stream-jobs/pom.xml

@@ -20,6 +20,7 @@
         <module>survey-stream-processor</module>
         <module>observation-stream-processor</module>
         <module>user-stream-processor</module>
+        <module>user-mapping-stream-processor</module>
         <module>mentoring-stream-processor</module>
     </modules>
 

stream-jobs/project-stream-processor/src/main/resources/project-stream.conf

@@ -28,4 +28,4 @@ postgres{
 
 reports{
   enabled = ["admin", "state", "district", "program", "solution"]
-}
+}

stream-jobs/project-stream-processor/src/test/resources/test.conf

@@ -27,4 +27,4 @@ postgres{
 
 reports{
   enabled = ["admin", "state", "district", "program", "solution"]
-}
+}