From 7c7bf1ec1c4539c488472d4540a568372f9480ca Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 30 Jan 2025 19:03:40 +0100
Subject: [PATCH 01/41] fix process garbage function in schedule cron task +
 dockerfile + profiling

fix profiling

fix gitignore

fix dockerignore

adding profiling in trace allocator

fix package.json

fix logger

fix chmod

fix build dockerfile

adding publish new version to docker hub when pushing in master

update node version

fix dockerfile

fix dockerfile

fix dockerfile

fix dockerfile

adding dockerfile

fix npm run dev

adding perf output for functions graph

adding gitignore on profiling files

adding profiling in dev scripts

fix not retry in study when not needed in event policy

run at start processConsistencyAndGarbage

fix using ms library to convert string duration to ms

skip garbage collection in case of current sha1 file not present in remote or in local or are differents

set to debug some logs

fix garbage collectio in case of Local and remote folders for activity not synced for activity

fix garbage collector in case not synced between local and remote

fix index previous version

fix time in min and convert in in ms

fix cron time

fix garbage collect to remove and retry + function to convert time in minute to cron timeschedule

fix cron schedule

fix processConsistencyAndGarbage

fix

fix adding logs

adding logs

fix trace allocator in dev logs

adding logs gitignore

using nodemon to restart trace allocator when changes in file in debug mode

fix docker startup

fix docker startup in debug mode

Fix paths list list local and remote hashes

fix list remote hashes

fix execution of docker-startup file
---
 .dockerignore                              |   5 +
 .github/workflows/docker-publish.yml       |  96 ++++
 .gitignore                                 |   3 +
 Dockerfile                                 |  29 +
 README.md                                  |   0
 docker-startup.sh                          |   9 +-
 jsconfig.json                              |   0
 logs/.gitignore                            |   4 +
 manual-scripts/compare-trace-one-folder.sh |   0
 manual-scripts/compare-trace.sh            |   0
 manual-scripts/concat-json-one-folder.sh   |   0
 manual-scripts/concat-json-parallel.sh     |   0
 manual-scripts/concat-json.sh              |   0
 manual-scripts/convert-json.sh             |   0
 package-lock.json                          | 639 ++++++++++++++-------
 package.json                               |  12 +-
 src/compactor.js                           |  36 +-
 src/config.js                              |   9 +-
 src/index.js                               |  93 ++-
 src/kafka.js                               |   0
 src/logger.js                              |  25 +-
 src/minio.js                               |   0
 src/profiling.js                           |  54 ++
 src/prueba.js                              |   0
 src/simva.js                               |   0
 src/state.js                               | 140 ++++-
 src/utils/array.js                         |  35 ++
 src/utils/date.js                          |  58 ++
 src/utils/errors.js                        |   0
 src/utils/file.js                          |   2 +-
 src/utils/misc.js                          |   0
 31 files changed, 932 insertions(+), 317 deletions(-)
 create mode 100755 .dockerignore
 create mode 100755 .github/workflows/docker-publish.yml
 mode change 100644 => 100755 .gitignore
 create mode 100755 Dockerfile
 mode change 100644 => 100755 README.md
 mode change 100644 => 100755 docker-startup.sh
 mode change 100644 => 100755 jsconfig.json
 create mode 100755 logs/.gitignore
 mode change 100644 => 100755 manual-scripts/compare-trace-one-folder.sh
 mode change 100644 => 100755 manual-scripts/compare-trace.sh
 mode change 100644 => 100755 manual-scripts/concat-json-one-folder.sh
 mode change 100644 => 100755 manual-scripts/concat-json-parallel.sh
 mode change 100644 => 100755 manual-scripts/concat-json.sh
 mode change 100644 => 100755 manual-scripts/convert-json.sh
 mode change 100644 => 100755 package-lock.json
 mode change 100644 => 100755 package.json
 mode change 100644 => 100755 src/compactor.js
 mode change 100644 => 100755 src/config.js
 mode change 100644 => 100755 src/index.js
 mode change 100644 => 100755 src/kafka.js
 mode change 100644 => 100755 src/logger.js
 mode change 100644 => 100755 src/minio.js
 create mode 100755 src/profiling.js
 mode change 100644 => 100755 src/prueba.js
 mode change 100644 => 100755 src/simva.js
 mode change 100644 => 100755 src/state.js
 create mode 100755 src/utils/array.js
 mode change 100644 => 100755 src/utils/date.js
 mode change 100644 => 100755 src/utils/errors.js
 mode change 100644 => 100755 src/utils/file.js
 mode change 100644 => 100755 src/utils/misc.js

diff --git a/.dockerignore b/.dockerignore
new file mode 100755
index 0000000..d3a2e64
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,5 @@
+node_modules/
+manual-scripts/
+.github
+Dockerfile
+.dockerignore
\ No newline at end of file
diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
new file mode 100755
index 0000000..04b9751
--- /dev/null
+++ b/.github/workflows/docker-publish.yml
@@ -0,0 +1,96 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  push:
+    branches: [ "master" ]
+    # Publish semver tags as releases.
+    tags: [ 'v*.*.*' ]
+  pull_request:
+    branches: [ "master" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: docker.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
+        with:
+          cosign-release: 'v2.2.4'
+
+      # Set up BuildKit Docker container builder to be able to build
+      # multi-platform images and export cache
+      # https://github.com/docker/setup-buildx-action
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Sign the resulting Docker image digest except on PRs.
+      # This will only write to the public Rekor transparency log when the Docker
+      # repository is public to avoid leaking data.  If you would like to publish
+      # transparency data even for private images, pass --force to cosign below.
+      # https://github.com/sigstore/cosign
+      - name: Sign the published Docker image
+        if: ${{ github.event_name != 'pull_request' }}
+        env:
+          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build-and-push.outputs.digest }}
+        # This step uses the identity token to provision an ephemeral certificate
+        # against the sigstore community Fulcio instance.
+        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
\ No newline at end of file
diff --git a/.gitignore b/.gitignore
old mode 100644
new mode 100755
index dcaac7f..fc0146e
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,6 @@
 
 # Config files
 config-test.js
+*.log
+*.heapsnapshot
+*.crt
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
new file mode 100755
index 0000000..3d390e8
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,29 @@
+FROM node:20.18.3-bullseye
+
+# Install ca-certificates and update them
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ca-certificates && \
+    update-ca-certificates && \
+    apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app
+
+# Set the working directory
+WORKDIR /home/node/app
+
+COPY --chown=node:node package*.json ./
+
+USER node
+
+RUN npm install
+
+# Copy the current directory contents into the container at /app
+COPY --chown=node:node . .
+
+RUN mkdir -p /home/node/logs 
+
+RUN chown -R node:node /home/node/logs
+
+
+
+CMD [ "/home/node/app/docker-startup.sh" ]
\ No newline at end of file
diff --git a/README.md b/README.md
old mode 100644
new mode 100755
diff --git a/docker-startup.sh b/docker-startup.sh
old mode 100644
new mode 100755
index 37a6e80..9dbf0a1
--- a/docker-startup.sh
+++ b/docker-startup.sh
@@ -6,4 +6,11 @@ if [[ ! -d "node_modules" ]]; then
   npm install
 fi
 
-npm start
\ No newline at end of file
+#start trace allocator
+echo "${NODE_ENV:-production}"
+if [[ "${NODE_ENV:-production}" == "development" ]]; then
+  npm run dev
+  #perf record -e cycles:u -g -- npm run dev > perf.out
+else
+  npm start
+fi
\ No newline at end of file
diff --git a/jsconfig.json b/jsconfig.json
old mode 100644
new mode 100755
diff --git a/logs/.gitignore b/logs/.gitignore
new file mode 100755
index 0000000..2191032
--- /dev/null
+++ b/logs/.gitignore
@@ -0,0 +1,4 @@
+# Ignore all files in this dir...
+*
+# ... except for this one.
+!.gitignore
\ No newline at end of file
diff --git a/manual-scripts/compare-trace-one-folder.sh b/manual-scripts/compare-trace-one-folder.sh
old mode 100644
new mode 100755
diff --git a/manual-scripts/compare-trace.sh b/manual-scripts/compare-trace.sh
old mode 100644
new mode 100755
diff --git a/manual-scripts/concat-json-one-folder.sh b/manual-scripts/concat-json-one-folder.sh
old mode 100644
new mode 100755
diff --git a/manual-scripts/concat-json-parallel.sh b/manual-scripts/concat-json-parallel.sh
old mode 100644
new mode 100755
diff --git a/manual-scripts/concat-json.sh b/manual-scripts/concat-json.sh
old mode 100644
new mode 100755
diff --git a/manual-scripts/convert-json.sh b/manual-scripts/convert-json.sh
old mode 100644
new mode 100755
diff --git a/package-lock.json b/package-lock.json
old mode 100644
new mode 100755
index e88ec2d..edb1d8d
--- a/package-lock.json
+++ b/package-lock.json
@@ -11,13 +11,16 @@
         "dotenv": "^16.3.1",
         "kafkajs": "^2.2.4",
         "minio": "8.0.1",
-        "pino": "^8.16.2",
+        "ms": "^2.1.3",
+        "node-cron": "3.0.3",
+        "pino": "^9.6.0",
         "tinyduration": "^3.3.0",
         "wretch": "^2.7.0"
       },
       "devDependencies": {
         "@types/node": "^20.9.0",
-        "pino-pretty": "^10.2.3"
+        "nodemon": "3.1.9",
+        "pino-pretty": "^13.0.0"
       },
       "engines": {
         "node": ">=20 <21"
@@ -44,15 +47,18 @@
       "license": "(Unlicense OR Apache-2.0)",
       "optional": true
     },
-    "node_modules/abort-controller": {
-      "version": "3.0.0",
-      "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz",
-      "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==",
+    "node_modules/anymatch": {
+      "version": "3.1.3",
+      "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz",
+      "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==",
+      "dev": true,
+      "license": "ISC",
       "dependencies": {
-        "event-target-shim": "^5.0.0"
+        "normalize-path": "^3.0.0",
+        "picomatch": "^2.0.4"
       },
       "engines": {
-        "node": ">=6.5"
+        "node": ">= 8"
       }
     },
     "node_modules/async": {
@@ -89,24 +95,18 @@
       "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==",
       "dev": true
     },
-    "node_modules/base64-js": {
-      "version": "1.5.1",
-      "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
-      "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
+    "node_modules/binary-extensions": {
+      "version": "2.3.0",
+      "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz",
+      "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/sindresorhus"
+      }
     },
     "node_modules/block-stream2": {
       "version": "2.1.0",
@@ -116,13 +116,17 @@
         "readable-stream": "^3.4.0"
       }
     },
-    "node_modules/brace-expansion": {
-      "version": "2.0.1",
-      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
-      "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
+    "node_modules/braces": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
+      "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
-        "balanced-match": "^1.0.0"
+        "fill-range": "^7.1.1"
+      },
+      "engines": {
+        "node": ">=8"
       }
     },
     "node_modules/browser-or-node": {
@@ -131,29 +135,6 @@
       "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==",
       "license": "MIT"
     },
-    "node_modules/buffer": {
-      "version": "6.0.3",
-      "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz",
-      "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ],
-      "dependencies": {
-        "base64-js": "^1.3.1",
-        "ieee754": "^1.2.1"
-      }
-    },
     "node_modules/buffer-crc32": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz",
@@ -182,12 +163,44 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/chokidar": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz",
+      "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "anymatch": "~3.1.2",
+        "braces": "~3.0.2",
+        "glob-parent": "~5.1.2",
+        "is-binary-path": "~2.1.0",
+        "is-glob": "~4.0.1",
+        "normalize-path": "~3.0.0",
+        "readdirp": "~3.6.0"
+      },
+      "engines": {
+        "node": ">= 8.10.0"
+      },
+      "funding": {
+        "url": "https://paulmillr.com/funding/"
+      },
+      "optionalDependencies": {
+        "fsevents": "~2.3.2"
+      }
+    },
     "node_modules/colorette": {
       "version": "2.0.20",
       "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz",
       "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==",
       "dev": true
     },
+    "node_modules/concat-map": {
+      "version": "0.0.1",
+      "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
+      "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/dateformat": {
       "version": "4.6.3",
       "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz",
@@ -197,6 +210,24 @@
         "node": "*"
       }
     },
+    "node_modules/debug": {
+      "version": "4.4.0",
+      "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz",
+      "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "ms": "^2.1.3"
+      },
+      "engines": {
+        "node": ">=6.0"
+      },
+      "peerDependenciesMeta": {
+        "supports-color": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/decode-uri-component": {
       "version": "0.2.2",
       "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz",
@@ -264,33 +295,18 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/event-target-shim": {
-      "version": "5.0.1",
-      "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz",
-      "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==",
-      "engines": {
-        "node": ">=6"
-      }
-    },
     "node_modules/eventemitter3": {
       "version": "5.0.1",
       "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz",
       "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==",
       "license": "MIT"
     },
-    "node_modules/events": {
-      "version": "3.3.0",
-      "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz",
-      "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==",
-      "engines": {
-        "node": ">=0.8.x"
-      }
-    },
     "node_modules/fast-copy": {
-      "version": "3.0.1",
-      "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.1.tgz",
-      "integrity": "sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==",
-      "dev": true
+      "version": "3.0.2",
+      "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz",
+      "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/fast-redact": {
       "version": "3.3.0",
@@ -328,6 +344,19 @@
         "fxparser": "src/cli/cli.js"
       }
     },
+    "node_modules/fill-range": {
+      "version": "7.1.1",
+      "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
+      "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "to-regex-range": "^5.0.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/filter-obj": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz",
@@ -346,11 +375,20 @@
         "is-callable": "^1.1.3"
       }
     },
-    "node_modules/fs.realpath": {
-      "version": "1.0.0",
-      "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
-      "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==",
-      "dev": true
+    "node_modules/fsevents": {
+      "version": "2.3.3",
+      "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz",
+      "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==",
+      "dev": true,
+      "hasInstallScript": true,
+      "license": "MIT",
+      "optional": true,
+      "os": [
+        "darwin"
+      ],
+      "engines": {
+        "node": "^8.16.0 || ^10.6.0 || >=11.0.0"
+      }
     },
     "node_modules/function-bind": {
       "version": "1.1.2",
@@ -380,23 +418,17 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
-    "node_modules/glob": {
-      "version": "8.1.0",
-      "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
-      "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
+    "node_modules/glob-parent": {
+      "version": "5.1.2",
+      "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz",
+      "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==",
       "dev": true,
+      "license": "ISC",
       "dependencies": {
-        "fs.realpath": "^1.0.0",
-        "inflight": "^1.0.4",
-        "inherits": "2",
-        "minimatch": "^5.0.1",
-        "once": "^1.3.0"
+        "is-glob": "^4.0.1"
       },
       "engines": {
-        "node": ">=12"
-      },
-      "funding": {
-        "url": "https://github.com/sponsors/isaacs"
+        "node": ">= 6"
       }
     },
     "node_modules/gopd": {
@@ -411,6 +443,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/has-flag": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz",
+      "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/has-property-descriptors": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz",
@@ -475,43 +517,18 @@
       }
     },
     "node_modules/help-me": {
-      "version": "4.2.0",
-      "resolved": "https://registry.npmjs.org/help-me/-/help-me-4.2.0.tgz",
-      "integrity": "sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA==",
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz",
+      "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==",
       "dev": true,
-      "dependencies": {
-        "glob": "^8.0.0",
-        "readable-stream": "^3.6.0"
-      }
-    },
-    "node_modules/ieee754": {
-      "version": "1.2.1",
-      "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
-      "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
-      "funding": [
-        {
-          "type": "github",
-          "url": "https://github.com/sponsors/feross"
-        },
-        {
-          "type": "patreon",
-          "url": "https://www.patreon.com/feross"
-        },
-        {
-          "type": "consulting",
-          "url": "https://feross.org/support"
-        }
-      ]
+      "license": "MIT"
     },
-    "node_modules/inflight": {
-      "version": "1.0.6",
-      "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
-      "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==",
+    "node_modules/ignore-by-default": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz",
+      "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==",
       "dev": true,
-      "dependencies": {
-        "once": "^1.3.0",
-        "wrappy": "1"
-      }
+      "license": "ISC"
     },
     "node_modules/inherits": {
       "version": "2.0.4",
@@ -543,6 +560,19 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-binary-path": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz",
+      "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "binary-extensions": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
     "node_modules/is-callable": {
       "version": "1.2.7",
       "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
@@ -555,6 +585,16 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-extglob": {
+      "version": "2.1.1",
+      "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz",
+      "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-generator-function": {
       "version": "1.0.10",
       "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz",
@@ -570,6 +610,29 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/is-glob": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz",
+      "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "is-extglob": "^2.1.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/is-number": {
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz",
+      "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.12.0"
+      }
+    },
     "node_modules/is-typed-array": {
       "version": "1.1.13",
       "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz",
@@ -627,18 +690,6 @@
         "node": ">= 0.6"
       }
     },
-    "node_modules/minimatch": {
-      "version": "5.1.6",
-      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
-      "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
-      "dev": true,
-      "dependencies": {
-        "brace-expansion": "^2.0.1"
-      },
-      "engines": {
-        "node": ">=10"
-      }
-    },
     "node_modules/minimist": {
       "version": "1.2.8",
       "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
@@ -673,6 +724,87 @@
         "node": "^16 || ^18 || >=20"
       }
     },
+    "node_modules/ms": {
+      "version": "2.1.3",
+      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
+      "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
+      "license": "MIT"
+    },
+    "node_modules/node-cron": {
+      "version": "3.0.3",
+      "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz",
+      "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==",
+      "license": "ISC",
+      "dependencies": {
+        "uuid": "8.3.2"
+      },
+      "engines": {
+        "node": ">=6.0.0"
+      }
+    },
+    "node_modules/nodemon": {
+      "version": "3.1.9",
+      "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.9.tgz",
+      "integrity": "sha512-hdr1oIb2p6ZSxu3PB2JWWYS7ZQ0qvaZsc3hK8DR8f02kRzc8rjYmxAIvdz+aYC+8F2IjNaB7HMcSDg8nQpJxyg==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "chokidar": "^3.5.2",
+        "debug": "^4",
+        "ignore-by-default": "^1.0.1",
+        "minimatch": "^3.1.2",
+        "pstree.remy": "^1.1.8",
+        "semver": "^7.5.3",
+        "simple-update-notifier": "^2.0.0",
+        "supports-color": "^5.5.0",
+        "touch": "^3.1.0",
+        "undefsafe": "^2.0.5"
+      },
+      "bin": {
+        "nodemon": "bin/nodemon.js"
+      },
+      "engines": {
+        "node": ">=10"
+      },
+      "funding": {
+        "type": "opencollective",
+        "url": "https://opencollective.com/nodemon"
+      }
+    },
+    "node_modules/nodemon/node_modules/brace-expansion": {
+      "version": "1.1.11",
+      "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz",
+      "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "balanced-match": "^1.0.0",
+        "concat-map": "0.0.1"
+      }
+    },
+    "node_modules/nodemon/node_modules/minimatch": {
+      "version": "3.1.2",
+      "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
+      "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==",
+      "dev": true,
+      "license": "ISC",
+      "dependencies": {
+        "brace-expansion": "^1.1.7"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/normalize-path": {
+      "version": "3.0.0",
+      "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz",
+      "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/on-exit-leak-free": {
       "version": "2.1.2",
       "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz",
@@ -690,96 +822,80 @@
         "wrappy": "1"
       }
     },
+    "node_modules/picomatch": {
+      "version": "2.3.1",
+      "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz",
+      "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==",
+      "dev": true,
+      "license": "MIT",
+      "engines": {
+        "node": ">=8.6"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/jonschlinkert"
+      }
+    },
     "node_modules/pino": {
-      "version": "8.16.2",
-      "resolved": "https://registry.npmjs.org/pino/-/pino-8.16.2.tgz",
-      "integrity": "sha512-2advCDGVEvkKu9TTVSa/kWW7Z3htI/sBKEZpqiHk6ive0i/7f5b1rsU8jn0aimxqfnSz5bj/nOYkwhBUn5xxvg==",
+      "version": "9.6.0",
+      "resolved": "https://registry.npmjs.org/pino/-/pino-9.6.0.tgz",
+      "integrity": "sha512-i85pKRCt4qMjZ1+L7sy2Ag4t1atFcdbEt76+7iRJn1g2BvsnRMGu9p8pivl9fs63M2kF/A0OacFZhTub+m/qMg==",
+      "license": "MIT",
       "dependencies": {
         "atomic-sleep": "^1.0.0",
         "fast-redact": "^3.1.1",
         "on-exit-leak-free": "^2.1.0",
-        "pino-abstract-transport": "v1.1.0",
-        "pino-std-serializers": "^6.0.0",
-        "process-warning": "^2.0.0",
+        "pino-abstract-transport": "^2.0.0",
+        "pino-std-serializers": "^7.0.0",
+        "process-warning": "^4.0.0",
         "quick-format-unescaped": "^4.0.3",
         "real-require": "^0.2.0",
         "safe-stable-stringify": "^2.3.1",
-        "sonic-boom": "^3.7.0",
-        "thread-stream": "^2.0.0"
+        "sonic-boom": "^4.0.1",
+        "thread-stream": "^3.0.0"
       },
       "bin": {
         "pino": "bin.js"
       }
     },
     "node_modules/pino-abstract-transport": {
-      "version": "1.1.0",
-      "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz",
-      "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==",
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz",
+      "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==",
+      "license": "MIT",
       "dependencies": {
-        "readable-stream": "^4.0.0",
         "split2": "^4.0.0"
       }
     },
-    "node_modules/pino-abstract-transport/node_modules/readable-stream": {
-      "version": "4.4.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz",
-      "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==",
-      "dependencies": {
-        "abort-controller": "^3.0.0",
-        "buffer": "^6.0.3",
-        "events": "^3.3.0",
-        "process": "^0.11.10",
-        "string_decoder": "^1.3.0"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      }
-    },
     "node_modules/pino-pretty": {
-      "version": "10.2.3",
-      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.3.tgz",
-      "integrity": "sha512-4jfIUc8TC1GPUfDyMSlW1STeORqkoxec71yhxIpLDQapUu8WOuoz2TTCoidrIssyz78LZC69whBMPIKCMbi3cw==",
+      "version": "13.0.0",
+      "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.0.0.tgz",
+      "integrity": "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==",
       "dev": true,
+      "license": "MIT",
       "dependencies": {
         "colorette": "^2.0.7",
         "dateformat": "^4.6.3",
-        "fast-copy": "^3.0.0",
+        "fast-copy": "^3.0.2",
         "fast-safe-stringify": "^2.1.1",
-        "help-me": "^4.0.1",
+        "help-me": "^5.0.0",
         "joycon": "^3.1.1",
         "minimist": "^1.2.6",
         "on-exit-leak-free": "^2.1.0",
-        "pino-abstract-transport": "^1.0.0",
+        "pino-abstract-transport": "^2.0.0",
         "pump": "^3.0.0",
-        "readable-stream": "^4.0.0",
         "secure-json-parse": "^2.4.0",
-        "sonic-boom": "^3.0.0",
+        "sonic-boom": "^4.0.1",
         "strip-json-comments": "^3.1.1"
       },
       "bin": {
         "pino-pretty": "bin.js"
       }
     },
-    "node_modules/pino-pretty/node_modules/readable-stream": {
-      "version": "4.4.2",
-      "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz",
-      "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==",
-      "dev": true,
-      "dependencies": {
-        "abort-controller": "^3.0.0",
-        "buffer": "^6.0.3",
-        "events": "^3.3.0",
-        "process": "^0.11.10",
-        "string_decoder": "^1.3.0"
-      },
-      "engines": {
-        "node": "^12.22.0 || ^14.17.0 || >=16.0.0"
-      }
-    },
     "node_modules/pino-std-serializers": {
-      "version": "6.2.2",
-      "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz",
-      "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA=="
+      "version": "7.0.0",
+      "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz",
+      "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==",
+      "license": "MIT"
     },
     "node_modules/possible-typed-array-names": {
       "version": "1.0.0",
@@ -790,18 +906,28 @@
         "node": ">= 0.4"
       }
     },
-    "node_modules/process": {
-      "version": "0.11.10",
-      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
-      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
-      "engines": {
-        "node": ">= 0.6.0"
-      }
-    },
     "node_modules/process-warning": {
-      "version": "2.3.0",
-      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.3.0.tgz",
-      "integrity": "sha512-N6mp1+2jpQr3oCFMz6SeHRGbv6Slb20bRhj4v3xR99HqNToAcOe1MFOp4tytyzOfJn+QtN8Rf7U/h2KAn4kC6g=="
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.1.tgz",
+      "integrity": "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==",
+      "funding": [
+        {
+          "type": "github",
+          "url": "https://github.com/sponsors/fastify"
+        },
+        {
+          "type": "opencollective",
+          "url": "https://opencollective.com/fastify"
+        }
+      ],
+      "license": "MIT"
+    },
+    "node_modules/pstree.remy": {
+      "version": "1.1.8",
+      "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz",
+      "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==",
+      "dev": true,
+      "license": "MIT"
     },
     "node_modules/pump": {
       "version": "3.0.0",
@@ -849,10 +975,24 @@
         "node": ">= 6"
       }
     },
+    "node_modules/readdirp": {
+      "version": "3.6.0",
+      "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz",
+      "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "picomatch": "^2.2.1"
+      },
+      "engines": {
+        "node": ">=8.10.0"
+      }
+    },
     "node_modules/real-require": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz",
       "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==",
+      "license": "MIT",
       "engines": {
         "node": ">= 12.13.0"
       }
@@ -896,6 +1036,19 @@
       "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==",
       "dev": true
     },
+    "node_modules/semver": {
+      "version": "7.7.1",
+      "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz",
+      "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==",
+      "dev": true,
+      "license": "ISC",
+      "bin": {
+        "semver": "bin/semver.js"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/set-function-length": {
       "version": "1.2.2",
       "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz",
@@ -913,10 +1066,24 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/simple-update-notifier": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz",
+      "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "semver": "^7.5.3"
+      },
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/sonic-boom": {
-      "version": "3.7.0",
-      "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.7.0.tgz",
-      "integrity": "sha512-IudtNvSqA/ObjN97tfgNmOKyDOs4dNcg4cUUsHDebqsgb8wGBBwb31LIgShNO8fye0dFI52X1+tFoKKI6Rq1Gg==",
+      "version": "4.2.0",
+      "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz",
+      "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==",
+      "license": "MIT",
       "dependencies": {
         "atomic-sleep": "^1.0.0"
       }
@@ -934,6 +1101,7 @@
       "version": "4.2.0",
       "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz",
       "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==",
+      "license": "ISC",
       "engines": {
         "node": ">= 10.x"
       }
@@ -988,10 +1156,24 @@
       "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==",
       "license": "MIT"
     },
+    "node_modules/supports-color": {
+      "version": "5.5.0",
+      "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz",
+      "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "has-flag": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/thread-stream": {
-      "version": "2.4.1",
-      "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz",
-      "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==",
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz",
+      "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==",
+      "license": "MIT",
       "dependencies": {
         "real-require": "^0.2.0"
       }
@@ -1010,6 +1192,36 @@
       "resolved": "https://registry.npmjs.org/tinyduration/-/tinyduration-3.3.0.tgz",
       "integrity": "sha512-sLR0iVUnnnyGEX/a3jhTA0QMK7UvakBqQJFLiibiuEYL6U1L85W+qApTZj6DcL1uoWQntYuL0gExoe9NU5B3PA=="
     },
+    "node_modules/to-regex-range": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
+      "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==",
+      "dev": true,
+      "license": "MIT",
+      "dependencies": {
+        "is-number": "^7.0.0"
+      },
+      "engines": {
+        "node": ">=8.0"
+      }
+    },
+    "node_modules/touch": {
+      "version": "3.1.1",
+      "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz",
+      "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==",
+      "dev": true,
+      "license": "ISC",
+      "bin": {
+        "nodetouch": "bin/nodetouch.js"
+      }
+    },
+    "node_modules/undefsafe": {
+      "version": "2.0.5",
+      "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz",
+      "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/undici-types": {
       "version": "5.26.5",
       "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz",
@@ -1034,6 +1246,15 @@
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
       "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="
     },
+    "node_modules/uuid": {
+      "version": "8.3.2",
+      "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
+      "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
+      "license": "MIT",
+      "bin": {
+        "uuid": "dist/bin/uuid"
+      }
+    },
     "node_modules/web-encoding": {
       "version": "1.1.5",
       "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz",
diff --git a/package.json b/package.json
old mode 100644
new mode 100755
index 343e77e..7dee975
--- a/package.json
+++ b/package.json
@@ -5,7 +5,8 @@
   "type": "module",
   "scripts": {
     "start": "node ./src/index.js",
-    "start:debug": "node --nolazy --inspect-brk=0.0.0.0:9229 ./src/index.js"
+    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --prof ./src/index.js",
+    "start:debug": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof ./src/index.js"
   },
   "engines": {
     "node": ">=20 <21"
@@ -14,12 +15,15 @@
     "dotenv": "^16.3.1",
     "minio": "8.0.1",
     "kafkajs":"^2.2.4",
-    "pino": "^8.16.2",
+    "pino": "^9.6.0",
     "tinyduration": "^3.3.0",
-    "wretch": "^2.7.0"
+    "wretch": "^2.7.0",
+    "node-cron": "3.0.3",
+    "ms": "^2.1.3"
   },
   "devDependencies": {
     "@types/node": "^20.9.0",
-    "pino-pretty": "^10.2.3"
+    "pino-pretty": "^13.0.0",
+    "nodemon":"3.1.9"
   }
 }
diff --git a/src/compactor.js b/src/compactor.js
old mode 100644
new mode 100755
index b02ffdf..c03f792
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -6,6 +6,7 @@ import { KafkaClient } from './kafka.js';
 import { getState } from './state.js';
 import { createHash } from 'node:crypto';
 import { binarySearch, diffArray } from './utils/misc.js';
+import { config } from './config.js';
 
 /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */
 /** @typedef {import('./simva.js').Activity} Activity */
@@ -143,7 +144,7 @@ export class Compactor {
 
         logger.info(`Known %d activities, received %d`, state.size, activities.length);
 
-        await this.#garbageCollectActivities(state, activities);
+        let activitiesToGo = await this.#garbageCollectActivities(state, activities);
 
         this.status.total = activities.length;
         for(let idx=0; idx < activities.length; idx++) {
@@ -160,13 +161,16 @@ export class Compactor {
                 logger.info(`New activity: %s`, activity._id);
                 activityState = await state.create(activity._id);
             }
-
-            const updated = await this.#updateActivityTraces(activityState);
-            if (!updated) continue;
-            await this.#distributeTrace(activityState);
-
-            if (activities.length % 5) {
-                await state.save();
+            if((config.concatEventPolicy === "true" && activitiesToGo.includes(activity._id)) || ! (config.concatEventPolicy === "true")) {
+                const updated = await this.#updateActivityTraces(activityState);
+                if (!updated) continue;
+                await this.#distributeTrace(activityState);
+    
+                if (activities.length % 5) {
+                    await state.save();
+                }
+            } else {
+                logger.debug('Not needed to process activity: %s because of the Concat Event Policy', activity._id);
             }
         }
         await state.save();
@@ -202,10 +206,13 @@ export class Compactor {
                     logger.error(error);
                 }
             }
+            logger.info('Activities to removed OK.');
         }
-
+        logger.info('Starting collecting state garbage.');
         // Garbage collect state files in activities
-        await state.garbageCollect();
+        const activityToPass = await state.garbageCollect();
+        logger.info('Collecting state garbage finished.');
+        return activityToPass;
     }
 
     /**
@@ -256,6 +263,13 @@ export class Compactor {
         logger.info(`Copied compacted file for activity %s`, activityState.activityId);
     }
 
+    async processConsistencyAndGarbage() {
+        logger.info('Check consistency');
+        await this.#checkConsistency();
+        logger.info('Start compaction');
+        await this.#compactActivities();
+    }
+
     // Method to process messages (acts as the callback for KafkaClient)
     /**
      * @param {any} message
@@ -267,8 +281,6 @@ export class Compactor {
             logger.info(message.value);
 
             let state = await getState(this.#opts, this.#minio);
-            let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] });
-            await this.#garbageCollectActivities(state, activities);
             // Set up the delimiter and the required bucket and path values
             let delimiter = '/';
             let bucket = this.#opts.minio.bucket;
diff --git a/src/config.js b/src/config.js
old mode 100644
new mode 100755
index 006f6ff..061277f
--- a/src/config.js
+++ b/src/config.js
@@ -1,5 +1,6 @@
 import 'dotenv/config';
 import * as inspector from 'inspector';
+import ms from "ms";
 
 /**
  * @typedef CompactorOptions
@@ -22,12 +23,12 @@ import * as inspector from 'inspector';
 export const config = {
     concatEventPolicy: process.env.CONCAT_EVENT_POLICY || 'true', // if true minio-events else previous version of trace allocator
     batchSize: process.env.BATCH_SIZE !== undefined ? parseInt(process.env.BATCH_SIZE) : 500,
-    maxDelay: process.env.MAX_DELAY !== undefined ? parseInt(process.env.MAX_DELAY) : 5*60*1000,
-    refreshInterval: process.env.REFRESH_INTERVAL !== undefined ? parseInt(process.env.REFRESH_INTERVAL) : 10*60*1000,
+    maxDelay: process.env.MAX_DELAY !== undefined ? ms(process.env.MAX_DELAY) : ms("5min"),
+    refreshInterval: process.env.REFRESH_INTERVAL !== undefined ? ms(process.env.REFRESH_INTERVAL) : ms("10min"),
     localStatePath: process.env.LOCAL_STATE || new URL('../state', import.meta.url).pathname,
     remoteStatePath: process.env.REMOTE_STATE || 'state',
     removeDryRun: process.env.REMOVE_DRY_RUN !== undefined ? (process.env.REMOVE_DRY_RUN.toLocaleLowerCase() === 'false' ? false : true) : true,
-    gcInterval: process.env.GC_INTERVAL !== undefined ? parseInt(process.env.GC_INTERVAL) : 10*24*60*60*1000,
+    gcInterval: process.env.GC_INTERVAL !== undefined ? ms(process.env.GC_INTERVAL) : ms("2h"),
     copyInsteadRename: process.env.COPY_INSTEAD_RENAME !== undefined ? (process.env.COPY_INSTEAD_RENAME.toLocaleLowerCase() === 'false' ? false : true) : true,
     tryRecovery: process.env.TRY_RECOVERY !== undefined ? (process.env.TRY_RECOVERY.toLocaleLowerCase() === 'false' ? false : true) : false,
     minio: {
@@ -51,7 +52,7 @@ export const config = {
     },
     kafka: {
         clientId: process.env.SIMVA_KAFKA_CLIENTID || 'my-client-id',
-        brokers: [ process.env.SIMVA_KAFKA_BROKER ] || ['localhost:9092'],
+        brokers: process.env.SIMVA_KAFKA_BROKER !== undefined ?  [ process.env.SIMVA_KAFKA_BROKER ] : ['localhost:9092'],
         groupId: process.env.SIMVA_KAFKA_GROUPID || 'my-group-id',
         topic: process.env.SIMVA_KAFKA_MINIO_TOPIC || 'minio-events'
     }
diff --git a/src/index.js b/src/index.js
old mode 100644
new mode 100755
index 5f19639..0b0520a
--- a/src/index.js
+++ b/src/index.js
@@ -1,9 +1,10 @@
 import { config, isInDebugMode } from './config.js';
 import { logger } from './logger.js';
-
+import cron from 'node-cron';
 import { Compactor } from './compactor.js';
 import { getState } from './state.js';
-
+import { convertTimeToCron } from "./utils/date.js";
+import "./profiling.js";
 logger.debug('Current config: %o', config);
 
 const compactor = new Compactor(config);
@@ -20,19 +21,15 @@ if (config.concatEventPolicy === "true") {
 }
 
 async function startKafkaProcess() {
-    let state = await getState(compactor.getOpts(), compactor.getMinioClient());
-
-    if(state.size === 0) {
-        try {
-            // Run the compact function immediately at launch
-            logger.info('Running compactor at launch...');
-            await run();
-            logger.info('Compactor initialized.');
-        } catch (error) {
-            logger.error('Error during compactor initialization:', error);
-        }
-    }
-
+    await compactor.processConsistencyAndGarbage();
+    // Schedule a task to run every x
+    let gcIntervalInMin=Math.round(config.gcInterval/(1000*60));
+    const cronTime = convertTimeToCron(gcIntervalInMin);
+    logger.info(cronTime);
+    cron.schedule(cronTime, async () => {
+        logger.info("Compactor starting process garbage scheduled task running every x minutes");
+        await compactor.processConsistencyAndGarbage();
+    });
     // Start consuming messages
     (async () => {
         await compactor.startKafkaConsumer();
@@ -56,35 +53,37 @@ async function startPrevVersionProcess() {
     } catch (error) {
         logger.error('Error during compactor initialization:', error);
     }
-}
-process.on('SIGTERM', () => {
-    logger.info('SIGTERM signal received: terminating');
-    if (intervalId !== undefined) {
-        clearInterval(intervalId);
-    }
-    compactor.shouldExit = true;
-});
-
-process.on('SIGINT', () => {
-    logger.info('SIGINT signal received');
-    if (intervalId !== undefined) {
-        clearInterval(intervalId);
-    }
-    compactor.shouldExit = true;
-    setTimeout(() => {
-        logger.info('Exiting');
-        process.exit();        
-    }, MAX_WAIT_TIME_ON_EXIT);
-});
-
-
-process.on('SIGUSR2',function(){
-    logger.info("SIGUSR2 signal received");
-    if (!compactor.status.processing) {
-        setImmediate(run);
-        logger.info(`Force run`);
-    }
-    const status = compactor.status;
-    const elapsedTime = compactor.elapsedTime;
-    logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`);
-});
+    
+    process.on('SIGTERM', () => {
+        logger.info('SIGTERM signal received: terminating');
+        if (intervalId !== undefined) {
+            clearInterval(intervalId);
+        }
+        compactor.shouldExit = true;
+    });
+    
+    process.on('SIGINT', () => {
+        logger.info('SIGINT signal received');
+        if (intervalId !== undefined) {
+            clearInterval(intervalId);
+        }
+        compactor.shouldExit = true;
+        setTimeout(() => {
+            logger.info('Exiting');
+            process.exit();        
+        }, MAX_WAIT_TIME_ON_EXIT);
+    });
+    
+    
+    process.on('SIGUSR2',function(){
+        logger.info("SIGUSR2 signal received");
+        if (!compactor.status.processing) {
+            setImmediate(run);
+            logger.info(`Force run`);
+        }
+        const status = compactor.status;
+        const elapsedTime = compactor.elapsedTime;
+        logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`);
+    });
+    
+}
\ No newline at end of file
diff --git a/src/kafka.js b/src/kafka.js
old mode 100644
new mode 100755
diff --git a/src/logger.js b/src/logger.js
old mode 100644
new mode 100755
index 9ef321c..0b0be2b
--- a/src/logger.js
+++ b/src/logger.js
@@ -1,9 +1,13 @@
 import { config } from './config.js';
 import pino from 'pino';
+import path, { join } from 'path';
 import { now } from './utils/date.js';
+import { fileURLToPath } from 'url';
 
-const logsFolder = new URL('../logs', import.meta.url);
-const logFile = `${logsFolder.pathname}/${now().toISOString()}.log`;
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+const logsFolder =process.env.LOG_FOLDER || path.join(__dirname, '../logs');
+const logFile = `${logsFolder}/${now().toISOString()}.log`;
 
 /** @type {{targets:import('pino').TransportTargetOptions[]}} */
 let transport = {
@@ -13,32 +17,21 @@ let transport = {
             level: (process.env.LOG_LEVEL || 'info').toLowerCase(),
             options: {
                 destination: logFile,
-                mkdir: true
+                singleLine: true
             }
         }
     ]
 };
 if (process.env.NODE_ENV !== 'production') {
-    transport.targets.push(
-            {
-                target: 'pino-pretty',
-                level: (process.env.LOG_LEVEL || 'info').toLowerCase(),
-                options: {
-                    ignore: 'pid,hostname'
-                }
-            }
-    );
-} else {
     transport.targets.push(
         {
-            target: 'pino/file',
+            target: 'pino-pretty',
             level: (process.env.LOG_LEVEL || 'info').toLowerCase(),
             options: {
-                singleLine: true,
                 ignore: 'pid,hostname'
             }
         }
-);   
+    );
 }
 
 /** @type {import('pino').LoggerOptions} */
diff --git a/src/minio.js b/src/minio.js
old mode 100644
new mode 100755
diff --git a/src/profiling.js b/src/profiling.js
new file mode 100755
index 0000000..946605b
--- /dev/null
+++ b/src/profiling.js
@@ -0,0 +1,54 @@
+import path from 'path';
+import { now } from './utils/date.js';
+import { logger } from './logger.js';
+import { fileURLToPath } from 'url';
+import cron from 'node-cron';
+import { convertTimeToCron } from "./utils/date.js";
+import v8 from 'v8';
+
+if(process.env.NODE_ENV == "development") {
+  logger.info("Profiling in progress...");
+  const __filename = fileURLToPath(import.meta.url);
+  const __dirname = path.dirname(__filename);
+  const profilingFolder =process.env.PROFILING_FOLDER || path.join(__dirname, '../../profiling');
+
+  // Schedule a task to run every x
+  let intervalInMin=30;
+  const cronTime = convertTimeToCron(intervalInMin);
+  logger.info(cronTime);
+  cron.schedule(cronTime, async () => {
+    logger.info(`schedule task for profiling running...`);
+    //let filename=`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`;
+    let filename=`${profilingFolder}/${v8.writeHeapSnapshot()}`;
+    logger.info(`Saved heapdump into ${v8.writeHeapSnapshot(filename)}`);
+  });
+}
+
+/*
+import {
+  Worker,
+  isMainThread,
+  parentPort,
+} from 'node:worker_threads';
+
+if (isMainThread) {
+  const worker = new Worker(__filename);
+
+  worker.once('message', (filename) => {
+    logger.info(`worker heapdump: ${filename}`);
+    // Now get a heapdump for the main thread.
+    logger.info(`main thread heapdump: ${v8.writeHeapSnapshot(`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`)}`);
+  });
+
+  // Tell the worker to create a heapdump.
+  worker.postMessage('heapdump');
+} else {
+  parentPort.once('message', (message) => {
+    if (message === 'heapdump') {
+      // Generate a heapdump for the worker
+      // and return the filename to the parent.
+      parentPort.postMessage(v8.writeHeapSnapshot(`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`));
+    }
+  });
+}
+*/
\ No newline at end of file
diff --git a/src/prueba.js b/src/prueba.js
old mode 100644
new mode 100755
diff --git a/src/simva.js b/src/simva.js
old mode 100644
new mode 100755
diff --git a/src/state.js b/src/state.js
old mode 100644
new mode 100755
index 4cb501f..a6c1dd0
--- a/src/state.js
+++ b/src/state.js
@@ -2,6 +2,7 @@ import { MinioClient } from './minio.js';
 import { logger } from './logger.js';
 import { join } from 'node:path';
 import { copyNoOverwrite, ensureDirectoryStructureExists, fileExists, forceRemove, listFiles, mktempPath, rename, withFile } from './utils/file.js';
+import { areArraysEqual, isStringArray, getFirstAndLastX } from './utils/array.js';
 import { duration, epoch, formatDuration, now, parseDate } from './utils/date.js';
 import { diffArray, diffSet } from './utils/misc.js';
 
@@ -70,37 +71,93 @@ export class ActivityCompactionState {
 
 	async garbageCollect() {
 		if (this.currentSha1 === undefined) {
-			return;
+			return 0;
 		}
 		const localHashesAndFiles = await this.#listLocalHashes();
 		const remoteHashesAndFiles = await this.#listRemoteHashes();
-
+		localHashesAndFiles.forEach((value, key) => {
+			logger.debug(`${key}: ${value.join(", ")}`);
+		});
+		remoteHashesAndFiles.forEach((value, key) => {
+			logger.debug(`${key}: ${value.join(", ")}`);
+		});
 		const localHashes = new Set(localHashesAndFiles.keys());
 		const remoteHashes = new Set(remoteHashesAndFiles.keys());
 		const setsDiff = diffSet(localHashes, remoteHashes);
-		if (setsDiff.added.length > 0 || setsDiff.removed.length > 0) {
-			logger.warn('Local and remote folders for activity not synced, garbage collection skipped: %s', this.activityId);
-			return;
-		}
-
+		logger.debug(setsDiff);
 		/** @type {string[]} */
 		let localFilesToRemove = [];
 		/** @type {string[]} */
 		let remoteFilesToRemove = [];
+		if (setsDiff.added.length > 0 || setsDiff.removed.length > 0) {
+			logger.warn('Local and remote folders for activity not synced for activity %s', this.activityId);
+			if(setsDiff.added.includes(this.currentSha1)) {
+				logger.warn('%s current local file not present: %s', this.currentSha1, this.activityId);
+				logger.info('garbage collection skipped: %s', this.activityId);
+				return;
+				//this.#minio.copyFromRemoteFile(this.#filesStateRemotePath(), this.#filesStateLocalPath());
+				//this.#minio.copyFromRemoteFile(this.#stateRemotePath(), this.#stateLocalPath());
+				//setsDiff.added= setsDiff.added.filter(sha1 => (sha1 !== this.currentSha1));
+			} else if(setsDiff.removed.includes(this.currentSha1)) {
+				logger.warn('%s current remote file not present: %s', this.currentSha1, this.activityId);
+				logger.info('garbage collection skipped: %s', this.activityId);
+				return;
+				//this.#minio.copyToRemoteFile(this.#filesStateLocalPath(), this.#filesStateRemotePath());
+				//this.#minio.copyToRemoteFile(this.#stateLocalPath(), this.#stateRemotePath());
+				//setsDiff.removed= setsDiff.removed.filter(sha1 => (sha1 !== this.currentSha1));
+			} else {
+				let localFileContent= (await this.#loadLocalState());
+				localFileContent.push("");
+				logger.debug("localFileContent:");
+				logger.debug(getFirstAndLastX(localFileContent, 3));
+				let remoteFileContent= (await this.#minio.getFile(this.#stateRemotePath())).split('\n');
+				logger.debug("remoteFileContent:");
+				logger.debug(getFirstAndLastX(remoteFileContent, 3));
+				if(areArraysEqual(localFileContent, remoteFileContent)) {
+					logger.info("Local file is the same that the remote file.");
+ 				} else {
+					logger.warn("Remote file is different that the local file.");
+					logger.warn('garbage collection skipped: %s', this.activityId);
+					return;
+					//this.#minio.copyFromRemoteFile(this.#filesStateRemotePath(), this.#filesStateLocalPath());
+					//this.#minio.copyFromRemoteFile(this.#stateRemotePath(), this.#stateLocalPath());
+				}
+			}
+			if(setsDiff.added.length > 0) {
+				setsDiff.added.forEach((hash) => {
+					const remoteFiles = remoteHashesAndFiles.get(hash);
+					if(isStringArray(remoteFiles)) {
+						remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles);
+					}
+				});
+			}
+
+			if(setsDiff.removed.length > 0) {
+				setsDiff.removed.forEach((hash) => {
+					const localFiles = localHashesAndFiles.get(hash);
+					if(isStringArray(localFiles)) {
+						localFilesToRemove = localFilesToRemove.concat(localFiles);
+					}
+				});
+			}
+		}
+
 		localHashes.forEach((hash) => {
 			if(hash !== this.currentSha1) {
 				const localFiles = localHashesAndFiles.get(hash);
-				localFilesToRemove = localFilesToRemove.concat(localFiles);
+				if(isStringArray(localFiles)) {
+					localFilesToRemove = localFilesToRemove.concat(localFiles);
+				}
 				const remoteFiles = remoteHashesAndFiles.get(hash);
-				remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles);
+				if(isStringArray(remoteFiles)) {
+					remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles);
+				}
 			}
 		});
-		setsDiff.added.forEach((hash) => {
-			const remoteFiles = remoteHashesAndFiles.get(hash);
-			remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles);
-		});
+		
 		await this.#removeRemoteFiles(remoteFilesToRemove);
 		await this.#removeLocalFiles(localFilesToRemove);
+		return 0;
 	}
 
 	/**
@@ -113,13 +170,14 @@ export class ActivityCompactionState {
 		const activityPath = join(this.#opts.localStatePath, this.activityId);
 		const files = await listFiles(activityPath);
 		for(const file of files) {
+			const filePath= join(activityPath, file);
 			const chunks = file.split('-');
 			const hash = chunks[0];
 			const entry = hashes.get(hash);
 			if (entry) {
-				entry.push(file);
+				entry.push(filePath);
 			} else {
-				hashes.set(hash, [file]);
+				hashes.set(hash, [filePath]);
 			}
 		}
 		if (this.currentSha1 !== undefined && hashes.size < 2 ) {
@@ -139,7 +197,7 @@ export class ActivityCompactionState {
 		const files = await this.#minio.listFiles(remotePath);
 		for(const file of files) {
 			const chunks = file.name.split('-');
-			const hash = chunks[0];
+			const hash = chunks[0].replace(remotePath, "");
 			const entry = hashes.get(hash);
 			if (entry) {
 				entry.push(file.name);
@@ -155,7 +213,6 @@ export class ActivityCompactionState {
 	 * @param {string[]} files 
 	 */
 	async #removeRemoteFiles(files) {
-		files = files.map((file) => `${this.#opts.remoteStatePath}/${this.activityId}/${file}`);
 		if (this.#opts.removeDryRun) {
 			logger.debug('DRY RUN - Removed remote files:\n %s', files.join(',\n'));
 		} else {
@@ -170,12 +227,11 @@ export class ActivityCompactionState {
 	 */
 	async #removeLocalFiles(files) {
 		for(const file of files) {
-			const localPath = join(this.#opts.localStatePath, this.activityId, file);
 			if (this.#opts.removeDryRun) {
-				logger.debug('DRY RUN - Removed local file: %s', localPath);
+				logger.debug('DRY RUN - Removed local file: %s', file);
 			} else {
-				await forceRemove(localPath);
-				logger.debug('Removed local file: %s', localPath);
+				await forceRemove(file);
+				logger.debug('Removed local file: %s', file);
 			}
 		}
 	}
@@ -296,6 +352,28 @@ export class ActivityCompactionState {
 		await rename(tmpPath, statePath, this.#opts.copyInsteadRename);
 	}
 
+
+	/**
+	 * 
+	 * @returns 
+	 */
+	async #loadLocalState() {
+		const statePath = this.#stateLocalPath();
+		const withState = withFile(statePath);
+
+		/** @type {string[]} */
+		const files = await withState(async (state) => {
+			/** @type {string[]} */
+			const files=[];
+			for await (const line of state.readLines()) {
+				files.push(line);
+			}
+			return files;
+		}, false);
+
+		return files;
+	}
+
 	/**
 	 * 
 	 * @param {string} [sha1]
@@ -445,17 +523,32 @@ export class CompactorState {
 	}
 
 	async garbageCollect() {
+        let activityToPass=[];
 		const nowDate = now();
 		const lastGC = this.#lastGC ?? epoch();
 		const elapsedTimeSinceLastGC = duration(lastGC, nowDate);
 		if (elapsedTimeSinceLastGC < this.#opts.gcInterval) {
-			return;
+			return [];
 		}
 
 		logger.debug('Garbage collection started');
 		for(const activity of this.#states.values()) {
 			try {
-				await activity.garbageCollect();
+				let result = await activity.garbageCollect();
+				logger.info("activity.garbageCollect : " + result);
+				switch(result) {
+					case 1:
+						logger.warn("Error during garbage collection.");
+						if(this.#opts.tryRecovery) {
+							logger.warn("Removing activity :" + activity.activityId);
+							await activity.clear();
+							await this.remove(activity.activityId);
+						}
+						activityToPass.push(activity.activityId);
+						break;
+					default:
+						logger.info("Everything ok during garbage collection.");		 
+				}
 			} catch (error) {
 				logger.error('Could not garbage collect: %s', activity.activityId);
 				logger.error(error);
@@ -464,6 +557,7 @@ export class CompactorState {
 		const finishTime = now();
 		logger.info('Garbage collection finished, took: %s', formatDuration(duration(nowDate, finishTime)));
 		this.#lastGC = finishTime;
+		return activityToPass;
 	}
 
 	/**
diff --git a/src/utils/array.js b/src/utils/array.js
new file mode 100755
index 0000000..39deab8
--- /dev/null
+++ b/src/utils/array.js
@@ -0,0 +1,35 @@
+/**
+ * Compare two arrays of strings. 
+ * 
+ * @param {string[]} arr1 
+ * @param {string[]} arr2 
+ */
+export function areArraysEqual(arr1, arr2) {
+    return arr1.length === arr2.length && arr1.every((val, index) => val === arr2[index]);
+}
+
+/**
+ * Check if the given `value` is an array of strings. 
+ * 
+ * @param {Object} value 
+ */
+export function isStringArray(value) {
+    return Array.isArray(value) && value.every(item => typeof item === "string");
+}
+
+/**
+ * Print the first and last `x` elements of an array.
+ * 
+ * @param {string[]} arr 
+ * @param {number} x 
+ */
+export function getFirstAndLastX(arr, x) {
+    if (arr.length <= 2 * x) {
+        return arr; // If the array is small, print it all
+    }
+
+    const firstX = arr.slice(0, x); // First X elements
+    const lastX = arr.slice(-x); // Last X elements
+
+    return [...firstX, "...", ...lastX]; // Using "..." as a placeholder
+}
\ No newline at end of file
diff --git a/src/utils/date.js b/src/utils/date.js
old mode 100644
new mode 100755
index c48b445..3ade916
--- a/src/utils/date.js
+++ b/src/utils/date.js
@@ -79,4 +79,62 @@ export function formatDuration(durationMillis) {
     duration.seconds = value;
 
     return Duration.serialize(duration);
+}
+
+/**
+ * 
+ * @param {number} durationMin 
+ * @returns {string}
+ */
+export function convertTimeToCron(durationMin) {
+    if (durationMin <= 0) {
+        throw new Error("Time should be greater than 0.");
+    }
+
+    if (durationMin < 60) {
+        return `*/${durationMin} * * * *`; // Runs every N minutes
+    }
+
+    const hours = Math.floor(durationMin / 60);
+    const extraMinutes = durationMin % 60;
+
+    if (hours < 24) {
+        let cronExpressions = [`0 */${hours} * * *`]; // Runs every X hours
+        if (extraMinutes > 0) {
+            cronExpressions.push(`${extraMinutes} */${hours} * * *`); // Extra minute offset
+        }
+        return cronExpressions.join("\n");
+    }
+
+    const days = Math.floor(hours / 24);
+    const extraHours = hours % 24;
+
+    if (days < 7) {
+        let cronExpressions = [`0 0 */${days} * *`]; // Runs every X days
+        if (extraHours > 0) {
+            cronExpressions.push(`0 */${extraHours} * * *`); // Extra hourly offset
+        }
+        return cronExpressions.join("\n");
+    }
+
+    const weeks = Math.floor(days / 7);
+    const extraDays = days % 7;
+
+    if (weeks < 4) {
+        let cronExpressions = [`0 0 * */${weeks} *`]; // Runs every X weeks
+        if (extraDays > 0) {
+            cronExpressions.push(`0 0 */${extraDays} * *`); // Extra daily offset
+        }
+        return cronExpressions.join("\n");
+    }
+
+    const months = Math.floor(weeks / 4);
+    const extraWeeks = weeks % 4;
+
+    let cronExpressions = [`0 0 1 */${months} *`]; // Runs every X months
+    if (extraWeeks > 0) {
+        cronExpressions.push(`0 0 * */${extraWeeks} *`); // Extra weekly offset
+    }
+
+    return cronExpressions.join("\n");
 }
\ No newline at end of file
diff --git a/src/utils/errors.js b/src/utils/errors.js
old mode 100644
new mode 100755
diff --git a/src/utils/file.js b/src/utils/file.js
old mode 100644
new mode 100755
index 0b0bbbb..561024a
--- a/src/utils/file.js
+++ b/src/utils/file.js
@@ -276,4 +276,4 @@ export async function rename(source, dest, useCopy = false) {
     } catch (error) {
         throw wrapError(error, `Could not rename '${source}' to '${dest}'`);
     }
-}
\ No newline at end of file
+}
diff --git a/src/utils/misc.js b/src/utils/misc.js
old mode 100644
new mode 100755

From 21da00621d5de7b804840349b21383bd7cba4d33 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 20 Feb 2025 16:26:42 +0100
Subject: [PATCH 02/41] fix profiling trace allocator memory

---
 src/profiling.js | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/src/profiling.js b/src/profiling.js
index 946605b..02814cc 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -5,6 +5,7 @@ import { fileURLToPath } from 'url';
 import cron from 'node-cron';
 import { convertTimeToCron } from "./utils/date.js";
 import v8 from 'v8';
+import process from 'node:process';
 
 if(process.env.NODE_ENV == "development") {
   logger.info("Profiling in progress...");
@@ -22,6 +23,18 @@ if(process.env.NODE_ENV == "development") {
     let filename=`${profilingFolder}/${v8.writeHeapSnapshot()}`;
     logger.info(`Saved heapdump into ${v8.writeHeapSnapshot(filename)}`);
   });
+
+  intervalInMin=1;
+  cron.schedule(convertTimeToCron(intervalInMin), async () => {
+    logger.info(`schedule task for profiling memory usage running...`);
+    logger.info(process.memoryUsage());
+    const heapUsed = process.memoryUsage().heapUsed / 1024 / 1024;
+    const arrayBuffers = process.memoryUsage().arrayBuffers / 1024 / 1024;
+    const rss = process.memoryUsage().rss / 1024 / 1024;
+    const external = process.memoryUsage().external / 1024 / 1024;
+    const heapTotal = process.memoryUsage().heapTotal / 1024 / 1024;
+    logger.info(`This app is currently using ${Math.floor(heapUsed)} MB of memory (heapTotal : ${Math.floor(heapTotal)} MB - rss : ${Math.floor(rss)} MB - arrayBuffers : ${Math.floor(arrayBuffers)} MB -external : ${Math.floor(external)} MB).`);
+  });
 }
 
 /*

From 9c0519043b2deef5033f3d7292b27501fc6cea5f Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 20 Feb 2025 19:08:57 +0100
Subject: [PATCH 03/41] fix profiling and log error in compactor

---
 src/compactor.js |  2 +-
 src/profiling.js | 21 +++++++++++++--------
 2 files changed, 14 insertions(+), 9 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index c03f792..a63a4e0 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -358,7 +358,7 @@ export class Compactor {
                         logger.info(activityState);
                         await state.save();
                     } catch(e) {
-                        logger.info(e);
+                        logger.error(e);
                     }
                 }
             } else {
diff --git a/src/profiling.js b/src/profiling.js
index 02814cc..c9d4a64 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -26,14 +26,19 @@ if(process.env.NODE_ENV == "development") {
 
   intervalInMin=1;
   cron.schedule(convertTimeToCron(intervalInMin), async () => {
-    logger.info(`schedule task for profiling memory usage running...`);
-    logger.info(process.memoryUsage());
-    const heapUsed = process.memoryUsage().heapUsed / 1024 / 1024;
-    const arrayBuffers = process.memoryUsage().arrayBuffers / 1024 / 1024;
-    const rss = process.memoryUsage().rss / 1024 / 1024;
-    const external = process.memoryUsage().external / 1024 / 1024;
-    const heapTotal = process.memoryUsage().heapTotal / 1024 / 1024;
-    logger.info(`This app is currently using ${Math.floor(heapUsed)} MB of memory (heapTotal : ${Math.floor(heapTotal)} MB - rss : ${Math.floor(rss)} MB - arrayBuffers : ${Math.floor(arrayBuffers)} MB -external : ${Math.floor(external)} MB).`);
+    let memoryUsage=process.memoryUsage();
+    let cpuUsage=process.cpuUsage();
+    logger.info(`MEMORY USAGE : ${JSON.stringify(memoryUsage)}`);
+    logger.info(`CPU USAGE : ${JSON.stringify(cpuUsage)}`);
+    const heapUsed = memoryUsage.heapUsed / 1024 / 1024;
+    const arrayBuffers = memoryUsage.arrayBuffers / 1024 / 1024;
+    const rss = memoryUsage.rss / 1024 / 1024;
+    const external = memoryUsage.external / 1024 / 1024;
+    const heapTotal = memoryUsage.heapTotal / 1024 / 1024;
+    logger.info(`MEMORY USAGE : This app is currently using ${Math.floor(heapUsed)} MB of memory (heapTotal : ${Math.floor(heapTotal)} MB - rss : ${Math.floor(rss)} MB - arrayBuffers : ${Math.floor(arrayBuffers)} MB -external : ${Math.floor(external)} MB).`);
+    const user = cpuUsage.user / 1000;
+    const system = cpuUsage.system / 1000;
+    logger.info(`CPU USAGE : This app is currently using user ${Math.floor(user)}s and system ${Math.floor(system)}s`);
   });
 }
 

From 9a52835554ebfe2a8b0bafc57f1264c0dedd9218 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 24 Feb 2025 07:08:37 +0100
Subject: [PATCH 04/41] fix compactor coping file from remote into remote

---
 package-lock.json | 12 +++----
 package.json      |  4 +--
 src/compactor.js  | 33 ++++++++++++------
 src/index.js      | 17 ++++++---
 src/minio.js      | 88 ++++++++++++++++++++++++++++++++++++++++++++++-
 src/profiling.js  | 18 +++++-----
 src/prueba2.js    | 27 +++++++++++++++
 src/state.js      |  7 ++++
 8 files changed, 173 insertions(+), 33 deletions(-)
 create mode 100644 src/prueba2.js

diff --git a/package-lock.json b/package-lock.json
index edb1d8d..8b89cc9 100755
--- a/package-lock.json
+++ b/package-lock.json
@@ -10,7 +10,7 @@
       "dependencies": {
         "dotenv": "^16.3.1",
         "kafkajs": "^2.2.4",
-        "minio": "8.0.1",
+        "minio": "8.0.4",
         "ms": "^2.1.3",
         "node-cron": "3.0.3",
         "pino": "^9.6.0",
@@ -700,9 +700,9 @@
       }
     },
     "node_modules/minio": {
-      "version": "8.0.1",
-      "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.1.tgz",
-      "integrity": "sha512-FzDO6yGnqLtm8sp3mXafWtiRUOslJSSg/aI0v9YbN5vjw5KLoODKAROCyi766NIvTSxcfHBrbhCSGk1A+MOzDg==",
+      "version": "8.0.4",
+      "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.4.tgz",
+      "integrity": "sha512-GVW7y2PNbzjjFJ9opVMGKvDNuRkyz3bMt1q7UrHs7bsKFWLXbSvMPffjE/HkVYWUjlD8kQwMaeqiHhhvZJJOfQ==",
       "license": "Apache-2.0",
       "dependencies": {
         "async": "^3.2.4",
@@ -710,7 +710,7 @@
         "browser-or-node": "^2.1.1",
         "buffer-crc32": "^1.0.0",
         "eventemitter3": "^5.0.1",
-        "fast-xml-parser": "^4.2.2",
+        "fast-xml-parser": "^4.4.1",
         "ipaddr.js": "^2.0.1",
         "lodash": "^4.17.21",
         "mime-types": "^2.1.35",
@@ -718,7 +718,7 @@
         "stream-json": "^1.8.0",
         "through2": "^4.0.2",
         "web-encoding": "^1.1.5",
-        "xml2js": "^0.5.0"
+        "xml2js": "^0.5.0 || ^0.6.2"
       },
       "engines": {
         "node": "^16 || ^18 || >=20"
diff --git a/package.json b/package.json
index 7dee975..fb9ebb4 100755
--- a/package.json
+++ b/package.json
@@ -5,7 +5,7 @@
   "type": "module",
   "scripts": {
     "start": "node ./src/index.js",
-    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --prof ./src/index.js",
+    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js",
     "start:debug": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof ./src/index.js"
   },
   "engines": {
@@ -13,7 +13,7 @@
   },
   "dependencies": {
     "dotenv": "^16.3.1",
-    "minio": "8.0.1",
+    "minio": "8.0.4",
     "kafkajs":"^2.2.4",
     "pino": "^9.6.0",
     "tinyduration": "^3.3.0",
diff --git a/src/compactor.js b/src/compactor.js
index a63a4e0..f926c14 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -161,16 +161,12 @@ export class Compactor {
                 logger.info(`New activity: %s`, activity._id);
                 activityState = await state.create(activity._id);
             }
-            if((config.concatEventPolicy === "true" && activitiesToGo.includes(activity._id)) || ! (config.concatEventPolicy === "true")) {
-                const updated = await this.#updateActivityTraces(activityState);
-                if (!updated) continue;
-                await this.#distributeTrace(activityState);
+            const updated = await this.#updateActivityTraces(activityState);
+            if (!updated) continue;
+            await this.#distributeTrace(activityState);
     
-                if (activities.length % 5) {
-                    await state.save();
-                }
-            } else {
-                logger.debug('Not needed to process activity: %s because of the Concat Event Policy', activity._id);
+            if (activities.length % 5) {
+                await state.save();
             }
         }
         await state.save();
@@ -256,10 +252,24 @@ export class Compactor {
      */
     async #distributeTrace(activityState) {
         const localStatePath = activityState.localStatePath;
+        const remoteStatePath = activityState.remoteStatePath;
         const outputDir = this.#opts.minio.outputs_dir;
         const tracesFilename = this.#opts.minio.traces_file;
         const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`;
-        await this.#minio.copyToRemoteFile(localStatePath, remotePath);
+        try {
+            //if(await this.#minio.fileExists(remotePath)) {
+            //    await this.#minio.copyWithinMinIO(remotePath, `${remotePath}.backup`);
+            //    await this.#minio.removeRemoteFile(remotePath);
+            //}
+            //await this.#minio.copyToRemoteFile(localStatePath, remotePath);
+            await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
+            logger.info("Object copied successfully!");
+        } catch (error) {
+            logger.error("Copy failed:");
+            logger.error(error);
+            //await this.#minio.copyWithinMinIO(`${remotePath}.backup`, remotePath);
+        }
+        //await this.#minio.removeRemoteFile(`${remotePath}.backup`);
         logger.info(`Copied compacted file for activity %s`, activityState.activityId);
     }
 
@@ -359,6 +369,9 @@ export class Compactor {
                         await state.save();
                     } catch(e) {
                         logger.error(e);
+                        //let list=await this.#minio.listMultipartUploads();
+                        //logger.info(list);
+                        //await this.#minio.abortMultipartUploads(list);
                     }
                 }
             } else {
diff --git a/src/index.js b/src/index.js
index 0b0520a..557b5e3 100755
--- a/src/index.js
+++ b/src/index.js
@@ -13,6 +13,7 @@ const run = compactor.compact.bind(compactor);
 const MAX_WAIT_TIME_ON_EXIT = 30*1000;
 
 let intervalId;
+//import "./prueba2.js";
 
 if (config.concatEventPolicy === "true") {
     await startKafkaProcess();
@@ -21,19 +22,25 @@ if (config.concatEventPolicy === "true") {
 }
 
 async function startKafkaProcess() {
-    await compactor.processConsistencyAndGarbage();
     // Schedule a task to run every x
     let gcIntervalInMin=Math.round(config.gcInterval/(1000*60));
     const cronTime = convertTimeToCron(gcIntervalInMin);
     logger.info(cronTime);
     cron.schedule(cronTime, async () => {
         logger.info("Compactor starting process garbage scheduled task running every x minutes");
-        await compactor.processConsistencyAndGarbage();
+        try {
+            await compactor.processConsistencyAndGarbage();
+        } catch(e) {
+            logger.error(e);
+        }
     });
+    try {
+        await compactor.processConsistencyAndGarbage();
+    } catch(e) {
+        logger.error(e);
+    }
     // Start consuming messages
-    (async () => {
-        await compactor.startKafkaConsumer();
-    })();
+    await compactor.startKafkaConsumer();
 }
 
 async function startPrevVersionProcess() {
diff --git a/src/minio.js b/src/minio.js
index bc227cf..fe11036 100755
--- a/src/minio.js
+++ b/src/minio.js
@@ -32,6 +32,12 @@ import { logger } from './logger.js';
  * @property {string} versionId versionId of the object.
  */
 
+/**
+ * @typedef MultipartUploadResult
+ * @property {string} uploadId uploadId of the object.
+ * @property {string} key key path of the object.
+ */
+
 export class MinioClient {
 
     /**
@@ -97,12 +103,74 @@ export class MinioClient {
      * 
      * @param {string} remotePath 
      * @param {string} localPath
-     * @returns {Promise<void>}
+     * @returns {Promise<FPutResult>}
      */
 	async copyToRemoteFile(localPath, remotePath) {
+        logger.debug(`Coping file ${localPath} to remote ${remotePath}`);
         return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath);
 	}
 
+    /**
+     * 
+     * @param {string} objectName 
+     * @param {string} uploadId
+     * @returns {Promise<void>}
+     */
+    async abortMultipartUpload(objectName, uploadId) {
+        await this.#minio.abortMultipartUpload(this.#opts.bucket, objectName, uploadId);
+    }
+
+    /**
+     * 
+     * @param {MultipartUploadResult[]} list
+     * @returns {Promise<void>}
+     */
+    async abortMultipartUploads(list) {
+        for (const item of list) {
+            try {
+                await this.#minio.abortMultipartUpload(this.#opts.bucket, item.key, item.uploadId);
+                logger.info(`Aborted upload: ${item.key} (Upload ID: ${item.uploadId})`);
+            } catch (error) {
+                logger.error(`Failed to abort ${item.key}:`, error);
+            }
+        }
+    }
+    
+
+    /**
+     * 
+     * @returns {Promise<MultipartUploadResult[]>}
+     */
+    async listMultipartUploads() {
+        return new Promise((resolve, reject) => {
+          const uploads = [];
+          
+          const stream = this.#minio.listIncompleteUploads(this.#opts.bucket, "", true);
+          
+          stream.on("data", (obj) => {
+            uploads.push(obj);
+            logger.info("Active Upload:", obj);
+          });
+      
+          stream.on("end", () => {
+            resolve(uploads);
+          });
+      
+          stream.on("error", (err) => {
+            reject(err);
+          });
+        });
+    }
+
+    async listV2MultipartUploads() {
+        this.#minio.listIncompleteUploads(this.#opts.bucket, "", true)
+          .on("data", async (obj) => {
+            logger.info(obj);
+            await this.#minio.abortMultipartUpload(this.#opts.bucket, obj.key, obj.uploadId);
+          })
+          .on("error", (err) => logger.error(err));
+      }
+
     /**
      * 
      * @param {string} file 
@@ -128,6 +196,7 @@ export class MinioClient {
      * @returns {Promise<void>}
      */
 	async removeRemoteFile(path) {
+        logger.debug(`removeRemoteFile file ${path}`);
         return this.#minio.removeObject(this.#opts.bucket, path);
 	}
 
@@ -152,6 +221,23 @@ export class MinioClient {
         return ! nextValue.done;
     }
 
+    
+    /**
+     * 
+     * @param {string} sourcePath 
+     * @param {string} destinationPath 
+     * @returns {Promise<void>}
+     */
+    async copyWithinMinIO(sourcePath, destinationPath) {
+        logger.debug(`Coping file into remote from ${sourcePath} to ${destinationPath}`);
+        this.#minio.copyObject(
+            this.#opts.bucket,
+            destinationPath,
+            `/${this.#opts.bucket}/${sourcePath}`
+          );
+      }
+      
+
 }
 
 function streamToString(stream) {
diff --git a/src/profiling.js b/src/profiling.js
index c9d4a64..56fbe9f 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -6,6 +6,7 @@ import cron from 'node-cron';
 import { convertTimeToCron } from "./utils/date.js";
 import v8 from 'v8';
 import process from 'node:process';
+import ms from 'ms';
 
 if(process.env.NODE_ENV == "development") {
   logger.info("Profiling in progress...");
@@ -13,19 +14,16 @@ if(process.env.NODE_ENV == "development") {
   const __dirname = path.dirname(__filename);
   const profilingFolder =process.env.PROFILING_FOLDER || path.join(__dirname, '../../profiling');
 
-  // Schedule a task to run every x
-  let intervalInMin=30;
-  const cronTime = convertTimeToCron(intervalInMin);
-  logger.info(cronTime);
-  cron.schedule(cronTime, async () => {
+  function heapdump() {
     logger.info(`schedule task for profiling running...`);
     //let filename=`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`;
     let filename=`${profilingFolder}/${v8.writeHeapSnapshot()}`;
     logger.info(`Saved heapdump into ${v8.writeHeapSnapshot(filename)}`);
-  });
+    setTimeout(heapdump, ms("30min"));
+  }
+  setTimeout(heapdump, ms("1min"));
 
-  intervalInMin=1;
-  cron.schedule(convertTimeToCron(intervalInMin), async () => {
+  function memoryAndCPUUsage() {
     let memoryUsage=process.memoryUsage();
     let cpuUsage=process.cpuUsage();
     logger.info(`MEMORY USAGE : ${JSON.stringify(memoryUsage)}`);
@@ -39,7 +37,9 @@ if(process.env.NODE_ENV == "development") {
     const user = cpuUsage.user / 1000;
     const system = cpuUsage.system / 1000;
     logger.info(`CPU USAGE : This app is currently using user ${Math.floor(user)}s and system ${Math.floor(system)}s`);
-  });
+    setTimeout(memoryAndCPUUsage, ms("1min"));
+  }
+  setTimeout(memoryAndCPUUsage, ms("1min"));
 }
 
 /*
diff --git a/src/prueba2.js b/src/prueba2.js
new file mode 100644
index 0000000..877b8c5
--- /dev/null
+++ b/src/prueba2.js
@@ -0,0 +1,27 @@
+import { MinioClient } from './minio.js';
+import { config } from './config.js';
+import { logger } from './logger.js';
+import ms from 'ms';
+let minio = new MinioClient(config.minio);
+
+async function copyToRemoteLittle() {
+    try {
+        await minio.copyToRemoteFile("/data/67b7515834931102b914d4ce/943b4a7d1e96c132e6ee4577076340f230477e82-files.txt", "outputs/67b7679e9289f002b94083e9/traces.json");
+        logger.info("Copy OK");
+    } catch(e) {
+        logger.error(e);
+    }
+    setTimeout(copyToRemote, ms("30s"));
+}
+
+async function copyToRemote() {
+    try {
+        await minio.copyToRemoteFile("/data/67b7679e9289f002b94083e9/traces.json", "outputs/67b7679e9289f002b94083e9/traces.json");
+        logger.info("Copy OK");
+    } catch(e) {
+        logger.error(e);
+    }
+    setTimeout(copyToRemoteLittle, ms("30s"));
+}
+
+copyToRemoteLittle()
\ No newline at end of file
diff --git a/src/state.js b/src/state.js
index a6c1dd0..8e441e2 100755
--- a/src/state.js
+++ b/src/state.js
@@ -392,6 +392,13 @@ export class ActivityCompactionState {
 		return this.#stateLocalPath();
 	}
 
+	/**
+	 * @returns
+	 */
+	get remoteStatePath() {
+		return this.#stateRemotePath();
+	}
+
 	/**
 	 * 
 	 * @param {string} [sha1]

From b0584eb3b03d4f8d0cbc002a7bd056cebfa8e597 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 24 Feb 2025 08:06:46 +0100
Subject: [PATCH 05/41] fix compactor metadata traces.json as application/json

---
 src/compactor.js | 13 +++++--------
 src/index.js     |  1 -
 src/minio.js     | 22 ++++++++++++++++++++--
 src/prueba2.js   | 14 ++++++++++++--
 4 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index f926c14..2dcd4ea 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -257,19 +257,16 @@ export class Compactor {
         const tracesFilename = this.#opts.minio.traces_file;
         const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`;
         try {
-            //if(await this.#minio.fileExists(remotePath)) {
-            //    await this.#minio.copyWithinMinIO(remotePath, `${remotePath}.backup`);
-            //    await this.#minio.removeRemoteFile(remotePath);
-            //}
-            //await this.#minio.copyToRemoteFile(localStatePath, remotePath);
-            await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
+            const metadata = {
+                "Content-Type": "application/json"
+            };
+            await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
+            //await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
             logger.info("Object copied successfully!");
         } catch (error) {
             logger.error("Copy failed:");
             logger.error(error);
-            //await this.#minio.copyWithinMinIO(`${remotePath}.backup`, remotePath);
         }
-        //await this.#minio.removeRemoteFile(`${remotePath}.backup`);
         logger.info(`Copied compacted file for activity %s`, activityState.activityId);
     }
 
diff --git a/src/index.js b/src/index.js
index 557b5e3..a6e4d27 100755
--- a/src/index.js
+++ b/src/index.js
@@ -2,7 +2,6 @@ import { config, isInDebugMode } from './config.js';
 import { logger } from './logger.js';
 import cron from 'node-cron';
 import { Compactor } from './compactor.js';
-import { getState } from './state.js';
 import { convertTimeToCron } from "./utils/date.js";
 import "./profiling.js";
 logger.debug('Current config: %o', config);
diff --git a/src/minio.js b/src/minio.js
index fe11036..c5eb197 100755
--- a/src/minio.js
+++ b/src/minio.js
@@ -38,6 +38,12 @@ import { logger } from './logger.js';
  * @property {string} key key path of the object.
  */
 
+/**
+ * @typedef MetadataObject
+ * @property {string} Version 
+ * @property {string} Content-Type
+ */
+
 export class MinioClient {
 
     /**
@@ -75,6 +81,13 @@ export class MinioClient {
         return files;
     }
 
+    /**
+     * @param {string} filePath
+     */
+    async getMetadataObject(filePath) {
+        return this.#minio.statObject(this.#opts.bucket, filePath);
+    }
+
     /**
      * 
      * @param {string} file 
@@ -103,11 +116,16 @@ export class MinioClient {
      * 
      * @param {string} remotePath 
      * @param {string} localPath
+     * @param {MetadataObject} metadata
      * @returns {Promise<FPutResult>}
      */
-	async copyToRemoteFile(localPath, remotePath) {
+	async copyToRemoteFile(localPath, remotePath, metadata) {
         logger.debug(`Coping file ${localPath} to remote ${remotePath}`);
-        return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath);
+        if(metadata) {
+            return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath, metadata);
+        } else {
+           return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath);
+        }
 	}
 
     /**
diff --git a/src/prueba2.js b/src/prueba2.js
index 877b8c5..e77599f 100644
--- a/src/prueba2.js
+++ b/src/prueba2.js
@@ -3,10 +3,19 @@ import { config } from './config.js';
 import { logger } from './logger.js';
 import ms from 'ms';
 let minio = new MinioClient(config.minio);
+const outputFilePath = "outputs/67b7679e9289f002b94083e9/traces.json";
+const copyLittleDataFilePath = "/data/67b7515834931102b914d4ce/943b4a7d1e96c132e6ee4577076340f230477e82-files.txt";
+//const copyDataFilePath = "/data/67b7679e9289f002b94083e9/02430353fc6b1c5b25d5852aab8c4e43a03de538-state.txt"; 
+const copyDataFilePath = "/data/67b7679e9289f002b94083e9/traces.txt";
 
+let metadata = {
+    "Content-Type": "application/json",
+    //"X-Amz-Meta-Version": "1.0"
+};
 async function copyToRemoteLittle() {
     try {
-        await minio.copyToRemoteFile("/data/67b7515834931102b914d4ce/943b4a7d1e96c132e6ee4577076340f230477e82-files.txt", "outputs/67b7679e9289f002b94083e9/traces.json");
+        logger.info(await minio.getMetadataObject(outputFilePath));
+        await minio.copyToRemoteFile(copyLittleDataFilePath,outputFilePath, metadata);
         logger.info("Copy OK");
     } catch(e) {
         logger.error(e);
@@ -16,7 +25,8 @@ async function copyToRemoteLittle() {
 
 async function copyToRemote() {
     try {
-        await minio.copyToRemoteFile("/data/67b7679e9289f002b94083e9/traces.json", "outputs/67b7679e9289f002b94083e9/traces.json");
+        logger.info(await minio.getMetadataObject(outputFilePath));
+        await minio.copyToRemoteFile(copyDataFilePath,outputFilePath, metadata);
         logger.info("Copy OK");
     } catch(e) {
         logger.error(e);

From 12974471d89a64adf8a0b7a259d481c9044e4fbd Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 24 Feb 2025 13:17:06 +0100
Subject: [PATCH 06/41] fix profiling using env variable

---
 src/config.js    | 2 ++
 src/profiling.js | 4 ++--
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/config.js b/src/config.js
index 061277f..10f95b7 100755
--- a/src/config.js
+++ b/src/config.js
@@ -4,6 +4,7 @@ import ms from "ms";
 
 /**
  * @typedef CompactorOptions
+ * @property {boolean} enabled_debug_profiling
  * @property {string} concatEventPolicy 
  * @property {number} batchSize
  * @property {number} maxDelay
@@ -21,6 +22,7 @@ import ms from "ms";
 
 /** @type {CompactorOptions} */
 export const config = {
+    enabled_debug_profiling : process.env.ENABLE_PROFILING == "true" ? true : false,
     concatEventPolicy: process.env.CONCAT_EVENT_POLICY || 'true', // if true minio-events else previous version of trace allocator
     batchSize: process.env.BATCH_SIZE !== undefined ? parseInt(process.env.BATCH_SIZE) : 500,
     maxDelay: process.env.MAX_DELAY !== undefined ? ms(process.env.MAX_DELAY) : ms("5min"),
diff --git a/src/profiling.js b/src/profiling.js
index 56fbe9f..9a29c4e 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -1,5 +1,5 @@
 import path from 'path';
-import { now } from './utils/date.js';
+import { config } from './config.js';
 import { logger } from './logger.js';
 import { fileURLToPath } from 'url';
 import cron from 'node-cron';
@@ -8,7 +8,7 @@ import v8 from 'v8';
 import process from 'node:process';
 import ms from 'ms';
 
-if(process.env.NODE_ENV == "development") {
+if(process.env.NODE_ENV == "development" && config.enabled_debug_profiling) {
   logger.info("Profiling in progress...");
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = path.dirname(__filename);

From b32052bdd0ee8d9c91a4e6ed3f865717b8b5174e Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 24 Feb 2025 13:53:46 +0100
Subject: [PATCH 07/41] fix profiling

---
 src/config.js    | 6 ++----
 src/index.js     | 2 +-
 src/profiling.js | 5 +----
 3 files changed, 4 insertions(+), 9 deletions(-)

diff --git a/src/config.js b/src/config.js
index 10f95b7..4dcc3e5 100755
--- a/src/config.js
+++ b/src/config.js
@@ -4,8 +4,7 @@ import ms from "ms";
 
 /**
  * @typedef CompactorOptions
- * @property {boolean} enabled_debug_profiling
- * @property {string} concatEventPolicy 
+ * @property {boolean} concatEventPolicy 
  * @property {number} batchSize
  * @property {number} maxDelay
  * @property {number} refreshInterval
@@ -22,8 +21,7 @@ import ms from "ms";
 
 /** @type {CompactorOptions} */
 export const config = {
-    enabled_debug_profiling : process.env.ENABLE_PROFILING == "true" ? true : false,
-    concatEventPolicy: process.env.CONCAT_EVENT_POLICY || 'true', // if true minio-events else previous version of trace allocator
+    concatEventPolicy: process.env.CONCAT_EVENT_POLICY !== undefined ? (process.env.CONCAT_EVENT_POLICY.toLocaleLowerCase() === 'false' ? false : true) : true, // if true minio-events else previous version of trace allocator
     batchSize: process.env.BATCH_SIZE !== undefined ? parseInt(process.env.BATCH_SIZE) : 500,
     maxDelay: process.env.MAX_DELAY !== undefined ? ms(process.env.MAX_DELAY) : ms("5min"),
     refreshInterval: process.env.REFRESH_INTERVAL !== undefined ? ms(process.env.REFRESH_INTERVAL) : ms("10min"),
diff --git a/src/index.js b/src/index.js
index a6e4d27..70f1913 100755
--- a/src/index.js
+++ b/src/index.js
@@ -14,7 +14,7 @@ const MAX_WAIT_TIME_ON_EXIT = 30*1000;
 let intervalId;
 //import "./prueba2.js";
 
-if (config.concatEventPolicy === "true") {
+if (config.concatEventPolicy) {
     await startKafkaProcess();
 } else {
     await startPrevVersionProcess();
diff --git a/src/profiling.js b/src/profiling.js
index 9a29c4e..f507ef6 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -1,14 +1,11 @@
 import path from 'path';
-import { config } from './config.js';
 import { logger } from './logger.js';
 import { fileURLToPath } from 'url';
-import cron from 'node-cron';
-import { convertTimeToCron } from "./utils/date.js";
 import v8 from 'v8';
 import process from 'node:process';
 import ms from 'ms';
 
-if(process.env.NODE_ENV == "development" && config.enabled_debug_profiling) {
+if(process.env.NODE_ENV == "development" && process.env.ENABLE_DEBUG_PROFILING == "true") {
   logger.info("Profiling in progress...");
   const __filename = fileURLToPath(import.meta.url);
   const __dirname = path.dirname(__filename);

From 621174d0e514246eb65c7a7ced3e87e9c16617f8 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 25 Feb 2025 15:39:50 +0100
Subject: [PATCH 08/41] update node version to LTS + remove docker publish
 action

---
 .github/workflows/docker-publish.yml | 96 ----------------------------
 Dockerfile                           |  2 +-
 2 files changed, 1 insertion(+), 97 deletions(-)
 delete mode 100755 .github/workflows/docker-publish.yml

diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml
deleted file mode 100755
index 04b9751..0000000
--- a/.github/workflows/docker-publish.yml
+++ /dev/null
@@ -1,96 +0,0 @@
-name: Docker
-
-# This workflow uses actions that are not certified by GitHub.
-# They are provided by a third-party and are governed by
-# separate terms of service, privacy policy, and support
-# documentation.
-
-on:
-  push:
-    branches: [ "master" ]
-    # Publish semver tags as releases.
-    tags: [ 'v*.*.*' ]
-  pull_request:
-    branches: [ "master" ]
-
-env:
-  # Use docker.io for Docker Hub if empty
-  REGISTRY: docker.io
-  # github.repository as <account>/<repo>
-  IMAGE_NAME: ${{ github.repository }}
-
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-      # This is used to complete the identity challenge
-      # with sigstore/fulcio when running outside of PRs.
-      id-token: write
-
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-
-      # Install the cosign tool except on PR
-      # https://github.com/sigstore/cosign-installer
-      - name: Install cosign
-        if: github.event_name != 'pull_request'
-        uses: sigstore/cosign-installer@59acb6260d9c0ba8f4a2f9d9b48431a222b68e20 #v3.5.0
-        with:
-          cosign-release: 'v2.2.4'
-
-      # Set up BuildKit Docker container builder to be able to build
-      # multi-platform images and export cache
-      # https://github.com/docker/setup-buildx-action
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@f95db51fddba0c2d1ec667646a06c2ce06100226 # v3.0.0
-
-      # Login against a Docker registry except on PR
-      # https://github.com/docker/login-action
-      - name: Log into registry ${{ env.REGISTRY }}
-        if: github.event_name != 'pull_request'
-        uses: docker/login-action@343f7c4344506bcbf9b4de18042ae17996df046d # v3.0.0
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      # Extract metadata (tags, labels) for Docker
-      # https://github.com/docker/metadata-action
-      - name: Extract Docker metadata
-        id: meta
-        uses: docker/metadata-action@96383f45573cb7f253c731d3b3ab81c87ef81934 # v5.0.0
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
-
-      # Build and push Docker image with Buildx (don't push on PR)
-      # https://github.com/docker/build-push-action
-      - name: Build and push Docker image
-        id: build-and-push
-        uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0
-        with:
-          context: .
-          push: ${{ github.event_name != 'pull_request' }}
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
-
-      # Sign the resulting Docker image digest except on PRs.
-      # This will only write to the public Rekor transparency log when the Docker
-      # repository is public to avoid leaking data.  If you would like to publish
-      # transparency data even for private images, pass --force to cosign below.
-      # https://github.com/sigstore/cosign
-      - name: Sign the published Docker image
-        if: ${{ github.event_name != 'pull_request' }}
-        env:
-          # https://docs.github.com/en/actions/security-guides/security-hardening-for-github-actions#using-an-intermediate-environment-variable
-          TAGS: ${{ steps.meta.outputs.tags }}
-          DIGEST: ${{ steps.build-and-push.outputs.digest }}
-        # This step uses the identity token to provision an ephemeral certificate
-        # against the sigstore community Fulcio instance.
-        run: echo "${TAGS}" | xargs -I {} cosign sign --yes {}@${DIGEST}
\ No newline at end of file
diff --git a/Dockerfile b/Dockerfile
index 3d390e8..df3f8f0 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM node:20.18.3-bullseye
+FROM node:22.14.0-bullseye
 
 # Install ca-certificates and update them
 RUN apt-get update && apt-get install -y --no-install-recommends \

From 3a0490324324969fab03f965baa7d74e56962424 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 25 Feb 2025 16:47:01 +0100
Subject: [PATCH 09/41] fix dockerfile to remove docker startup bash

---
 Dockerfile        |  3 +--
 docker-startup.sh | 16 ----------------
 2 files changed, 1 insertion(+), 18 deletions(-)
 delete mode 100755 docker-startup.sh

diff --git a/Dockerfile b/Dockerfile
index df3f8f0..7cae36d 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -25,5 +25,4 @@ RUN mkdir -p /home/node/logs
 RUN chown -R node:node /home/node/logs
 
 
-
-CMD [ "/home/node/app/docker-startup.sh" ]
\ No newline at end of file
+CMD [ "npm", "start" ]
\ No newline at end of file
diff --git a/docker-startup.sh b/docker-startup.sh
deleted file mode 100755
index 9dbf0a1..0000000
--- a/docker-startup.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-[[ "${DEBUG:-false}" == "true" ]] && set -x
-
-if [[ ! -d "node_modules" ]]; then
-  npm install
-fi
-
-#start trace allocator
-echo "${NODE_ENV:-production}"
-if [[ "${NODE_ENV:-production}" == "development" ]]; then
-  npm run dev
-  #perf record -e cycles:u -g -- npm run dev > perf.out
-else
-  npm start
-fi
\ No newline at end of file

From ca1641a2025abf8860fda3005d47d85510c5c75a Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Wed, 26 Feb 2025 11:21:28 +0100
Subject: [PATCH 10/41] fix package.json + dockerignore

---
 .dockerignore     | 4 +++-
 package-lock.json | 4 ++--
 package.json      | 4 ++--
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/.dockerignore b/.dockerignore
index d3a2e64..df3c4b1 100755
--- a/.dockerignore
+++ b/.dockerignore
@@ -2,4 +2,6 @@ node_modules/
 manual-scripts/
 .github
 Dockerfile
-.dockerignore
\ No newline at end of file
+.dockerignore
+*.heapsnapshot
+*.log
\ No newline at end of file
diff --git a/package-lock.json b/package-lock.json
index 8b89cc9..7d5dff3 100755
--- a/package-lock.json
+++ b/package-lock.json
@@ -19,11 +19,11 @@
       },
       "devDependencies": {
         "@types/node": "^20.9.0",
-        "nodemon": "3.1.9",
+        "nodemon": "^3.1.9",
         "pino-pretty": "^13.0.0"
       },
       "engines": {
-        "node": ">=20 <21"
+        "node": ">=20 <24"
       }
     },
     ".yalc/dayjs-es": {
diff --git a/package.json b/package.json
index fb9ebb4..7978df9 100755
--- a/package.json
+++ b/package.json
@@ -9,7 +9,7 @@
     "start:debug": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof ./src/index.js"
   },
   "engines": {
-    "node": ">=20 <21"
+    "node": ">=20 <24"
   },
   "dependencies": {
     "dotenv": "^16.3.1",
@@ -24,6 +24,6 @@
   "devDependencies": {
     "@types/node": "^20.9.0",
     "pino-pretty": "^13.0.0",
-    "nodemon":"3.1.9"
+    "nodemon":"^3.1.9"
   }
 }

From 42b91bd615d2d0441a29a0afb23b3d2254bcbaca Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Wed, 26 Feb 2025 19:48:29 +0100
Subject: [PATCH 11/41] fix DemoCA certificate + docker compose example

---
 .gitignore          |  3 +--
 DemoCA.crt          | 20 ++++++++++++++++
 docker-compose.yaml | 56 +++++++++++++++++++++++++++++++++++++++++++++
 3 files changed, 77 insertions(+), 2 deletions(-)
 create mode 100755 DemoCA.crt
 create mode 100755 docker-compose.yaml

diff --git a/.gitignore b/.gitignore
index fc0146e..4e4670d 100755
--- a/.gitignore
+++ b/.gitignore
@@ -4,5 +4,4 @@
 # Config files
 config-test.js
 *.log
-*.heapsnapshot
-*.crt
\ No newline at end of file
+*.heapsnapshot
\ No newline at end of file
diff --git a/DemoCA.crt b/DemoCA.crt
new file mode 100755
index 0000000..eec572b
--- /dev/null
+++ b/DemoCA.crt
@@ -0,0 +1,20 @@
+-----BEGIN CERTIFICATE-----
+MIIDSzCCAjOgAwIBAgIUCmGeUFwNiGdJZLwpkevTf29wjrowDQYJKoZIhvcNAQEL
+BQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwNjI1MTk0NjA4WhcNMjkw
+NjIyMTk0NjA4WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcN
+AQEBBQADggEPADCCAQoCggEBANKAEz3dPmYJvwjhSBssR0jP5JK7CO8iOqzhcqzy
+cFb05hr9unPtPS76lrEEkYSgklwW5X4aVTOfqrJkc0RyoLwyYjmPei45xRiFeTfu
+FA1cAwbRAriTuOySvp8U0H/J8WDCNrNrhDags9wzbtBZuTJt1UCNYXYJyHMF2NE3
+7fO86x3AVW30v0iiHa00ev2Fk5YZkKUYuL0Dv2Ul3SbWFRq3hKjFypkOK+Iur/Ye
+wW7XLAoVatOvlFZCmQ+kD59b0L9QCXqDU2odlwzgNrMpczfTyuzJKQyxm0iLJgxW
+QvC329dkjyq/5RR7RvWjpjzyAopXMo5ZE4CWgHQz97Rfhv8CAwEAAaOBkDCBjTAd
+BgNVHQ4EFgQUNOD0d021fHYcir4UebCMny2eQVAwUQYDVR0jBEowSIAUNOD0d021
+fHYcir4UebCMny2eQVChGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghQKYZ5Q
+XA2IZ0lkvCmR69N/b3COujAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkq
+hkiG9w0BAQsFAAOCAQEAFwfZVGQnd1ReZeL4uyvS2ZVI3U2GYFGrdxOiC+1nt3TG
+QF5W+4RN2394Mv1DaUIjtzAuKanR73qyiVjxkFetsSj4dItzbZQBCpRT32E1EHb2
+aFXbTJ76JGPHed0wmAdCxA+zlEjKZr7B73sS2GPfSLJw3CsQYMtVSCXLCox4DUrW
+yOlF2kgO1OGI4kPbtw+CKWwCh0x0StckYo5tI8MF1U5UBtAXzPJ2pXyuc7t0Hejy
+MMKILaiIrqCtK/9BdQWW3iqDhhcLX6VrPb33QNA7PV9UBVAfBPPIejDLSv1aS/X9
+SuegPnlhRzBl1PL0zAMlEGI9GR3HruCJoDKo9MlR7g==
+-----END CERTIFICATE-----
diff --git a/docker-compose.yaml b/docker-compose.yaml
new file mode 100755
index 0000000..9f22823
--- /dev/null
+++ b/docker-compose.yaml
@@ -0,0 +1,56 @@
+version: '3'
+
+services:
+  simva-trace-allocator:
+    build: ./
+    command: [ "npm", "run", "dev" ]
+    stdin_open: true
+    tty: true
+    environment:
+      #DEBUG
+      DEBUG: true
+      #LOG
+      LOG_LEVEL: debug
+      LOG_FOLDER: /home/node/logs
+      #NODE
+      NODE_ENV: development
+      NODE_EXTRA_CA_CERTS: /home/node/app/DemoCA.crt
+      #TRACE ALLOCATOR
+      CONCAT_EVENT_POLICY: true
+      BATCH_SIZE: 100
+      MAX_DELAY: 5min
+      REFRESH_INTERVAL: 4h
+      LOCAL_STATE: /data
+      REMOTE_STATE: state
+      REMOVE_DRY_RUN: true
+      GC_INTERVAL: 2h
+      COPY_INSTEAD_RENAME: true
+      TRY_RECOVERY: true
+      #MINIO
+      MINIO_HOST: minio-api.external.test
+      MINIO_SSL: true
+      MINIO_PORT: 443
+      MINIO_ACCESS_KEY: minio
+      MINIO_SECRET_KEY: secret
+      MINIO_BUCKET: traces
+      MINIO_TOPICS_DIR: kafka-topics
+      MINIO_TRACES_TOPIC: traces
+      MINIO_OUTPUTS_DIR: outputs
+      MINIO_TRACES_FILE: traces.json
+      #SIMVA API
+      SIMVA_HOST: simva-api.external.test
+      SIMVA_PROTOCOL: https
+      SIMVA_PORT: 443
+      SIMVA_USER: admin
+      SIMVA_PASSWORD: password
+      SIMVA_WAIT_TIME: 10
+      SIMVA_MAX_RETRIES: 20
+      #KAFKA
+      SIMVA_KAFKA_CLIENTID: simva_trace_allocator
+      SIMVA_KAFKA_BROKER: kafka1.internal.test:9092
+      SIMVA_KAFKA_GROUPID: simva_trace_allocator
+      SIMVA_KAFKA_MINIO_TOPIC: minio-events
+    hostname: simva-trace-allocator.internal.test
+    volumes:
+      - ./:/home/node/app
+      - /home/node/app/node_modules #exclude node_modules
\ No newline at end of file

From 681b38f7b2ed95d6c42976dca26f52c87c5b3e5d Mon Sep 17 00:00:00 2001
From: Ivan Martinez-Ortiz <imartinezortiz@gmail.com>
Date: Thu, 6 Mar 2025 17:11:46 +0100
Subject: [PATCH 12/41] refactor shasum computation in an utility function

---
 src/compactor.js | 23 +++-------------------
 src/utils/sha.js | 51 ++++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 54 insertions(+), 20 deletions(-)
 create mode 100644 src/utils/sha.js

diff --git a/src/compactor.js b/src/compactor.js
index 2dcd4ea..288d9c2 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -4,9 +4,9 @@ import { MinioClient } from './minio.js';
 import { SimvaClient } from './simva.js';
 import { KafkaClient } from './kafka.js';
 import { getState } from './state.js';
-import { createHash } from 'node:crypto';
 import { binarySearch, diffArray } from './utils/misc.js';
 import { config } from './config.js';
+import { sha1sums } from './utils/sha.js';
 
 /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */
 /** @typedef {import('./simva.js').Activity} Activity */
@@ -219,12 +219,7 @@ export class Compactor {
     async #updateActivityTraces(activityState) {
         let traceFiles = (await this.#minio.getTraces(activityState.activityId)).map((o) => o.name);
         traceFiles.sort();
-        const hash = createSha1();
-        for(const traceFile of traceFiles) {
-            hash.update(traceFile);
-            hash.update('\n');
-        }
-        const sha1 = hash.digest('hex');
+        const sha1 = sha1sums(traceFiles);
         if (sha1 === activityState.currentSha1) {
             logger.debug(`Nothing to do for activity %s`, activityState.activityId);
             return false;
@@ -387,10 +382,7 @@ export class Compactor {
      * @returns {Promise<boolean>} false if nothing new
      */
     async #updateActivityTracesFromPath(activityState, keyPath) {
-        const hash = createSha1();
-        hash.update(keyPath);
-        hash.update('\n');
-        const sha1 = hash.digest('hex');
+        const sha1 = sha1sums(keyPath);
         const nowDate = now();
         const filesToAdd = [keyPath];
         logger.info(`Compacting activity %s`, activityState.activityId);
@@ -436,12 +428,3 @@ export class Compactor {
         return this.#opts;
     }
 }
-
-/**
- * Create sha1 hash function.
- *
- * @returns 
- */
-function createSha1() {
-	return createHash('sha1');
-}
diff --git a/src/utils/sha.js b/src/utils/sha.js
new file mode 100644
index 0000000..329db0c
--- /dev/null
+++ b/src/utils/sha.js
@@ -0,0 +1,51 @@
+import { createHash } from 'node:crypto';
+
+/**
+ * @overload
+ * 
+ * @param {string} str 
+ * 
+ * @returns {string}
+ */
+/**
+ * @overload
+ * 
+ * @param {string[]} strings 
+ * 
+ * @returns {string}
+ */
+/**
+ * 
+ * @param {string | string[]} strings 
+ * 
+ * @returns {string}
+ */
+export function sha1sums(strings) {
+    /** @type {string[]} */
+    let str;
+    if (Array.isArray(strings)) {
+        str = strings;
+    } else if (typeof strings === 'string') {
+        str = [ strings ]
+    } else {
+        throw new TypeError(`'strings' must be string or string[]`);
+    }
+
+    const hash = createSha1();
+    for(const s of strings) {
+        hash.update(s);
+        hash.update('\n');
+    }
+    const sha1 = hash.digest('hex');
+    return sha1;
+}
+
+
+/**
+ * Create sha1 hash function.
+ *
+ * @returns 
+ */
+function createSha1() {
+	return createHash('sha1');
+}

From e8493095b5b9adeba0e60359a7386b50264d845c Mon Sep 17 00:00:00 2001
From: Ivan Martinez-Ortiz <imartinezortiz@gmail.com>
Date: Thu, 6 Mar 2025 17:12:03 +0100
Subject: [PATCH 13/41] minor refactor

---
 src/minio.js | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/src/minio.js b/src/minio.js
index c5eb197..8d8ef40 100755
--- a/src/minio.js
+++ b/src/minio.js
@@ -116,16 +116,12 @@ export class MinioClient {
      * 
      * @param {string} remotePath 
      * @param {string} localPath
-     * @param {MetadataObject} metadata
+     * @param {MetadataObject} [metadata]
      * @returns {Promise<FPutResult>}
      */
 	async copyToRemoteFile(localPath, remotePath, metadata) {
         logger.debug(`Coping file ${localPath} to remote ${remotePath}`);
-        if(metadata) {
-            return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath, metadata);
-        } else {
-           return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath);
-        }
+        return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath, metadata);
 	}
 
     /**

From de3aa662d51cf340f4bee848237344d6dfea1576 Mon Sep 17 00:00:00 2001
From: Ivan Martinez-Ortiz <imartinezortiz@gmail.com>
Date: Thu, 6 Mar 2025 17:12:47 +0100
Subject: [PATCH 14/41] rename method to improve readability

---
 src/compactor.js | 2 +-
 src/kafka.js     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 288d9c2..db18f46 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -396,7 +396,7 @@ export class Compactor {
         try {
             logger.info('Compactor starting Kafka consumption...');
             // Start Kafka consumption and pass the processMessage as a callback
-            await this.#kafka.consumeLatestMessages(this.processMessage.bind(this));
+            await this.#kafka.startKafkaConsumer(this.processMessage.bind(this));
         } catch (error) {
             console.error('Error starting Compactor:', error);
         }
diff --git a/src/kafka.js b/src/kafka.js
index c6b5068..4bfbcef 100755
--- a/src/kafka.js
+++ b/src/kafka.js
@@ -32,7 +32,7 @@ export class KafkaClient {
     }
 
     // Run the consumer and process messages
-    async consumeLatestMessages(onMessage) {
+    async startKafkaConsumer(onMessage) {
         try {
             await this.connect();
 

From 05e107ab86f0de3cbe3489b2efb99500bd678085 Mon Sep 17 00:00:00 2001
From: Ivan Martinez-Ortiz <imartinezortiz@gmail.com>
Date: Thu, 6 Mar 2025 17:14:00 +0100
Subject: [PATCH 15/41] do not eat errors without properly management.

Kafka consumer does not commit offset if process message function throws an exception
---
 src/compactor.js | 170 +++++++++++++++++++++++------------------------
 1 file changed, 82 insertions(+), 88 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index db18f46..64e18bb 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -277,102 +277,96 @@ export class Compactor {
      * @param {any} message
      */
     async processMessage(message) {
-        try {
-            // Log the received message
-            logger.info('Received message:');
-            logger.info(message.value);
-
-            let state = await getState(this.#opts, this.#minio);
-            // Set up the delimiter and the required bucket and path values
-            let delimiter = '/';
-            let bucket = this.#opts.minio.bucket;
-            
-            // Build the path to traces topic
-            let tracestopicspath = `${this.#opts.minio.topics_dir}${delimiter}${this.#opts.minio.traces_topic}${delimiter}_id=`;
+        // Log the received message
+        logger.info('Received message:');
+        logger.info(message.value);
+
+        let state = await getState(this.#opts, this.#minio);
+        // Set up the delimiter and the required bucket and path values
+        let delimiter = '/';
+        let bucket = this.#opts.minio.bucket;
         
-            // Log the constructed path
-            logger.info(`Trace topic path: ${tracestopicspath}`);
+        // Build the path to traces topic
+        let tracestopicspath = `${this.#opts.minio.topics_dir}${delimiter}${this.#opts.minio.traces_topic}${delimiter}_id=`;
+    
+        // Log the constructed path
+        logger.info(`Trace topic path: ${tracestopicspath}`);
+    
+        // Parse the message value (assuming it's a JSON string)
+        let ev = JSON.parse(message.value);
+        let key = ev.Key;
         
-            // Parse the message value (assuming it's a JSON string)
-            let ev = JSON.parse(message.value);
-            let key = ev.Key;
-            
-            // Log the key extracted from the message
-            logger.info(`Received Key: ${key}`);
+        // Log the key extracted from the message
+        logger.info(`Received Key: ${key}`);
+    
+        // Remove the bucket and trace topic path from the key to get the key value
+        let keyvalue = key.replace(`${bucket}${delimiter}${tracestopicspath}`, "");
         
-            // Remove the bucket and trace topic path from the key to get the key value
-            let keyvalue = key.replace(`${bucket}${delimiter}${tracestopicspath}`, "");
-            
-            // Log the key value after removal
-            logger.info(`Key value without bucket and path: ${keyvalue}`);
+        // Log the key value after removal
+        logger.info(`Key value without bucket and path: ${keyvalue}`);
+    
+        // Split the key value to extract activityId and filename
+        let added = keyvalue.split(delimiter);
         
-            // Split the key value to extract activityId and filename
-            let added = keyvalue.split(delimiter);
-            
-            // Initialize variables for activityId and filename
-            let activityId = null;
-            let filename = null;
-            let keyWithoutBucket = null;
+        // Initialize variables for activityId and filename
+        let activityId = null;
+        let filename = null;
+        let keyWithoutBucket = null;
+    
+        // If the split key has exactly 2 parts, extract activityId and filename
+        if (added.length === 2) {
+            throw new Error('Key format is unexpected. Unable to extract activityId and filename.');
+        }
+        activityId = added[0];
+        filename = added[1];
+        keyWithoutBucket = `${tracestopicspath}${activityId}${delimiter}${filename}`;
+
+        // Log the extracted values
+        logger.info(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`);
         
-            // If the split key has exactly 2 parts, extract activityId and filename
-            if (added.length === 2) {
-                activityId = added[0];
-                filename = added[1];
-                keyWithoutBucket = `${tracestopicspath}${activityId}${delimiter}${filename}`;
-
-                // Log the extracted values
-                logger.info(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`);
-                
-                // ActivityState
-                let activityState = state.get(activityId);
-                if (activityState === undefined) {
-                    logger.info(`New activity: %s`, activityId);
-                    activityState = await state.create(activityId);
-                }
-                logger.info(activityState);
-                // compute which files need to be appended
-                const activityFiles = (await activityState.files()).sort();
-                logger.info(activityFiles);
-                
-                if(activityFiles.includes(keyWithoutBucket)) {
-                    logger.warn("Already consumed.")
-                } else {
-                    try {
-                        let positionvalue=-binarySearch(activityFiles, keyWithoutBucket, true, (a,b)=> { 
-                            if(typeof a == "string" && typeof b == "string" ) {
-                                return a.localeCompare(b); 
-                            } else {
-                                return -1;
-                            }
-                        })-1;
-                        var nextposition=activityFiles.length;
-                        logger.info(keyWithoutBucket);
-                        logger.info("positionvalue:");
-                        logger.info(positionvalue);
-                        logger.info("nextposition:");
-                        logger.info(nextposition);
-                        if(positionvalue < nextposition) {
-                            logger.warn("Not ordered. Should have been consumed before.")
-                        }
-                        await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket);
-                        logger.info(activityState);
-                        await this.#distributeTrace(activityState);
-                        logger.info(activityState);
-                        await state.save();
-                    } catch(e) {
-                        logger.error(e);
-                        //let list=await this.#minio.listMultipartUploads();
-                        //logger.info(list);
-                        //await this.#minio.abortMultipartUploads(list);
-                    }
-                }
+        // ActivityState
+        let activityState = state.get(activityId);
+        if (activityState === undefined) {
+            logger.info(`New activity: %s`, activityId);
+            activityState = await state.create(activityId);
+        }
+        logger.info(activityState);
+        // compute which files need to be appended
+        const activityFiles = (await activityState.files()).sort();
+        logger.info(activityFiles);
+        
+        if(activityFiles.includes(keyWithoutBucket)) {
+            logger.warn("Already consumed: %s", keyWithoutBucket);
+            return;
+        }
+
+        let positionvalue=-binarySearch(activityFiles, keyWithoutBucket, true, (a,b)=> { 
+            if(typeof a == "string" && typeof b == "string" ) {
+                return a.localeCompare(b); 
             } else {
-                logger.warn('Key format is unexpected. Unable to extract activityId and filename.');
+                return -1;
             }
-        } catch(e) {
-            logger.debug('Error processing message:');
-            logger.debug(e);
+        })-1;
+
+        const nextposition = activityFiles.length;
+
+        logger.info(keyWithoutBucket);
+        logger.info("positionvalue:");
+        logger.info(positionvalue);
+        logger.info("nextposition:");
+        logger.info(nextposition);
+
+        if(positionvalue < nextposition) {
+            logger.warn("Not ordered. Should have been consumed before.")
         }
+
+        await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket);
+        logger.info(activityState);
+
+        await this.#distributeTrace(activityState);
+        logger.info(activityState);
+
+        await state.save();
     }
      
     /**

From b9272f3f08b0275d9fb84770268c41e9a31b03db Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 10:42:09 +0100
Subject: [PATCH 16/41] fix remoteOutputPath + log debug +  state version +
 remove owners in actvity states

---
 src/compactor.js | 66 +++++++++++++++++++++++-------------------------
 src/kafka.js     |  2 +-
 src/state.js     | 34 ++++++++++++++++++++-----
 3 files changed, 60 insertions(+), 42 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 64e18bb..56c826e 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -97,9 +97,6 @@ export class Compactor {
 
         logger.info(`Known %d activities, received %d`, state.size, activities.length);
 
-        const outputDir = this.#opts.minio.outputs_dir;
-        const tracesFilename = this.#opts.minio.traces_file;
-
         this.status.total = activities.length;
         const inconsistent = [];
         for(let idx=0; idx < activities.length; idx++) {
@@ -118,11 +115,11 @@ export class Compactor {
             }
 
             let consistent = await activityState.checkConsistency();
-            const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`;
-                if (! await this.#minio.fileExists(remotePath) ) {
-                    logger.warn('Compact file for activity \'%s\' not found: %s', activityState.activityId, remotePath);
-                    consistent = false;
-                }
+            const remotePath = activityState.remoteOutputPath;
+            if (! await this.#minio.fileExists(remotePath) ) {
+                logger.warn('Compact file for activity \'%s\' not found: %s', activityState.activityId, remotePath);
+                consistent = false;
+            }
             if (!consistent) {
                 inconsistent.push(activity._id);
             }
@@ -248,9 +245,7 @@ export class Compactor {
     async #distributeTrace(activityState) {
         const localStatePath = activityState.localStatePath;
         const remoteStatePath = activityState.remoteStatePath;
-        const outputDir = this.#opts.minio.outputs_dir;
-        const tracesFilename = this.#opts.minio.traces_file;
-        const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`;
+        const remotePath = activityState.remoteOutputPath;
         try {
             const metadata = {
                 "Content-Type": "application/json"
@@ -278,8 +273,8 @@ export class Compactor {
      */
     async processMessage(message) {
         // Log the received message
-        logger.info('Received message:');
-        logger.info(message.value);
+        logger.debug('Received message:');
+        logger.debug(message.value);
 
         let state = await getState(this.#opts, this.#minio);
         // Set up the delimiter and the required bucket and path values
@@ -290,20 +285,20 @@ export class Compactor {
         let tracestopicspath = `${this.#opts.minio.topics_dir}${delimiter}${this.#opts.minio.traces_topic}${delimiter}_id=`;
     
         // Log the constructed path
-        logger.info(`Trace topic path: ${tracestopicspath}`);
+        logger.debug(`Trace topic path: ${tracestopicspath}`);
     
         // Parse the message value (assuming it's a JSON string)
         let ev = JSON.parse(message.value);
         let key = ev.Key;
         
         // Log the key extracted from the message
-        logger.info(`Received Key: ${key}`);
+        logger.debug(`Received Key: ${key}`);
     
         // Remove the bucket and trace topic path from the key to get the key value
         let keyvalue = key.replace(`${bucket}${delimiter}${tracestopicspath}`, "");
         
         // Log the key value after removal
-        logger.info(`Key value without bucket and path: ${keyvalue}`);
+        logger.debug(`Key value without bucket and path: ${keyvalue}`);
     
         // Split the key value to extract activityId and filename
         let added = keyvalue.split(delimiter);
@@ -322,18 +317,18 @@ export class Compactor {
         keyWithoutBucket = `${tracestopicspath}${activityId}${delimiter}${filename}`;
 
         // Log the extracted values
-        logger.info(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`);
+        logger.debug(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`);
         
         // ActivityState
         let activityState = state.get(activityId);
         if (activityState === undefined) {
-            logger.info(`New activity: %s`, activityId);
+            logger.debug(`New activity: %s`, activityId);
             activityState = await state.create(activityId);
         }
-        logger.info(activityState);
+        logger.debug(activityState);
         // compute which files need to be appended
-        const activityFiles = (await activityState.files()).sort();
-        logger.info(activityFiles);
+        const activityFiles = (await activityState.files());
+        logger.debug(activityFiles);
         
         if(activityFiles.includes(keyWithoutBucket)) {
             logger.warn("Already consumed: %s", keyWithoutBucket);
@@ -350,21 +345,24 @@ export class Compactor {
 
         const nextposition = activityFiles.length;
 
-        logger.info(keyWithoutBucket);
-        logger.info("positionvalue:");
-        logger.info(positionvalue);
-        logger.info("nextposition:");
-        logger.info(nextposition);
+        logger.debug(keyWithoutBucket);
+        logger.debug("positionvalue:");
+        logger.debug(positionvalue);
+        logger.debug("nextposition:");
+        logger.debug(nextposition);
 
         if(positionvalue < nextposition) {
             logger.warn("Not ordered. Should have been consumed before.")
         }
 
-        await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket);
-        logger.info(activityState);
+        activityFiles.splice(nextposition, 0, keyWithoutBucket);
+        logger.debug(activityFiles);
+        const sha1 = sha1sums(activityFiles);
+        await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket, sha1);
+        logger.debug(activityState);
 
         await this.#distributeTrace(activityState);
-        logger.info(activityState);
+        logger.debug(activityState);
 
         await state.save();
     }
@@ -373,14 +371,14 @@ export class Compactor {
      * Update Activity Traces From Path
      * @param {ActivityCompactionState} activityState 
      * @param {string} keyPath
+     * @param {string} sha1
      * @returns {Promise<boolean>} false if nothing new
      */
-    async #updateActivityTracesFromPath(activityState, keyPath) {
-        const sha1 = sha1sums(keyPath);
+    async #updateActivityTracesFromPath(activityState, keyPath, sha1) {
         const nowDate = now();
         const filesToAdd = [keyPath];
         logger.info(`Compacting activity %s`, activityState.activityId);
-        logger.info(filesToAdd);
+        logger.debug(filesToAdd);
         await activityState.update(filesToAdd, nowDate, sha1);
         return true;
     }
@@ -392,7 +390,7 @@ export class Compactor {
             // Start Kafka consumption and pass the processMessage as a callback
             await this.#kafka.startKafkaConsumer(this.processMessage.bind(this));
         } catch (error) {
-            console.error('Error starting Compactor:', error);
+            logger.error('Error starting Compactor:', error);
         }
     }
 
@@ -402,7 +400,7 @@ export class Compactor {
             await this.#kafka.disconnect();
             logger.info('Compactor stopped Kafka consumption.');
         } catch (error) {
-            console.error('Error stopping Compactor:', error);
+            logger.error('Error stopping Compactor:', error);
         }
     }
     
diff --git a/src/kafka.js b/src/kafka.js
index 4bfbcef..bfcf13f 100755
--- a/src/kafka.js
+++ b/src/kafka.js
@@ -59,7 +59,7 @@ export class KafkaClient {
                 }
             });
         } catch (error) {
-            console.error('Error consuming messages:', error);
+            console.error('Error starting Kafka messages consumer :', error);
         }
     }
 
diff --git a/src/state.js b/src/state.js
index 8e441e2..7ae6e5b 100755
--- a/src/state.js
+++ b/src/state.js
@@ -11,6 +11,7 @@ import { diffArray, diffSet } from './utils/misc.js';
 /**
  * @typedef SerializedCompactorState
  * @property {string} lastGC
+ * @property {number} version
  * @property {Map<string, ActivityCompactionState>} states
  */
 
@@ -26,7 +27,6 @@ export class ActivityCompactionState {
 		this.activityId = activityId;
 		this.#opts = opts;
 		this.#minio = minio;
-		this.owners = [];
 		this.currentSha1 = null;
 		this.lastUpdate = epoch();
 	}
@@ -40,9 +40,6 @@ export class ActivityCompactionState {
 	/** @type {string} */
 	activityId;
 
-	/** @type {string[]} */
-	owners;
-
 	/** @type {string} */
 	currentSha1;
 
@@ -294,6 +291,17 @@ export class ActivityCompactionState {
 		return path;
 	}
 
+	/**
+	 * 
+	 * @returns 
+	 */
+	#outputRemotePath() {
+		const path = join(this.#opts.minio.outputs_dir, this.activityId,this.#opts.minio.traces_file);
+		return path;
+	}
+
+	
+
 	async #copyFromRemoteFilesState() {
 		const remotePath = this.#filesStateRemotePath();
 		const localPath = this.#filesStateLocalPath();
@@ -399,6 +407,13 @@ export class ActivityCompactionState {
 		return this.#stateRemotePath();
 	}
 
+	/**
+	 * @returns
+	 */
+	get remoteOutputPath() {
+		return this.#outputRemotePath();
+	}
+
 	/**
 	 * 
 	 * @param {string} [sha1]
@@ -495,6 +510,7 @@ export class CompactorState {
 		this.#minio = minio;
 		this.#states = new Map();
 		this.#lastGC = null;
+		this.#version = null;
 	}
 	/** @type {CompactorOptions} opts */
 	#opts;
@@ -508,6 +524,9 @@ export class CompactorState {
 	/** @type {Date} */
 	#lastGC;
 
+	/** @type {Number} */
+	#version;
+
 	async init() {
 		let loaded = await this.#loadLocalState();
 		if (loaded) return;
@@ -622,11 +641,14 @@ export class CompactorState {
 			}
 		}
 		this.#lastGC = serializedState.lastGC !== null ? new Date(Date.parse(serializedState.lastGC)) : null;
+		this.#version = serializedState.version !== null ? serializedState.version : null ;
 	}
 
 	async save() {
+		this.#version = this.#version !== null ? 0 : this.#version+1;
 		/** @type {SerializedCompactorState} */
 		const serializedState = {
+			version: this.#version,
 			states: this.#states,
 			lastGC: this.#lastGC !== null ? this.#lastGC.toISOString() : null
 		}
@@ -719,8 +741,7 @@ function replacer(key, value) {
 			value: {
 				activityId : value.activityId,
 				lastUpdate : value.lastUpdate.toISOString(),
-				currentSha1 : value.currentSha1,
-				owners : value.owners
+				currentSha1 : value.currentSha1
 			}
 		}
 	}
@@ -743,7 +764,6 @@ function withContextReviver(opts, minio) {
 				const activityState = new ActivityCompactionState(value.activityId, opts, minio);
 				activityState.lastUpdate = parseDate(value.lastUpdate);
 				activityState.currentSha1 = value.currentSha1;
-				activityState.owners = value.owners;
 				return activityState;
 			}
 		}

From 0f61adde830074c92b30d709132680248849e0fc Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 10:43:09 +0100
Subject: [PATCH 17/41] kafka start in index.js

---
 src/compactor.js |  30 +--------
 src/index.js     | 157 +++++++++++++++++++++++++----------------------
 2 files changed, 87 insertions(+), 100 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 56c826e..e835966 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -2,10 +2,8 @@ import { now, duration, formatDuration } from './utils/date.js';
 import { logger } from './logger.js';
 import { MinioClient } from './minio.js'; 
 import { SimvaClient } from './simva.js';
-import { KafkaClient } from './kafka.js';
 import { getState } from './state.js';
 import { binarySearch, diffArray } from './utils/misc.js';
-import { config } from './config.js';
 import { sha1sums } from './utils/sha.js';
 
 /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */
@@ -29,7 +27,6 @@ export class Compactor {
         this.#opts = opts;
         this.#minio = new MinioClient(opts.minio);
         this.#simva = new SimvaClient(opts.simva);
-        this.#kafka = new KafkaClient(opts.kafka);
         this.shouldExit = false;
         this.status = {
             processing: false,
@@ -44,9 +41,6 @@ export class Compactor {
     /** @type {MinioClient} */
     #minio;
 
-    /** @type {KafkaClient} */
-    #kafka;
-
     /** @type {SimvaClient} */
     #simva;
 
@@ -248,7 +242,8 @@ export class Compactor {
         const remotePath = activityState.remoteOutputPath;
         try {
             const metadata = {
-                "Content-Type": "application/json"
+                "Content-Type": "application/json",
+                "Version": "1"
             };
             await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
             //await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
@@ -382,27 +377,6 @@ export class Compactor {
         await activityState.update(filesToAdd, nowDate, sha1);
         return true;
     }
-
-    // Method to start consuming messages using KafkaClient
-    async startKafkaConsumer() {
-        try {
-            logger.info('Compactor starting Kafka consumption...');
-            // Start Kafka consumption and pass the processMessage as a callback
-            await this.#kafka.startKafkaConsumer(this.processMessage.bind(this));
-        } catch (error) {
-            logger.error('Error starting Compactor:', error);
-        }
-    }
-
-    // Method to stop consuming messages
-    async stopKafkaConsumer() {
-        try {
-            await this.#kafka.disconnect();
-            logger.info('Compactor stopped Kafka consumption.');
-        } catch (error) {
-            logger.error('Error stopping Compactor:', error);
-        }
-    }
     
     /**
     * Get MinioClient 
diff --git a/src/index.js b/src/index.js
index 70f1913..9e527fa 100755
--- a/src/index.js
+++ b/src/index.js
@@ -3,93 +3,106 @@ import { logger } from './logger.js';
 import cron from 'node-cron';
 import { Compactor } from './compactor.js';
 import { convertTimeToCron } from "./utils/date.js";
+import { KafkaClient } from './kafka.js';
 import "./profiling.js";
+import ms from 'ms';
 logger.debug('Current config: %o', config);
 
 const compactor = new Compactor(config);
 const run = compactor.compact.bind(compactor);
 
-const MAX_WAIT_TIME_ON_EXIT = 30*1000;
-
 let intervalId;
-//import "./prueba2.js";
 
 if (config.concatEventPolicy) {
+    const kafka = new KafkaClient(config.kafka);
     await startKafkaProcess();
-} else {
-    await startPrevVersionProcess();
-}
-
-async function startKafkaProcess() {
-    // Schedule a task to run every x
-    let gcIntervalInMin=Math.round(config.gcInterval/(1000*60));
-    const cronTime = convertTimeToCron(gcIntervalInMin);
-    logger.info(cronTime);
-    cron.schedule(cronTime, async () => {
-        logger.info("Compactor starting process garbage scheduled task running every x minutes");
+    async function startKafkaProcess() {
+        // Schedule a task to run every x
+        let gcIntervalInMin=Math.round(config.gcInterval/(1000*60));
+        const cronTime = convertTimeToCron(gcIntervalInMin);
+        logger.info(cronTime);
+        const task = async () => {
+            logger.info("Compactor starting process garbage scheduled task running every x minutes");
+            try {
+                await compactor.processConsistencyAndGarbage();
+            } catch(e) {
+                logger.error(e);
+            }
+        };
+        
+        await task();
+        
+        cron.schedule(cronTime, task);
+        
         try {
-            await compactor.processConsistencyAndGarbage();
-        } catch(e) {
-            logger.error(e);
+            logger.info('Compactor starting Kafka consumption...');
+            // Start Kafka consumption and pass the processMessage as a callback
+            await kafka.startKafkaConsumer(compactor.processMessage);
+        } catch (error) {
+            logger.error('Error starting Compactor:', error);
         }
-    });
-    try {
-        await compactor.processConsistencyAndGarbage();
-    } catch(e) {
-        logger.error(e);
     }
-    // Start consuming messages
-    await compactor.startKafkaConsumer();
-}
-
-async function startPrevVersionProcess() {
-    try {
-        // Run the compact function immediately at launch
-        logger.info('Running compactor at launch...');
-        await run();
-
-        // Set an interval to run compact periodically if not in debug mode
-        if (!isInDebugMode()) {
-            intervalId = setInterval(run, config.refreshInterval);
-        } else {
-            await run();
+    // Method to stop consuming messages
+    async function stopKafkaConsumer() {
+        try {
+            await kafka.disconnect();
+            logger.info('Compactor stopped Kafka consumption.');
+        } catch (error) {
+            logger.error('Error stopping Compactor:', error);
         }
-
-        logger.info('Compactor initialized.');
-    } catch (error) {
-        logger.error('Error during compactor initialization:', error);
     }
+} else {    
+    const MAX_WAIT_TIME_ON_EXIT = ms("30 sec");
+    await startPrevVersionProcess();
+
+    async function startPrevVersionProcess() {
+        try {
+            // Run the compact function immediately at launch
+            logger.info('Running compactor at launch...');
+            await run();
     
-    process.on('SIGTERM', () => {
-        logger.info('SIGTERM signal received: terminating');
-        if (intervalId !== undefined) {
-            clearInterval(intervalId);
-        }
-        compactor.shouldExit = true;
-    });
-    
-    process.on('SIGINT', () => {
-        logger.info('SIGINT signal received');
-        if (intervalId !== undefined) {
-            clearInterval(intervalId);
-        }
-        compactor.shouldExit = true;
-        setTimeout(() => {
-            logger.info('Exiting');
-            process.exit();        
-        }, MAX_WAIT_TIME_ON_EXIT);
-    });
-    
+            // Set an interval to run compact periodically if not in debug mode
+            if (!isInDebugMode()) {
+                intervalId = setInterval(run, config.refreshInterval);
+            } else {
+                await run();
+            }
     
-    process.on('SIGUSR2',function(){
-        logger.info("SIGUSR2 signal received");
-        if (!compactor.status.processing) {
-            setImmediate(run);
-            logger.info(`Force run`);
+            logger.info('Compactor initialized.');
+        } catch (error) {
+            logger.error('Error during compactor initialization:', error);
         }
-        const status = compactor.status;
-        const elapsedTime = compactor.elapsedTime;
-        logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`);
-    });
-    
-}
\ No newline at end of file
+        
+        process.on('SIGTERM', () => {
+            logger.info('SIGTERM signal received: terminating');
+            if (intervalId !== undefined) {
+                clearInterval(intervalId);
+            }
+            compactor.shouldExit = true;
+        });
+        
+        process.on('SIGINT', () => {
+            logger.info('SIGINT signal received');
+            if (intervalId !== undefined) {
+                clearInterval(intervalId);
+            }
+            compactor.shouldExit = true;
+            setTimeout(() => {
+                logger.info('Exiting');
+                process.exit();        
+            }, MAX_WAIT_TIME_ON_EXIT);
+        });
+        
+        
+        process.on('SIGUSR2',function(){
+            logger.info("SIGUSR2 signal received");
+            if (!compactor.status.processing) {
+                setImmediate(run);
+                logger.info(`Force run`);
+            }
+            const status = compactor.status;
+            const elapsedTime = compactor.elapsedTime;
+            logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`);
+        });      
+    }
+}

From 4d494228046ba3f210702c89e3a36cf7e143e5f3 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 10:43:26 +0100
Subject: [PATCH 18/41] remove test2 js

---
 src/prueba2.js | 37 -------------------------------------
 1 file changed, 37 deletions(-)
 delete mode 100644 src/prueba2.js

diff --git a/src/prueba2.js b/src/prueba2.js
deleted file mode 100644
index e77599f..0000000
--- a/src/prueba2.js
+++ /dev/null
@@ -1,37 +0,0 @@
-import { MinioClient } from './minio.js';
-import { config } from './config.js';
-import { logger } from './logger.js';
-import ms from 'ms';
-let minio = new MinioClient(config.minio);
-const outputFilePath = "outputs/67b7679e9289f002b94083e9/traces.json";
-const copyLittleDataFilePath = "/data/67b7515834931102b914d4ce/943b4a7d1e96c132e6ee4577076340f230477e82-files.txt";
-//const copyDataFilePath = "/data/67b7679e9289f002b94083e9/02430353fc6b1c5b25d5852aab8c4e43a03de538-state.txt"; 
-const copyDataFilePath = "/data/67b7679e9289f002b94083e9/traces.txt";
-
-let metadata = {
-    "Content-Type": "application/json",
-    //"X-Amz-Meta-Version": "1.0"
-};
-async function copyToRemoteLittle() {
-    try {
-        logger.info(await minio.getMetadataObject(outputFilePath));
-        await minio.copyToRemoteFile(copyLittleDataFilePath,outputFilePath, metadata);
-        logger.info("Copy OK");
-    } catch(e) {
-        logger.error(e);
-    }
-    setTimeout(copyToRemote, ms("30s"));
-}
-
-async function copyToRemote() {
-    try {
-        logger.info(await minio.getMetadataObject(outputFilePath));
-        await minio.copyToRemoteFile(copyDataFilePath,outputFilePath, metadata);
-        logger.info("Copy OK");
-    } catch(e) {
-        logger.error(e);
-    }
-    setTimeout(copyToRemoteLittle, ms("30s"));
-}
-
-copyToRemoteLittle()
\ No newline at end of file

From e2ed8015fe3112248407ef439d09ad0d66b639f2 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 11:25:04 +0100
Subject: [PATCH 19/41] insertOrdered via binary search for files state

---
 src/compactor.js | 30 ++++--------------------------
 src/state.js     | 44 ++++++++++++++++++++++++++++++++++++++------
 2 files changed, 42 insertions(+), 32 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index e835966..dfa68a1 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -208,8 +208,7 @@ export class Compactor {
      * @returns {Promise<boolean>} false if nothing new
      */
     async #updateActivityTraces(activityState) {
-        let traceFiles = (await this.#minio.getTraces(activityState.activityId)).map((o) => o.name);
-        traceFiles.sort();
+        let traceFiles = (await this.#minio.getTraces(activityState.activityId)).map((o) => o.name).sort();
         const sha1 = sha1sums(traceFiles);
         if (sha1 === activityState.currentSha1) {
             logger.debug(`Nothing to do for activity %s`, activityState.activityId);
@@ -329,30 +328,9 @@ export class Compactor {
             logger.warn("Already consumed: %s", keyWithoutBucket);
             return;
         }
-
-        let positionvalue=-binarySearch(activityFiles, keyWithoutBucket, true, (a,b)=> { 
-            if(typeof a == "string" && typeof b == "string" ) {
-                return a.localeCompare(b); 
-            } else {
-                return -1;
-            }
-        })-1;
-
-        const nextposition = activityFiles.length;
-
-        logger.debug(keyWithoutBucket);
-        logger.debug("positionvalue:");
-        logger.debug(positionvalue);
-        logger.debug("nextposition:");
-        logger.debug(nextposition);
-
-        if(positionvalue < nextposition) {
-            logger.warn("Not ordered. Should have been consumed before.")
-        }
-
-        activityFiles.splice(nextposition, 0, keyWithoutBucket);
-        logger.debug(activityFiles);
-        const sha1 = sha1sums(activityFiles);
+        const newActivityFiles= await activityState.insertOrdered([keyWithoutBucket]);
+        logger.debug(newActivityFiles);
+        const sha1 = sha1sums(newActivityFiles);
         await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket, sha1);
         logger.debug(activityState);
 
diff --git a/src/state.js b/src/state.js
index 7ae6e5b..d616528 100755
--- a/src/state.js
+++ b/src/state.js
@@ -4,7 +4,7 @@ import { join } from 'node:path';
 import { copyNoOverwrite, ensureDirectoryStructureExists, fileExists, forceRemove, listFiles, mktempPath, rename, withFile } from './utils/file.js';
 import { areArraysEqual, isStringArray, getFirstAndLastX } from './utils/array.js';
 import { duration, epoch, formatDuration, now, parseDate } from './utils/date.js';
-import { diffArray, diffSet } from './utils/misc.js';
+import { binarySearch, diffArray, diffSet } from './utils/misc.js';
 
 /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */
 
@@ -66,6 +66,39 @@ export class ActivityCompactionState {
 		return files;
 	}
 
+	/**
+	 * 
+	 * @param {string[]} files 
+	 * @returns {Promise<string[]>}
+	 */
+	async insertOrdered(files) {
+		let activityFiles = await this.files();
+		for(let file in files) {
+			let positionvalue=-binarySearch(activityFiles, file, true, (a,b)=> { 
+				if(typeof a == "string" && typeof b == "string" ) {
+					return a.localeCompare(b); 
+				} else {
+					return -1;
+				}
+			})-1;
+
+			const nextposition = activityFiles.length;
+			logger.info(file);
+			logger.info("positionvalue:");
+			logger.info(positionvalue);
+			logger.info("nextposition:");
+			logger.info(nextposition);
+			
+			if(positionvalue < nextposition) {
+				logger.warn("Not ordered. Should have been consumed before.")
+			}
+			
+			activityFiles.splice(nextposition, 0, file);
+			logger.info(activityFiles);
+		}
+		return files;
+	}
+
 	async garbageCollect() {
 		if (this.currentSha1 === undefined) {
 			return 0;
@@ -437,12 +470,11 @@ export class ActivityCompactionState {
 			const currentFilesStatePath = this.#filesStateLocalPath();
 			await copyNoOverwrite(currentFilesStatePath, tmpPath);
 		}
+		const stateFiles = await this.insertOrdered(filesToAdd);
 		const withFilesState = withFile(tmpPath, 'a');
-		await withFilesState(async (filesState) => {
-			for (const line of filesToAdd) {
-				await filesState.write(line+'\n');
-			}
-		});
+		await withFilesState(async (file) => {
+			await file.writeFile(stateFiles.join("\n"));
+		})
 		await rename(tmpPath, filesStatePath, this.#opts.copyInsteadRename);
 	}
 

From cf36e64b97d9428d62b81b4d2dadaf0d845b3eea Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 11:52:18 +0100
Subject: [PATCH 20/41] fix logger.info

---
 src/index.js | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/src/index.js b/src/index.js
index 9e527fa..67667b9 100755
--- a/src/index.js
+++ b/src/index.js
@@ -22,7 +22,7 @@ if (config.concatEventPolicy) {
         const cronTime = convertTimeToCron(gcIntervalInMin);
         logger.info(cronTime);
         const task = async () => {
-            logger.info("Compactor starting process garbage scheduled task running every x minutes");
+            logger.info("Compactor starting process garbage scheduled task running every %s minutes", gcIntervalInMin);
             try {
                 await compactor.processConsistencyAndGarbage();
             } catch(e) {
@@ -39,7 +39,7 @@ if (config.concatEventPolicy) {
             // Start Kafka consumption and pass the processMessage as a callback
             await kafka.startKafkaConsumer(compactor.processMessage);
         } catch (error) {
-            logger.error('Error starting Compactor:', error);
+            logger.error('Error starting Compactor: %o', error);
         }
     }
     // Method to stop consuming messages
@@ -48,7 +48,7 @@ if (config.concatEventPolicy) {
             await kafka.disconnect();
             logger.info('Compactor stopped Kafka consumption.');
         } catch (error) {
-            logger.error('Error stopping Compactor:', error);
+            logger.error('Error stopping Compactor:: %o', error);
         }
     }
 } else {    
@@ -70,7 +70,7 @@ if (config.concatEventPolicy) {
     
             logger.info('Compactor initialized.');
         } catch (error) {
-            logger.error('Error during compactor initialization:', error);
+            logger.error('Error during compactor initialization:: %o', error);
         }
         
         process.on('SIGTERM', () => {

From 9a65b5a6c9c0bf7e98af0d0c2a5236fa30c495e2 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 15:58:45 +0100
Subject: [PATCH 21/41] fix throw error condition

---
 src/compactor.js | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index dfa68a1..5755814 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -303,7 +303,7 @@ export class Compactor {
         let keyWithoutBucket = null;
     
         // If the split key has exactly 2 parts, extract activityId and filename
-        if (added.length === 2) {
+        if (added.length !== 2) {
             throw new Error('Key format is unexpected. Unable to extract activityId and filename.');
         }
         activityId = added[0];
@@ -329,7 +329,6 @@ export class Compactor {
             return;
         }
         const newActivityFiles= await activityState.insertOrdered([keyWithoutBucket]);
-        logger.debug(newActivityFiles);
         const sha1 = sha1sums(newActivityFiles);
         await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket, sha1);
         logger.debug(activityState);

From 552e2c3f83aec7dad7723646d323cc7849f608a2 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 15:59:12 +0100
Subject: [PATCH 22/41] bind function processMessage to compactor

---
 src/index.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/index.js b/src/index.js
index 67667b9..a2e6ff7 100755
--- a/src/index.js
+++ b/src/index.js
@@ -37,7 +37,7 @@ if (config.concatEventPolicy) {
         try {
             logger.info('Compactor starting Kafka consumption...');
             // Start Kafka consumption and pass the processMessage as a callback
-            await kafka.startKafkaConsumer(compactor.processMessage);
+            await kafka.startKafkaConsumer(compactor.processMessage.bind(compactor));
         } catch (error) {
             logger.error('Error starting Compactor: %o', error);
         }

From 0d238e5dd658ff83816c4640d60d3afe6495aaa0 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Mon, 10 Mar 2025 16:00:18 +0100
Subject: [PATCH 23/41] fix insertOrdered fonction and saveLocalFilesState
 function using that insertOrdered

---
 src/state.js | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/src/state.js b/src/state.js
index d616528..3bde685 100755
--- a/src/state.js
+++ b/src/state.js
@@ -73,7 +73,8 @@ export class ActivityCompactionState {
 	 */
 	async insertOrdered(files) {
 		let activityFiles = await this.files();
-		for(let file in files) {
+		for(let i in files) {
+			let file = files[i];
 			let positionvalue=-binarySearch(activityFiles, file, true, (a,b)=> { 
 				if(typeof a == "string" && typeof b == "string" ) {
 					return a.localeCompare(b); 
@@ -83,20 +84,21 @@ export class ActivityCompactionState {
 			})-1;
 
 			const nextposition = activityFiles.length;
-			logger.info(file);
-			logger.info("positionvalue:");
-			logger.info(positionvalue);
-			logger.info("nextposition:");
-			logger.info(nextposition);
+			logger.debug(file);
+			logger.debug("positionvalue:");
+			logger.debug(positionvalue);
+			logger.debug("nextposition:");
+			logger.debug(nextposition);
 			
 			if(positionvalue < nextposition) {
 				logger.warn("Not ordered. Should have been consumed before.")
+				activityFiles.splice(positionvalue, 0, file);
+			} else {
+				activityFiles.push(file);
 			}
-			
-			activityFiles.splice(nextposition, 0, file);
-			logger.info(activityFiles);
+			logger.debug(activityFiles);
 		}
-		return files;
+		return activityFiles;
 	}
 
 	async garbageCollect() {
@@ -466,14 +468,10 @@ export class ActivityCompactionState {
 	async #saveLocalFilesState(filesToAdd, sha1) {
 		const filesStatePath = this.#filesStateLocalPath(sha1);
 		const tmpPath = mktempPath();
-		if (this.currentSha1 !== null && this.currentSha1 !== sha1) {
-			const currentFilesStatePath = this.#filesStateLocalPath();
-			await copyNoOverwrite(currentFilesStatePath, tmpPath);
-		}
-		const stateFiles = await this.insertOrdered(filesToAdd);
+		const stateFiles = (await this.insertOrdered(filesToAdd)).join('\r\n');
 		const withFilesState = withFile(tmpPath, 'a');
 		await withFilesState(async (file) => {
-			await file.writeFile(stateFiles.join("\n"));
+			await file.writeFile(stateFiles);
 		})
 		await rename(tmpPath, filesStatePath, this.#opts.copyInsteadRename);
 	}

From 2856d2db41a3731138747e5dc3edd2f3ef8399c4 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 11 Mar 2025 09:12:49 +0100
Subject: [PATCH 24/41] fix version gestion load and save

---
 src/compactor.js | 9 +++++++--
 src/state.js     | 8 ++++----
 2 files changed, 11 insertions(+), 6 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 5755814..fd6403b 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -138,6 +138,7 @@ export class Compactor {
         let activitiesToGo = await this.#garbageCollectActivities(state, activities);
 
         this.status.total = activities.length;
+        let leastoneUpdated=false;
         for(let idx=0; idx < activities.length; idx++) {
             if (this.shouldExit) {
                 break;
@@ -154,13 +155,17 @@ export class Compactor {
             }
             const updated = await this.#updateActivityTraces(activityState);
             if (!updated) continue;
+            
+            leastoneUpdated=updated;
             await this.#distributeTrace(activityState);
-    
+
             if (activities.length % 5) {
                 await state.save();
             }
         }
-        await state.save();
+        if(leastoneUpdated) {
+            await state.save();
+        }
     }
 
     /**
diff --git a/src/state.js b/src/state.js
index 3bde685..0b2bce5 100755
--- a/src/state.js
+++ b/src/state.js
@@ -11,7 +11,7 @@ import { binarySearch, diffArray, diffSet } from './utils/misc.js';
 /**
  * @typedef SerializedCompactorState
  * @property {string} lastGC
- * @property {number} version
+ * @property {string} version
  * @property {Map<string, ActivityCompactionState>} states
  */
 
@@ -671,14 +671,14 @@ export class CompactorState {
 			}
 		}
 		this.#lastGC = serializedState.lastGC !== null ? new Date(Date.parse(serializedState.lastGC)) : null;
-		this.#version = serializedState.version !== null ? serializedState.version : null ;
+		this.#version = serializedState.version !== null ? parseInt(serializedState.version) : 0 ;
 	}
 
 	async save() {
-		this.#version = this.#version !== null ? 0 : this.#version+1;
+		this.#version =this.#version+1;
 		/** @type {SerializedCompactorState} */
 		const serializedState = {
-			version: this.#version,
+			version: this.#version.toString(),
 			states: this.#states,
 			lastGC: this.#lastGC !== null ? this.#lastGC.toISOString() : null
 		}

From 9d63c6b8c0646d35697d85938d2e7e0d2b5b0533 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 11 Mar 2025 09:54:05 +0100
Subject: [PATCH 25/41] adding checking version adding a localTempPath to copy
 minio state.json file

---
 src/state.js | 54 +++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 45 insertions(+), 9 deletions(-)

diff --git a/src/state.js b/src/state.js
index 0b2bce5..0959f78 100755
--- a/src/state.js
+++ b/src/state.js
@@ -558,11 +558,29 @@ export class CompactorState {
 	#version;
 
 	async init() {
-		let loaded = await this.#loadLocalState();
-		if (loaded) return;
-		loaded = await this.#loadRemoteState();
-		if (!loaded) {
+		logger.debug("Loading Local State...")
+		let localStateLoaded = await this.#loadLocalState(false);
+		let localStateVersion=this.#version;
+		logger.debug("Loading Remote State...");
+		let remoteStateLoaded = await this.#loadRemoteState(true);
+		let remoteStateVersion=this.#version;
+		if (!localStateLoaded && !remoteStateLoaded) {
 			logger.warn('Seems that we are running for the first time');
+			return;
+		}
+		logger.debug(`Local version : ${localStateVersion } - Remote version : ${remoteStateVersion }`)
+		if(remoteStateVersion > localStateVersion) {
+			logger.debug("The version in remote is more uptodate. Taking this version.");
+			await this.#loadRemoteState(false);
+			return;
+		} else if(remoteStateVersion < localStateVersion) {
+			logger.debug("The version in local is more uptodate. Taking this version.");
+			await this.#loadLocalState(false);
+			await this.#copyStateToRemote();
+			return;
+		} else {
+			logger.debug("The versions in local and in remote are the same.");
+			return;
 		}
 	}
 
@@ -617,10 +635,16 @@ export class CompactorState {
 	}
 
 	/**
+	 * @param {boolean } loadTemp
 	 * @return {Promise<boolean>} true if config has been loaded
 	 */
-	async #loadLocalState() {
-		const path = this.#localPath;
+	async #loadLocalState(loadTemp) {
+		let path;
+		if(loadTemp) {
+			path = this.#localTempPath;
+		} else {
+			path = this.#localPath;
+		}
 		const withState = withFile(path);
 		const result = await withState(async (file) => {
 			const content = await file.readFile('utf-8');
@@ -638,13 +662,25 @@ export class CompactorState {
 		return path;
 	}
 
+	get #localTempPath () {
+		const path = join(this.#opts.localStatePath, `temp_${STATE_FILENAME}`);
+		return path;
+	}
+
 	/**
+	 * @param {boolean} loadTemp
 	 * @return {Promise<boolean>} true if config has been loaded
 	 */
-	async #loadRemoteState() {
+	async #loadRemoteState(loadTemp) {
+		let path;
+		if(loadTemp) {
+			path = this.#localTempPath;
+		} else {
+			path = this.#localPath;
+		}
 		try {
-			await this.#minio.copyFromRemoteFile(this.#remotePath, this.#localPath);
-			return this.#loadLocalState();
+			await this.#minio.copyFromRemoteFile(this.#remotePath, path);
+			return this.#loadLocalState(loadTemp);
 		} catch (e) {
 			logger.warn(e);
 		}

From 8746ec9df47eed837d1ffaefcaa33a5fe1ba50a5 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 11 Mar 2025 18:42:49 +0100
Subject: [PATCH 26/41] fix dev and dev profiling

---
 package.json | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/package.json b/package.json
index 7978df9..0652993 100755
--- a/package.json
+++ b/package.json
@@ -5,8 +5,8 @@
   "type": "module",
   "scripts": {
     "start": "node ./src/index.js",
-    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js",
-    "start:debug": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof ./src/index.js"
+    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --max-old-space-size=1024 ./src/index.js",
+    "dev:profiling": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js"
   },
   "engines": {
     "node": ">=20 <24"

From cba4493d72c6aa3d45087ead4e113fa4f43615df Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 18 Mar 2025 11:44:56 +0100
Subject: [PATCH 27/41] fix already consumed

---
 src/compactor.js | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index fd6403b..25cdc92 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -225,14 +225,23 @@ export class Compactor {
         const { added: filesToAdd } = diffArray(activityFiles, traceFiles);
         const nowDate = now();
         const elapsedTime = duration(activityState.lastUpdate, nowDate);
-        if (filesToAdd.length < this.#opts.batchSize && elapsedTime < this.#opts.maxDelay) {
+        let filesToConsume=[];
+        for(let file in filesToAdd) {
+            logger.debug(file);
+            if(activityFiles.includes(file)) {
+                logger.warn("Already consumed: %s", file);
+            } else {
+                filesToConsume.push(file);
+            }
+        }
+        if (filesToConsume.length < this.#opts.batchSize && elapsedTime < this.#opts.maxDelay) {
             const durationStr = formatDuration(elapsedTime);
-            logger.debug(`Update postponed elapsedTime=%s, batchSize=%d for activity %s`, durationStr, filesToAdd.length, activityState.activityId);
+            logger.debug(`Update postponed elapsedTime=%s, batchSize=%d for activity %s`, durationStr, filesToConsume.length, activityState.activityId);
             return false;
         }
 
         logger.info(`Compacting activity %s`, activityState.activityId);
-        await activityState.update(filesToAdd, nowDate, sha1);
+        await activityState.update(filesToConsume, nowDate, sha1);
         return true;
     }
 

From e23f07443abad87c50a0127cf08648d24e15827a Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 18 Mar 2025 11:55:01 +0100
Subject: [PATCH 28/41] fix case version = NaN

---
 src/state.js | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/state.js b/src/state.js
index 0959f78..79a8cff 100755
--- a/src/state.js
+++ b/src/state.js
@@ -707,7 +707,7 @@ export class CompactorState {
 			}
 		}
 		this.#lastGC = serializedState.lastGC !== null ? new Date(Date.parse(serializedState.lastGC)) : null;
-		this.#version = serializedState.version !== null ? parseInt(serializedState.version) : 0 ;
+		this.#version = serializedState.version !== null ? (serializedState.version !== "NaN" ? parseInt(serializedState.version) : 0) : 0 ;
 	}
 
 	async save() {

From 60b32955951a637cb98043454b506556c2b9f17b Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 18 Mar 2025 12:01:47 +0100
Subject: [PATCH 29/41] fix using index

---
 src/compactor.js | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/compactor.js b/src/compactor.js
index 25cdc92..ad5bc03 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -226,7 +226,8 @@ export class Compactor {
         const nowDate = now();
         const elapsedTime = duration(activityState.lastUpdate, nowDate);
         let filesToConsume=[];
-        for(let file in filesToAdd) {
+        for(let index in filesToAdd) {
+            let file=filesToAdd[index];
             logger.debug(file);
             if(activityFiles.includes(file)) {
                 logger.warn("Already consumed: %s", file);

From 4416888b61b4bd3b20342c039dcf7d9d9a330983 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Tue, 18 Mar 2025 12:07:02 +0100
Subject: [PATCH 30/41] fix length == 0

---
 src/compactor.js | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index ad5bc03..75a39e0 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -242,8 +242,12 @@ export class Compactor {
         }
 
         logger.info(`Compacting activity %s`, activityState.activityId);
-        await activityState.update(filesToConsume, nowDate, sha1);
-        return true;
+        if(filesToConsume.length > 0) {
+            await activityState.update(filesToConsume, nowDate, sha1);
+            return true;
+        } else {
+            return false;
+        }
     }
 
     /**

From 14087594103df0e9ffaeff2caa4c0b23b21ad271 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Wed, 26 Mar 2025 18:36:21 +0100
Subject: [PATCH 31/41] .clinic/

---
 .gitignore   |  1 +
 Dockerfile   |  2 ++
 package.json | 10 +++++++---
 3 files changed, 10 insertions(+), 3 deletions(-)

diff --git a/.gitignore b/.gitignore
index 4e4670d..bbf9a99 100755
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
 /node_modules/
 /tmp/*
+.clinic/
 
 # Config files
 config-test.js
diff --git a/Dockerfile b/Dockerfile
index 7cae36d..07a388a 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -13,6 +13,8 @@ WORKDIR /home/node/app
 
 COPY --chown=node:node package*.json ./
 
+RUN npm install -g clinic
+
 USER node
 
 RUN npm install
diff --git a/package.json b/package.json
index 0652993..6cebf0f 100755
--- a/package.json
+++ b/package.json
@@ -4,9 +4,13 @@
   "private": true,
   "type": "module",
   "scripts": {
-    "start": "node ./src/index.js",
-    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --max-old-space-size=1024 ./src/index.js",
-    "dev:profiling": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js"
+    "start": "node ./src/index.js $NODE_ARGS",
+    "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --max-old-space-size=1024 ./src/index.js $NODE_ARGS",
+    "dev:profiling": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js $NODE_ARGS",
+    "dev:clinic:doctor": "clinic doctor $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS",
+    "dev:clinic:bubbleprof": "clinic bubbleprof $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS",
+    "dev:clinic:flame": "clinic flame $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS",
+    "dev:clinic:heapprofiler": "clinic heapprofiler $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS"
   },
   "engines": {
     "node": ">=20 <24"

From 32d8da930b450e854a1eb1d49e7a44b86bdf995f Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 27 Mar 2025 10:11:22 +0100
Subject: [PATCH 32/41] remove profiling part

---
 src/profiling.js | 61 +-----------------------------------------------
 1 file changed, 1 insertion(+), 60 deletions(-)

diff --git a/src/profiling.js b/src/profiling.js
index f507ef6..00e3cca 100755
--- a/src/profiling.js
+++ b/src/profiling.js
@@ -7,63 +7,4 @@ import ms from 'ms';
 
 if(process.env.NODE_ENV == "development" && process.env.ENABLE_DEBUG_PROFILING == "true") {
   logger.info("Profiling in progress...");
-  const __filename = fileURLToPath(import.meta.url);
-  const __dirname = path.dirname(__filename);
-  const profilingFolder =process.env.PROFILING_FOLDER || path.join(__dirname, '../../profiling');
-
-  function heapdump() {
-    logger.info(`schedule task for profiling running...`);
-    //let filename=`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`;
-    let filename=`${profilingFolder}/${v8.writeHeapSnapshot()}`;
-    logger.info(`Saved heapdump into ${v8.writeHeapSnapshot(filename)}`);
-    setTimeout(heapdump, ms("30min"));
-  }
-  setTimeout(heapdump, ms("1min"));
-
-  function memoryAndCPUUsage() {
-    let memoryUsage=process.memoryUsage();
-    let cpuUsage=process.cpuUsage();
-    logger.info(`MEMORY USAGE : ${JSON.stringify(memoryUsage)}`);
-    logger.info(`CPU USAGE : ${JSON.stringify(cpuUsage)}`);
-    const heapUsed = memoryUsage.heapUsed / 1024 / 1024;
-    const arrayBuffers = memoryUsage.arrayBuffers / 1024 / 1024;
-    const rss = memoryUsage.rss / 1024 / 1024;
-    const external = memoryUsage.external / 1024 / 1024;
-    const heapTotal = memoryUsage.heapTotal / 1024 / 1024;
-    logger.info(`MEMORY USAGE : This app is currently using ${Math.floor(heapUsed)} MB of memory (heapTotal : ${Math.floor(heapTotal)} MB - rss : ${Math.floor(rss)} MB - arrayBuffers : ${Math.floor(arrayBuffers)} MB -external : ${Math.floor(external)} MB).`);
-    const user = cpuUsage.user / 1000;
-    const system = cpuUsage.system / 1000;
-    logger.info(`CPU USAGE : This app is currently using user ${Math.floor(user)}s and system ${Math.floor(system)}s`);
-    setTimeout(memoryAndCPUUsage, ms("1min"));
-  }
-  setTimeout(memoryAndCPUUsage, ms("1min"));
-}
-
-/*
-import {
-  Worker,
-  isMainThread,
-  parentPort,
-} from 'node:worker_threads';
-
-if (isMainThread) {
-  const worker = new Worker(__filename);
-
-  worker.once('message', (filename) => {
-    logger.info(`worker heapdump: ${filename}`);
-    // Now get a heapdump for the main thread.
-    logger.info(`main thread heapdump: ${v8.writeHeapSnapshot(`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`)}`);
-  });
-
-  // Tell the worker to create a heapdump.
-  worker.postMessage('heapdump');
-} else {
-  parentPort.once('message', (message) => {
-    if (message === 'heapdump') {
-      // Generate a heapdump for the worker
-      // and return the filename to the parent.
-      parentPort.postMessage(v8.writeHeapSnapshot(`${profilingFolder}/Heap.${now().toISOString()}.heapsnapshot`));
-    }
-  });
-}
-*/
\ No newline at end of file
+}
\ No newline at end of file

From 9e709c015185d35eb2525c557eebe21d9f907d87 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 27 Mar 2025 18:02:19 +0100
Subject: [PATCH 33/41] remove unused functions

---
 src/minio.js | 65 +---------------------------------------------------
 1 file changed, 1 insertion(+), 64 deletions(-)

diff --git a/src/minio.js b/src/minio.js
index 8d8ef40..b79e9a2 100755
--- a/src/minio.js
+++ b/src/minio.js
@@ -124,67 +124,6 @@ export class MinioClient {
         return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath, metadata);
 	}
 
-    /**
-     * 
-     * @param {string} objectName 
-     * @param {string} uploadId
-     * @returns {Promise<void>}
-     */
-    async abortMultipartUpload(objectName, uploadId) {
-        await this.#minio.abortMultipartUpload(this.#opts.bucket, objectName, uploadId);
-    }
-
-    /**
-     * 
-     * @param {MultipartUploadResult[]} list
-     * @returns {Promise<void>}
-     */
-    async abortMultipartUploads(list) {
-        for (const item of list) {
-            try {
-                await this.#minio.abortMultipartUpload(this.#opts.bucket, item.key, item.uploadId);
-                logger.info(`Aborted upload: ${item.key} (Upload ID: ${item.uploadId})`);
-            } catch (error) {
-                logger.error(`Failed to abort ${item.key}:`, error);
-            }
-        }
-    }
-    
-
-    /**
-     * 
-     * @returns {Promise<MultipartUploadResult[]>}
-     */
-    async listMultipartUploads() {
-        return new Promise((resolve, reject) => {
-          const uploads = [];
-          
-          const stream = this.#minio.listIncompleteUploads(this.#opts.bucket, "", true);
-          
-          stream.on("data", (obj) => {
-            uploads.push(obj);
-            logger.info("Active Upload:", obj);
-          });
-      
-          stream.on("end", () => {
-            resolve(uploads);
-          });
-      
-          stream.on("error", (err) => {
-            reject(err);
-          });
-        });
-    }
-
-    async listV2MultipartUploads() {
-        this.#minio.listIncompleteUploads(this.#opts.bucket, "", true)
-          .on("data", async (obj) => {
-            logger.info(obj);
-            await this.#minio.abortMultipartUpload(this.#opts.bucket, obj.key, obj.uploadId);
-          })
-          .on("error", (err) => logger.error(err));
-      }
-
     /**
      * 
      * @param {string} file 
@@ -249,9 +188,7 @@ export class MinioClient {
             destinationPath,
             `/${this.#opts.bucket}/${sourcePath}`
           );
-      }
-      
-
+    }
 }
 
 function streamToString(stream) {

From b5f30474f34b9adedf86184c03c163c9772a288d Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 27 Mar 2025 18:02:55 +0100
Subject: [PATCH 34/41] rethrow errors

---
 src/compactor.js | 3 +++
 src/kafka.js     | 3 +++
 src/state.js     | 3 +++
 3 files changed, 9 insertions(+)

diff --git a/src/compactor.js b/src/compactor.js
index 75a39e0..21b6871 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -65,6 +65,7 @@ export class Compactor {
             } catch (e) {
                 logger.error(e);
                 logger.error('####### ERROR WHILE PROCESSING THE TRACES. !!!!!!');
+                throw e;
             }
             this.status.processing = false;
         } else {
@@ -196,6 +197,7 @@ export class Compactor {
                 } catch(error) {
                     logger.error('Could not remove activity: %s', activityId);
                     logger.error(error);
+                    throw error;
                 }
             }
             logger.info('Activities to removed OK.');
@@ -269,6 +271,7 @@ export class Compactor {
         } catch (error) {
             logger.error("Copy failed:");
             logger.error(error);
+            throw error;
         }
         logger.info(`Copied compacted file for activity %s`, activityState.activityId);
     }
diff --git a/src/kafka.js b/src/kafka.js
index bfcf13f..4b54870 100755
--- a/src/kafka.js
+++ b/src/kafka.js
@@ -28,6 +28,7 @@ export class KafkaClient {
             logger.info(`Subscribed to topic: ${this.topic}`);
         } catch (error) {
             console.error('Error connecting to Kafka:', error);
+            throw error;
         }
     }
 
@@ -60,6 +61,7 @@ export class KafkaClient {
             });
         } catch (error) {
             console.error('Error starting Kafka messages consumer :', error);
+            throw error;
         }
     }
 
@@ -70,6 +72,7 @@ export class KafkaClient {
             logger.info('Kafka consumer disconnected');
         } catch (error) {
             console.error('Error disconnecting from Kafka:', error);
+            throw error;
         }
     }
 }
diff --git a/src/state.js b/src/state.js
index 79a8cff..f64ce95 100755
--- a/src/state.js
+++ b/src/state.js
@@ -626,6 +626,7 @@ export class CompactorState {
 			} catch (error) {
 				logger.error('Could not garbage collect: %s', activity.activityId);
 				logger.error(error);
+				throw error;
 			}
 		}
 		const finishTime = now();
@@ -683,6 +684,7 @@ export class CompactorState {
 			return this.#loadLocalState(loadTemp);
 		} catch (e) {
 			logger.warn(e);
+			throw e;
 		}
 		return false;
 	}
@@ -704,6 +706,7 @@ export class CompactorState {
 			} catch (error) {
 				logger.error('Could not initialize activity: ', activity.activityId);
 				logger.error(error);
+				throw error;
 			}
 		}
 		this.#lastGC = serializedState.lastGC !== null ? new Date(Date.parse(serializedState.lastGC)) : null;

From d60dba02e766359b9261c3beb5fbc5c9c9aa6a0b Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Thu, 27 Mar 2025 18:26:04 +0100
Subject: [PATCH 35/41] fix using copy withinMinio  to remove state path to
 output folder path

---
 src/compactor.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 21b6871..b07e759 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -265,8 +265,8 @@ export class Compactor {
                 "Content-Type": "application/json",
                 "Version": "1"
             };
-            await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
-            //await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
+            //await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
+            await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
             logger.info("Object copied successfully!");
         } catch (error) {
             logger.error("Copy failed:");

From b0c518d0de323912e6893b58f2ce3704315e57cd Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <62715763+jujusb@users.noreply.github.com>
Date: Tue, 1 Apr 2025 12:37:54 +0200
Subject: [PATCH 36/41] fix as state files as .json + adding metadata to those
 files

---
 src/compactor.js |  4 ++--
 src/state.js     | 34 +++++++++++++++++++++++++++++-----
 2 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index b07e759..21b6871 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -265,8 +265,8 @@ export class Compactor {
                 "Content-Type": "application/json",
                 "Version": "1"
             };
-            //await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
-            await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
+            await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
+            //await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
             logger.info("Object copied successfully!");
         } catch (error) {
             logger.error("Copy failed:");
diff --git a/src/state.js b/src/state.js
index f64ce95..b885e62 100755
--- a/src/state.js
+++ b/src/state.js
@@ -424,7 +424,7 @@ export class ActivityCompactionState {
 	 */
 	#stateLocalPath(sha1) {
 		sha1 = sha1 || this.currentSha1;
-		const path = join(this.#opts.localStatePath, this.activityId, `${sha1}-state.txt`);
+		const path = join(this.#opts.localStatePath, this.activityId, `${sha1}-state.json`);
 		return path;
 	}
 
@@ -456,7 +456,7 @@ export class ActivityCompactionState {
 	 */
 	#stateRemotePath(sha1) {
 		sha1 = sha1 || this.currentSha1;
-		return `${this.#opts.remoteStatePath}/${this.activityId}/${sha1}-state.txt`;
+		return `${this.#opts.remoteStatePath}/${this.activityId}/${sha1}-state.json`;
 	}
 
 	/**
@@ -493,7 +493,11 @@ export class ActivityCompactionState {
 	async #copyToRemoteState(sha1) {
 		const localPath = this.#stateLocalPath(sha1);
 		const remotePath = this.#stateRemotePath(sha1);
-		await this.#minio.copyToRemoteFile(localPath, remotePath);
+		let metadata={
+			"Content-Type": "application/json",
+			"Version": "1"
+		};
+		await this.#minio.copyToRemoteFile(localPath, remotePath, metadata);
 	}
 
 	async checkConsistency() {
@@ -505,6 +509,27 @@ export class ActivityCompactionState {
 		let consistent = true;
 		const localStatePath = this.#stateLocalPath();
 		const localFilesStatePath = this.#filesStateLocalPath();
+
+		const remoteStatePath = this.#stateRemotePath();
+		const remoteFilesStatePath = this.#filesStateRemotePath();
+
+		if (! await fileExists(localStatePath)) {
+			let previousLocalStatePath=localStatePath.replace(".json",".txt");
+			if(await fileExists(previousLocalStatePath)) {
+				try {
+					await rename(previousLocalStatePath, localStatePath, this.#opts.copyInsteadRename);
+					let metadata= {
+						"Content-Type": "application/json",
+						"Version": "1"
+					};
+					await this.#minio.copyToRemoteFile(localStatePath, remoteStatePath, metadata);
+				} catch(error) {
+				    logger.error("Copy failed:");
+					logger.error(error);
+					throw error;
+				}
+			}
+		}
 		if (! await fileExists(localStatePath)) {
 			logger.warn('Local state file for activity \'%s\' not found: %s', this.activityId, localStatePath);
 			consistent = false;
@@ -513,8 +538,7 @@ export class ActivityCompactionState {
 			logger.warn('Local files state for activity \'%s\' not found: %s', this.activityId, localFilesStatePath);
 			consistent = false;
 		}
-		const remoteStatePath = this.#stateRemotePath();
-		const remoteFilesStatePath = this.#filesStateRemotePath();
+		
 		if (! await this.#minio.fileExists(remoteStatePath)) {
 			logger.warn('Remote state file for activity \'%s\' not found: %s', this.activityId, remoteStatePath);
 			consistent = false;

From ef1e610f1ef7b5a8cf397d40f4e1626986afe5c2 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <62715763+jujusb@users.noreply.github.com>
Date: Tue, 1 Apr 2025 13:00:20 +0200
Subject: [PATCH 37/41] fix compactor to distribute trace copying it within
 minio

---
 src/compactor.js | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 21b6871..6db69ac 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -261,15 +261,15 @@ export class Compactor {
         const remoteStatePath = activityState.remoteStatePath;
         const remotePath = activityState.remoteOutputPath;
         try {
-            const metadata = {
-                "Content-Type": "application/json",
-                "Version": "1"
-            };
-            await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
-            //await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
+            //const metadata = {
+            //    "Content-Type": "application/json",
+            //    "Version": "1"
+            //};
+            //await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata);
+            await this.#minio.copyWithinMinIO(remoteStatePath, remotePath);
             logger.info("Object copied successfully!");
         } catch (error) {
-            logger.error("Copy failed:");
+            logger.error("Copy compacted file failed:");
             logger.error(error);
             throw error;
         }

From dc2c68321515cc43a46a1507d509a95c45fc6bf0 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <62715763+jujusb@users.noreply.github.com>
Date: Tue, 1 Apr 2025 14:30:40 +0200
Subject: [PATCH 38/41] fix copy from remote

---
 src/state.js | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/state.js b/src/state.js
index b885e62..e15646b 100755
--- a/src/state.js
+++ b/src/state.js
@@ -528,6 +528,22 @@ export class ActivityCompactionState {
 					logger.error(error);
 					throw error;
 				}
+			} else {
+				let previousRemoteStatePath=remoteStatePath.replace(".json",".txt");
+				if(await this.#minio.fileExists(previousRemoteStatePath)) {
+					try {
+						await this.#minio.copyFromRemoteFile(previousRemoteStatePath, localStatePath);
+						let metadata= {
+							"Content-Type": "application/json",
+							"Version": "1"
+						};
+						await this.#minio.copyToRemoteFile(localStatePath, remoteStatePath, metadata);
+					} catch(error) {
+						logger.error("Copy failed:");
+						logger.error(error);
+						throw error;
+					}
+				}
 			}
 		}
 		if (! await fileExists(localStatePath)) {

From d2649b64b5f440e0deed2ca767f44c2ccd770f7e Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <62715763+jujusb@users.noreply.github.com>
Date: Fri, 11 Apr 2025 18:42:33 +0200
Subject: [PATCH 39/41] fix not found local and remote states

---
 src/state.js | 39 ++++++++++++++++++++++-----------------
 1 file changed, 22 insertions(+), 17 deletions(-)

diff --git a/src/state.js b/src/state.js
index e15646b..c8aa785 100755
--- a/src/state.js
+++ b/src/state.js
@@ -680,22 +680,28 @@ export class CompactorState {
 	 * @return {Promise<boolean>} true if config has been loaded
 	 */
 	async #loadLocalState(loadTemp) {
-		let path;
-		if(loadTemp) {
-			path = this.#localTempPath;
-		} else {
-			path = this.#localPath;
-		}
-		const withState = withFile(path);
-		const result = await withState(async (file) => {
-			const content = await file.readFile('utf-8');
-			await this.#initState(content);
-			return true;
-		}, false);
-		if (result !== undefined) {
-			return result;
+		try {
+			let path;
+			if(loadTemp) {
+				path = this.#localTempPath;
+			} else {
+				path = this.#localPath;
+			}
+			const withState = withFile(path);
+			const result = await withState(async (file) => {
+				const content = await file.readFile('utf-8');
+				await this.#initState(content);
+				return true;
+			}, false);
+			if (result !== undefined) {
+				return result;
+			}
+			return false;
+		} catch(e) {
+			logger.warn(e);
+			return false;
 		}
-		return false;
+		
 	}
 
 	get #localPath () {
@@ -724,9 +730,8 @@ export class CompactorState {
 			return this.#loadLocalState(loadTemp);
 		} catch (e) {
 			logger.warn(e);
-			throw e;
+			return false;
 		}
-		return false;
 	}
 
 	get #remotePath() {

From 81e9561531a400faf8fda2953bc9e36e70fcfcc8 Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Wed, 18 Jun 2025 17:49:53 +0200
Subject: [PATCH 40/41] adding manual and limesurvey activities to compactor

---
 src/compactor.js | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/compactor.js b/src/compactor.js
index 6db69ac..fccc977 100755
--- a/src/compactor.js
+++ b/src/compactor.js
@@ -88,7 +88,7 @@ export class Compactor {
     async #checkConsistency() {
         let state = await getState(this.#opts, this.#minio);
 
-        let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] });
+        let activities = await this.#simva.getActivities({ type: ['gameplay', 'manual', 'limesurvey', 'miniokafka', 'rageminio'] });
 
         logger.info(`Known %d activities, received %d`, state.size, activities.length);
 
@@ -132,7 +132,7 @@ export class Compactor {
     async #compactActivities() {
         let state = await getState(this.#opts, this.#minio);
 
-        let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] });
+        let activities = await this.#simva.getActivities({ type: ['gameplay', 'manual', 'limesurvey', 'miniokafka', 'rageminio'] });
 
         logger.info(`Known %d activities, received %d`, state.size, activities.length);
 

From 8a143b7cfcc60d7280f8fa2ebc3bbc9ce090434f Mon Sep 17 00:00:00 2001
From: Julio SANTILARIO BERTHILIER <jsantilariob@gmail.com>
Date: Sun, 26 Jul 2026 23:47:58 +0200
Subject: [PATCH 41/41] update Dockerfile multistages

---
 Dockerfile | 40 +++++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 15 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 07a388a..330de44 100755
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,30 +1,40 @@
-FROM node:22.14.0-bullseye
+# ---------------------------------------
+# Base
+# ---------------------------------------
+FROM timbru31/node-alpine-git:22 AS base
 
-# Install ca-certificates and update them
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    ca-certificates && \
-    update-ca-certificates && \
-    apt-get clean && rm -rf /var/lib/apt/lists/*
+WORKDIR /app
 
-RUN mkdir -p /home/node/app/node_modules && chown -R node:node /home/node/app
+# Common dependencies
+RUN apk add --no-cache curl ca-certificates bash
 
-# Set the working directory
-WORKDIR /home/node/app
+FROM base AS deps
 
-COPY --chown=node:node package*.json ./
+COPY --chown=node:node package.json package-lock.json ./
+RUN --mount=type=cache,target=/root/.npm npm ci
+
+FROM deps AS dev
 
 RUN npm install -g clinic
+RUN chown -R node:node /app/node_modules
 
 USER node
+EXPOSE 3050
+
+# Default CMD, can be overridden by docker-compose
+CMD [ "npm", "run", "dev" ]
 
-RUN npm install
+FROM node:22-alpine AS prod
 
-# Copy the current directory contents into the container at /app
-COPY --chown=node:node . .
+WORKDIR /app
 
-RUN mkdir -p /home/node/logs 
+COPY --chown=node:node --from=deps /app/node_modules ./node_modules
+COPY --chown=node:node ./src ./src
+COPY --chown=node:node ./jsconfig.json ./jsconfig.json
 
-RUN chown -R node:node /home/node/logs
+USER node
 
+# Make port 3050 available to the world outside this container
+EXPOSE 3050
 
 CMD [ "npm", "start" ]
\ No newline at end of file