diff --git a/.dockerignore b/.dockerignore new file mode 100755 index 0000000..df3c4b1 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,7 @@ +node_modules/ +manual-scripts/ +.github +Dockerfile +.dockerignore +*.heapsnapshot +*.log \ No newline at end of file diff --git a/.gitignore b/.gitignore old mode 100644 new mode 100755 index dcaac7f..bbf9a99 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,8 @@ /node_modules/ /tmp/* +.clinic/ # Config files config-test.js +*.log +*.heapsnapshot \ No newline at end of file diff --git a/DemoCA.crt b/DemoCA.crt new file mode 100755 index 0000000..eec572b --- /dev/null +++ b/DemoCA.crt @@ -0,0 +1,20 @@ +-----BEGIN CERTIFICATE----- +MIIDSzCCAjOgAwIBAgIUCmGeUFwNiGdJZLwpkevTf29wjrowDQYJKoZIhvcNAQEL +BQAwFjEUMBIGA1UEAwwLRWFzeS1SU0EgQ0EwHhcNMTkwNjI1MTk0NjA4WhcNMjkw +NjIyMTk0NjA4WjAWMRQwEgYDVQQDDAtFYXN5LVJTQSBDQTCCASIwDQYJKoZIhvcN +AQEBBQADggEPADCCAQoCggEBANKAEz3dPmYJvwjhSBssR0jP5JK7CO8iOqzhcqzy +cFb05hr9unPtPS76lrEEkYSgklwW5X4aVTOfqrJkc0RyoLwyYjmPei45xRiFeTfu +FA1cAwbRAriTuOySvp8U0H/J8WDCNrNrhDags9wzbtBZuTJt1UCNYXYJyHMF2NE3 +7fO86x3AVW30v0iiHa00ev2Fk5YZkKUYuL0Dv2Ul3SbWFRq3hKjFypkOK+Iur/Ye +wW7XLAoVatOvlFZCmQ+kD59b0L9QCXqDU2odlwzgNrMpczfTyuzJKQyxm0iLJgxW +QvC329dkjyq/5RR7RvWjpjzyAopXMo5ZE4CWgHQz97Rfhv8CAwEAAaOBkDCBjTAd +BgNVHQ4EFgQUNOD0d021fHYcir4UebCMny2eQVAwUQYDVR0jBEowSIAUNOD0d021 +fHYcir4UebCMny2eQVChGqQYMBYxFDASBgNVBAMMC0Vhc3ktUlNBIENBghQKYZ5Q +XA2IZ0lkvCmR69N/b3COujAMBgNVHRMEBTADAQH/MAsGA1UdDwQEAwIBBjANBgkq +hkiG9w0BAQsFAAOCAQEAFwfZVGQnd1ReZeL4uyvS2ZVI3U2GYFGrdxOiC+1nt3TG +QF5W+4RN2394Mv1DaUIjtzAuKanR73qyiVjxkFetsSj4dItzbZQBCpRT32E1EHb2 +aFXbTJ76JGPHed0wmAdCxA+zlEjKZr7B73sS2GPfSLJw3CsQYMtVSCXLCox4DUrW +yOlF2kgO1OGI4kPbtw+CKWwCh0x0StckYo5tI8MF1U5UBtAXzPJ2pXyuc7t0Hejy +MMKILaiIrqCtK/9BdQWW3iqDhhcLX6VrPb33QNA7PV9UBVAfBPPIejDLSv1aS/X9 +SuegPnlhRzBl1PL0zAMlEGI9GR3HruCJoDKo9MlR7g== +-----END CERTIFICATE----- diff --git a/Dockerfile b/Dockerfile new file mode 100755 index 0000000..330de44 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,40 @@ +# --------------------------------------- +# Base +# --------------------------------------- +FROM timbru31/node-alpine-git:22 AS base + +WORKDIR /app + +# Common dependencies +RUN apk add --no-cache curl ca-certificates bash + +FROM base AS deps + +COPY --chown=node:node package.json package-lock.json ./ +RUN --mount=type=cache,target=/root/.npm npm ci + +FROM deps AS dev + +RUN npm install -g clinic +RUN chown -R node:node /app/node_modules + +USER node +EXPOSE 3050 + +# Default CMD, can be overridden by docker-compose +CMD [ "npm", "run", "dev" ] + +FROM node:22-alpine AS prod + +WORKDIR /app + +COPY --chown=node:node --from=deps /app/node_modules ./node_modules +COPY --chown=node:node ./src ./src +COPY --chown=node:node ./jsconfig.json ./jsconfig.json + +USER node + +# Make port 3050 available to the world outside this container +EXPOSE 3050 + +CMD [ "npm", "start" ] \ No newline at end of file diff --git a/README.md b/README.md old mode 100644 new mode 100755 diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100755 index 0000000..9f22823 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,56 @@ +version: '3' + +services: + simva-trace-allocator: + build: ./ + command: [ "npm", "run", "dev" ] + stdin_open: true + tty: true + environment: + #DEBUG + DEBUG: true + #LOG + LOG_LEVEL: debug + LOG_FOLDER: /home/node/logs + #NODE + NODE_ENV: development + NODE_EXTRA_CA_CERTS: /home/node/app/DemoCA.crt + #TRACE ALLOCATOR + CONCAT_EVENT_POLICY: true + BATCH_SIZE: 100 + MAX_DELAY: 5min + REFRESH_INTERVAL: 4h + LOCAL_STATE: /data + REMOTE_STATE: state + REMOVE_DRY_RUN: true + GC_INTERVAL: 2h + COPY_INSTEAD_RENAME: true + TRY_RECOVERY: true + #MINIO + MINIO_HOST: minio-api.external.test + MINIO_SSL: true + MINIO_PORT: 443 + MINIO_ACCESS_KEY: minio + MINIO_SECRET_KEY: secret + MINIO_BUCKET: traces + MINIO_TOPICS_DIR: kafka-topics + MINIO_TRACES_TOPIC: traces + MINIO_OUTPUTS_DIR: outputs + MINIO_TRACES_FILE: traces.json + #SIMVA API + SIMVA_HOST: simva-api.external.test + SIMVA_PROTOCOL: https + SIMVA_PORT: 443 + SIMVA_USER: admin + SIMVA_PASSWORD: password + SIMVA_WAIT_TIME: 10 + SIMVA_MAX_RETRIES: 20 + #KAFKA + SIMVA_KAFKA_CLIENTID: simva_trace_allocator + SIMVA_KAFKA_BROKER: kafka1.internal.test:9092 + SIMVA_KAFKA_GROUPID: simva_trace_allocator + SIMVA_KAFKA_MINIO_TOPIC: minio-events + hostname: simva-trace-allocator.internal.test + volumes: + - ./:/home/node/app + - /home/node/app/node_modules #exclude node_modules \ No newline at end of file diff --git a/docker-startup.sh b/docker-startup.sh deleted file mode 100644 index 37a6e80..0000000 --- a/docker-startup.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -[[ "${DEBUG:-false}" == "true" ]] && set -x - -if [[ ! -d "node_modules" ]]; then - npm install -fi - -npm start \ No newline at end of file diff --git a/jsconfig.json b/jsconfig.json old mode 100644 new mode 100755 diff --git a/logs/.gitignore b/logs/.gitignore new file mode 100755 index 0000000..2191032 --- /dev/null +++ b/logs/.gitignore @@ -0,0 +1,4 @@ +# Ignore all files in this dir... +* +# ... except for this one. +!.gitignore \ No newline at end of file diff --git a/manual-scripts/compare-trace-one-folder.sh b/manual-scripts/compare-trace-one-folder.sh old mode 100644 new mode 100755 diff --git a/manual-scripts/compare-trace.sh b/manual-scripts/compare-trace.sh old mode 100644 new mode 100755 diff --git a/manual-scripts/concat-json-one-folder.sh b/manual-scripts/concat-json-one-folder.sh old mode 100644 new mode 100755 diff --git a/manual-scripts/concat-json-parallel.sh b/manual-scripts/concat-json-parallel.sh old mode 100644 new mode 100755 diff --git a/manual-scripts/concat-json.sh b/manual-scripts/concat-json.sh old mode 100644 new mode 100755 diff --git a/manual-scripts/convert-json.sh b/manual-scripts/convert-json.sh old mode 100644 new mode 100755 diff --git a/package-lock.json b/package-lock.json old mode 100644 new mode 100755 index e88ec2d..7d5dff3 --- a/package-lock.json +++ b/package-lock.json @@ -10,17 +10,20 @@ "dependencies": { "dotenv": "^16.3.1", "kafkajs": "^2.2.4", - "minio": "8.0.1", - "pino": "^8.16.2", + "minio": "8.0.4", + "ms": "^2.1.3", + "node-cron": "3.0.3", + "pino": "^9.6.0", "tinyduration": "^3.3.0", "wretch": "^2.7.0" }, "devDependencies": { "@types/node": "^20.9.0", - "pino-pretty": "^10.2.3" + "nodemon": "^3.1.9", + "pino-pretty": "^13.0.0" }, "engines": { - "node": ">=20 <21" + "node": ">=20 <24" } }, ".yalc/dayjs-es": { @@ -44,15 +47,18 @@ "license": "(Unlicense OR Apache-2.0)", "optional": true }, - "node_modules/abort-controller": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/abort-controller/-/abort-controller-3.0.0.tgz", - "integrity": "sha512-h8lQ8tacZYnR3vNQTgibj+tODHI5/+l06Au2Pcriv/Gmet0eaj4TwWH41sO9wnHDiQsEj19q0drzdWdeAHtweg==", + "node_modules/anymatch": { + "version": "3.1.3", + "resolved": "https://registry.npmjs.org/anymatch/-/anymatch-3.1.3.tgz", + "integrity": "sha512-KMReFUr0B4t+D+OBkjR3KYqvocp2XaSzO55UcB6mgQMd3KbcE+mWTyvVV7D/zsdEbNnV6acZUutkiHQXvTr1Rw==", + "dev": true, + "license": "ISC", "dependencies": { - "event-target-shim": "^5.0.0" + "normalize-path": "^3.0.0", + "picomatch": "^2.0.4" }, "engines": { - "node": ">=6.5" + "node": ">= 8" } }, "node_modules/async": { @@ -89,24 +95,18 @@ "integrity": "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw==", "dev": true }, - "node_modules/base64-js": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz", - "integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] + "node_modules/binary-extensions": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", + "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } }, "node_modules/block-stream2": { "version": "2.1.0", @@ -116,13 +116,17 @@ "readable-stream": "^3.4.0" } }, - "node_modules/brace-expansion": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", - "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "node_modules/braces": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", + "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", "dev": true, + "license": "MIT", "dependencies": { - "balanced-match": "^1.0.0" + "fill-range": "^7.1.1" + }, + "engines": { + "node": ">=8" } }, "node_modules/browser-or-node": { @@ -131,29 +135,6 @@ "integrity": "sha512-8CVjaLJGuSKMVTxJ2DpBl5XnlNDiT4cQFeuCJJrvJmts9YrTZDizTX7PjC2s6W4x+MBGZeEY6dGMrF04/6Hgqg==", "license": "MIT" }, - "node_modules/buffer": { - "version": "6.0.3", - "resolved": "https://registry.npmjs.org/buffer/-/buffer-6.0.3.tgz", - "integrity": "sha512-FTiCpNxtwiZZHEZbcbTIcZjERVICn9yq/pDFkTl95/AxzD1naBctN7YO68riM/gLSDY7sdrMby8hofADYuuqOA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ], - "dependencies": { - "base64-js": "^1.3.1", - "ieee754": "^1.2.1" - } - }, "node_modules/buffer-crc32": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-1.0.0.tgz", @@ -182,12 +163,44 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/chokidar": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", + "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "dev": true, + "license": "MIT", + "dependencies": { + "anymatch": "~3.1.2", + "braces": "~3.0.2", + "glob-parent": "~5.1.2", + "is-binary-path": "~2.1.0", + "is-glob": "~4.0.1", + "normalize-path": "~3.0.0", + "readdirp": "~3.6.0" + }, + "engines": { + "node": ">= 8.10.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + }, + "optionalDependencies": { + "fsevents": "~2.3.2" + } + }, "node_modules/colorette": { "version": "2.0.20", "resolved": "https://registry.npmjs.org/colorette/-/colorette-2.0.20.tgz", "integrity": "sha512-IfEDxwoWIjkeXL1eXcDiow4UbKjhLdq6/EuSVR9GMN7KVH3r9gQ83e73hsz1Nd1T3ijd5xv1wcWRYO+D6kCI2w==", "dev": true }, + "node_modules/concat-map": { + "version": "0.0.1", + "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", + "integrity": "sha512-/Srv4dswyQNBfohGpz9o6Yb3Gz3SrUDqBH5rTuhGR7ahtlbYKnVxw2bCFMRljaA7EXHaXZ8wsHdodFvbkhKmqg==", + "dev": true, + "license": "MIT" + }, "node_modules/dateformat": { "version": "4.6.3", "resolved": "https://registry.npmjs.org/dateformat/-/dateformat-4.6.3.tgz", @@ -197,6 +210,24 @@ "node": "*" } }, + "node_modules/debug": { + "version": "4.4.0", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.0.tgz", + "integrity": "sha512-6WTZ/IxCY/T6BALoZHaE4ctp9xm+Z5kY/pzYaCHRFeyVhojxlrm+46y68HA6hr0TcwEssoxNiDEUJQjfPZ/RYA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, "node_modules/decode-uri-component": { "version": "0.2.2", "resolved": "https://registry.npmjs.org/decode-uri-component/-/decode-uri-component-0.2.2.tgz", @@ -264,33 +295,18 @@ "node": ">= 0.4" } }, - "node_modules/event-target-shim": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/event-target-shim/-/event-target-shim-5.0.1.tgz", - "integrity": "sha512-i/2XbnSz/uxRCU6+NdVJgKWDTM427+MqYbkQzD321DuCQJUqOuJKIA0IM2+W2xtYHdKOmZ4dR6fExsd4SXL+WQ==", - "engines": { - "node": ">=6" - } - }, "node_modules/eventemitter3": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/eventemitter3/-/eventemitter3-5.0.1.tgz", "integrity": "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA==", "license": "MIT" }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/fast-copy": { - "version": "3.0.1", - "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.1.tgz", - "integrity": "sha512-Knr7NOtK3HWRYGtHoJrjkaWepqT8thIVGAwt0p0aUs1zqkAzXZV4vo9fFNwyb5fcqK1GKYFYxldQdIDVKhUAfA==", - "dev": true + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/fast-copy/-/fast-copy-3.0.2.tgz", + "integrity": "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ==", + "dev": true, + "license": "MIT" }, "node_modules/fast-redact": { "version": "3.3.0", @@ -328,6 +344,19 @@ "fxparser": "src/cli/cli.js" } }, + "node_modules/fill-range": { + "version": "7.1.1", + "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", + "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "to-regex-range": "^5.0.1" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/filter-obj": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/filter-obj/-/filter-obj-1.1.0.tgz", @@ -346,11 +375,20 @@ "is-callable": "^1.1.3" } }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } }, "node_modules/function-bind": { "version": "1.1.2", @@ -380,23 +418,17 @@ "url": "https://github.com/sponsors/ljharb" } }, - "node_modules/glob": { - "version": "8.1.0", - "resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz", - "integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==", + "node_modules/glob-parent": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", + "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", "dev": true, + "license": "ISC", "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^5.0.1", - "once": "^1.3.0" + "is-glob": "^4.0.1" }, "engines": { - "node": ">=12" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" + "node": ">= 6" } }, "node_modules/gopd": { @@ -411,6 +443,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, "node_modules/has-property-descriptors": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", @@ -475,43 +517,18 @@ } }, "node_modules/help-me": { - "version": "4.2.0", - "resolved": "https://registry.npmjs.org/help-me/-/help-me-4.2.0.tgz", - "integrity": "sha512-TAOnTB8Tz5Dw8penUuzHVrKNKlCIbwwbHnXraNJxPwf8LRtE2HlM84RYuezMFcwOJmoYOCWVDyJ8TQGxn9PgxA==", + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/help-me/-/help-me-5.0.0.tgz", + "integrity": "sha512-7xgomUX6ADmcYzFik0HzAxh/73YlKR9bmFzf51CZwR+b6YtzU2m0u49hQCqV6SvlqIqsaxovfwdvbnsw3b/zpg==", "dev": true, - "dependencies": { - "glob": "^8.0.0", - "readable-stream": "^3.6.0" - } - }, - "node_modules/ieee754": { - "version": "1.2.1", - "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", - "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==", - "funding": [ - { - "type": "github", - "url": "https://github.com/sponsors/feross" - }, - { - "type": "patreon", - "url": "https://www.patreon.com/feross" - }, - { - "type": "consulting", - "url": "https://feross.org/support" - } - ] + "license": "MIT" }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", "dev": true, - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } + "license": "ISC" }, "node_modules/inherits": { "version": "2.0.4", @@ -543,6 +560,19 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-binary-path": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", + "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", + "dev": true, + "license": "MIT", + "dependencies": { + "binary-extensions": "^2.0.0" + }, + "engines": { + "node": ">=8" + } + }, "node_modules/is-callable": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz", @@ -555,6 +585,16 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-extglob": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", + "integrity": "sha512-SbKbANkN603Vi4jEZv49LeVJMn4yGwsbzZworEoyEiutsN3nJYdbO36zfhGJ6QEDpOZIFkDtnq5JRxmvl3jsoQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/is-generator-function": { "version": "1.0.10", "resolved": "https://registry.npmjs.org/is-generator-function/-/is-generator-function-1.0.10.tgz", @@ -570,6 +610,29 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/is-glob": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/is-glob/-/is-glob-4.0.3.tgz", + "integrity": "sha512-xelSayHH36ZgE7ZWhli7pW34hNbNl8Ojv5KVmkJD4hBdD3th8Tfk9vYasLM+mXWOZhFkgZfxhLSnrwRr4elSSg==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-extglob": "^2.1.1" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/is-number": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", + "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.12.0" + } + }, "node_modules/is-typed-array": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/is-typed-array/-/is-typed-array-1.1.13.tgz", @@ -627,18 +690,6 @@ "node": ">= 0.6" } }, - "node_modules/minimatch": { - "version": "5.1.6", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz", - "integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==", - "dev": true, - "dependencies": { - "brace-expansion": "^2.0.1" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -649,9 +700,9 @@ } }, "node_modules/minio": { - "version": "8.0.1", - "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.1.tgz", - "integrity": "sha512-FzDO6yGnqLtm8sp3mXafWtiRUOslJSSg/aI0v9YbN5vjw5KLoODKAROCyi766NIvTSxcfHBrbhCSGk1A+MOzDg==", + "version": "8.0.4", + "resolved": "https://registry.npmjs.org/minio/-/minio-8.0.4.tgz", + "integrity": "sha512-GVW7y2PNbzjjFJ9opVMGKvDNuRkyz3bMt1q7UrHs7bsKFWLXbSvMPffjE/HkVYWUjlD8kQwMaeqiHhhvZJJOfQ==", "license": "Apache-2.0", "dependencies": { "async": "^3.2.4", @@ -659,7 +710,7 @@ "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", - "fast-xml-parser": "^4.2.2", + "fast-xml-parser": "^4.4.1", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", @@ -667,12 +718,93 @@ "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", - "xml2js": "^0.5.0" + "xml2js": "^0.5.0 || ^0.6.2" }, "engines": { "node": "^16 || ^18 || >=20" } }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/node-cron": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/node-cron/-/node-cron-3.0.3.tgz", + "integrity": "sha512-dOal67//nohNgYWb+nWmg5dkFdIwDm8EpeGYMekPMrngV3637lqnX0lbUcCtgibHTz6SEz7DAIjKvKDFYCnO1A==", + "license": "ISC", + "dependencies": { + "uuid": "8.3.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/nodemon": { + "version": "3.1.9", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.9.tgz", + "integrity": "sha512-hdr1oIb2p6ZSxu3PB2JWWYS7ZQ0qvaZsc3hK8DR8f02kRzc8rjYmxAIvdz+aYC+8F2IjNaB7HMcSDg8nQpJxyg==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^3.1.2", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/brace-expansion": { + "version": "1.1.11", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", + "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", + "dev": true, + "license": "MIT", + "dependencies": { + "balanced-match": "^1.0.0", + "concat-map": "0.0.1" + } + }, + "node_modules/nodemon/node_modules/minimatch": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", + "dev": true, + "license": "ISC", + "dependencies": { + "brace-expansion": "^1.1.7" + }, + "engines": { + "node": "*" + } + }, + "node_modules/normalize-path": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/normalize-path/-/normalize-path-3.0.0.tgz", + "integrity": "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/on-exit-leak-free": { "version": "2.1.2", "resolved": "https://registry.npmjs.org/on-exit-leak-free/-/on-exit-leak-free-2.1.2.tgz", @@ -690,96 +822,80 @@ "wrappy": "1" } }, + "node_modules/picomatch": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.1.tgz", + "integrity": "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=8.6" + }, + "funding": { + "url": "https://github.com/sponsors/jonschlinkert" + } + }, "node_modules/pino": { - "version": "8.16.2", - "resolved": "https://registry.npmjs.org/pino/-/pino-8.16.2.tgz", - "integrity": "sha512-2advCDGVEvkKu9TTVSa/kWW7Z3htI/sBKEZpqiHk6ive0i/7f5b1rsU8jn0aimxqfnSz5bj/nOYkwhBUn5xxvg==", + "version": "9.6.0", + "resolved": "https://registry.npmjs.org/pino/-/pino-9.6.0.tgz", + "integrity": "sha512-i85pKRCt4qMjZ1+L7sy2Ag4t1atFcdbEt76+7iRJn1g2BvsnRMGu9p8pivl9fs63M2kF/A0OacFZhTub+m/qMg==", + "license": "MIT", "dependencies": { "atomic-sleep": "^1.0.0", "fast-redact": "^3.1.1", "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "v1.1.0", - "pino-std-serializers": "^6.0.0", - "process-warning": "^2.0.0", + "pino-abstract-transport": "^2.0.0", + "pino-std-serializers": "^7.0.0", + "process-warning": "^4.0.0", "quick-format-unescaped": "^4.0.3", "real-require": "^0.2.0", "safe-stable-stringify": "^2.3.1", - "sonic-boom": "^3.7.0", - "thread-stream": "^2.0.0" + "sonic-boom": "^4.0.1", + "thread-stream": "^3.0.0" }, "bin": { "pino": "bin.js" } }, "node_modules/pino-abstract-transport": { - "version": "1.1.0", - "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-1.1.0.tgz", - "integrity": "sha512-lsleG3/2a/JIWUtf9Q5gUNErBqwIu1tUKTT3dUzaf5DySw9ra1wcqKjJjLX1VTY64Wk1eEOYsVGSaGfCK85ekA==", + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/pino-abstract-transport/-/pino-abstract-transport-2.0.0.tgz", + "integrity": "sha512-F63x5tizV6WCh4R6RHyi2Ml+M70DNRXt/+HANowMflpgGFMAym/VKm6G7ZOQRjqN7XbGxK1Lg9t6ZrtzOaivMw==", + "license": "MIT", "dependencies": { - "readable-stream": "^4.0.0", "split2": "^4.0.0" } }, - "node_modules/pino-abstract-transport/node_modules/readable-stream": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", - "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==", - "dependencies": { - "abort-controller": "^3.0.0", - "buffer": "^6.0.3", - "events": "^3.3.0", - "process": "^0.11.10", - "string_decoder": "^1.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, "node_modules/pino-pretty": { - "version": "10.2.3", - "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-10.2.3.tgz", - "integrity": "sha512-4jfIUc8TC1GPUfDyMSlW1STeORqkoxec71yhxIpLDQapUu8WOuoz2TTCoidrIssyz78LZC69whBMPIKCMbi3cw==", + "version": "13.0.0", + "resolved": "https://registry.npmjs.org/pino-pretty/-/pino-pretty-13.0.0.tgz", + "integrity": "sha512-cQBBIVG3YajgoUjo1FdKVRX6t9XPxwB9lcNJVD5GCnNM4Y6T12YYx8c6zEejxQsU0wrg9TwmDulcE9LR7qcJqA==", "dev": true, + "license": "MIT", "dependencies": { "colorette": "^2.0.7", "dateformat": "^4.6.3", - "fast-copy": "^3.0.0", + "fast-copy": "^3.0.2", "fast-safe-stringify": "^2.1.1", - "help-me": "^4.0.1", + "help-me": "^5.0.0", "joycon": "^3.1.1", "minimist": "^1.2.6", "on-exit-leak-free": "^2.1.0", - "pino-abstract-transport": "^1.0.0", + "pino-abstract-transport": "^2.0.0", "pump": "^3.0.0", - "readable-stream": "^4.0.0", "secure-json-parse": "^2.4.0", - "sonic-boom": "^3.0.0", + "sonic-boom": "^4.0.1", "strip-json-comments": "^3.1.1" }, "bin": { "pino-pretty": "bin.js" } }, - "node_modules/pino-pretty/node_modules/readable-stream": { - "version": "4.4.2", - "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-4.4.2.tgz", - "integrity": "sha512-Lk/fICSyIhodxy1IDK2HazkeGjSmezAWX2egdtJnYhtzKEsBPJowlI6F6LPb5tqIQILrMbx22S5o3GuJavPusA==", - "dev": true, - "dependencies": { - "abort-controller": "^3.0.0", - "buffer": "^6.0.3", - "events": "^3.3.0", - "process": "^0.11.10", - "string_decoder": "^1.3.0" - }, - "engines": { - "node": "^12.22.0 || ^14.17.0 || >=16.0.0" - } - }, "node_modules/pino-std-serializers": { - "version": "6.2.2", - "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-6.2.2.tgz", - "integrity": "sha512-cHjPPsE+vhj/tnhCy/wiMh3M3z3h/j15zHQX+S9GkTBgqJuTuJzYJ4gUyACLhDaJ7kk9ba9iRDmbH2tJU03OiA==" + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/pino-std-serializers/-/pino-std-serializers-7.0.0.tgz", + "integrity": "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA==", + "license": "MIT" }, "node_modules/possible-typed-array-names": { "version": "1.0.0", @@ -790,18 +906,28 @@ "node": ">= 0.4" } }, - "node_modules/process": { - "version": "0.11.10", - "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz", - "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==", - "engines": { - "node": ">= 0.6.0" - } - }, "node_modules/process-warning": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-2.3.0.tgz", - "integrity": "sha512-N6mp1+2jpQr3oCFMz6SeHRGbv6Slb20bRhj4v3xR99HqNToAcOe1MFOp4tytyzOfJn+QtN8Rf7U/h2KAn4kC6g==" + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/process-warning/-/process-warning-4.0.1.tgz", + "integrity": "sha512-3c2LzQ3rY9d0hc1emcsHhfT9Jwz0cChib/QN89oME2R451w5fy3f0afAhERFZAwrbDU43wk12d0ORBpDVME50Q==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fastify" + }, + { + "type": "opencollective", + "url": "https://opencollective.com/fastify" + } + ], + "license": "MIT" + }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" }, "node_modules/pump": { "version": "3.0.0", @@ -849,10 +975,24 @@ "node": ">= 6" } }, + "node_modules/readdirp": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", + "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "picomatch": "^2.2.1" + }, + "engines": { + "node": ">=8.10.0" + } + }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", "integrity": "sha512-57frrGM/OCTLqLOAh0mhVA9VBMHd+9U7Zb2THMGdBUoZVOtGbJzjxsYGDJ3A9AYYCP4hn6y1TVbaOfzWtm5GFg==", + "license": "MIT", "engines": { "node": ">= 12.13.0" } @@ -896,6 +1036,19 @@ "integrity": "sha512-6aU+Rwsezw7VR8/nyvKTx8QpWH9FrcYiXXlqC4z5d5XQBDRqtbfsRjnwGyqbi3gddNtWHuEk9OANUotL26qKUw==", "dev": true }, + "node_modules/semver": { + "version": "7.7.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.7.1.tgz", + "integrity": "sha512-hlq8tAfn0m/61p4BVRcPzIGr6LKiMwo4VM6dGi6pt4qcRkmNzTcWq6eCEjEh+qXjkMDvPlOFFSGwQjoEa6gyMA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/set-function-length": { "version": "1.2.2", "resolved": "https://registry.npmjs.org/set-function-length/-/set-function-length-1.2.2.tgz", @@ -913,10 +1066,24 @@ "node": ">= 0.4" } }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/sonic-boom": { - "version": "3.7.0", - "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-3.7.0.tgz", - "integrity": "sha512-IudtNvSqA/ObjN97tfgNmOKyDOs4dNcg4cUUsHDebqsgb8wGBBwb31LIgShNO8fye0dFI52X1+tFoKKI6Rq1Gg==", + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/sonic-boom/-/sonic-boom-4.2.0.tgz", + "integrity": "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww==", + "license": "MIT", "dependencies": { "atomic-sleep": "^1.0.0" } @@ -934,6 +1101,7 @@ "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", "engines": { "node": ">= 10.x" } @@ -988,10 +1156,24 @@ "integrity": "sha512-J8bbNyKKXl5qYcR36TIO8W3mVGVHrmmxsd5PAItGkmyzwJvybiw2IVq5nqd0i4LSNSkB/sx9VHllbfFdr9k1JA==", "license": "MIT" }, + "node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/thread-stream": { - "version": "2.4.1", - "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-2.4.1.tgz", - "integrity": "sha512-d/Ex2iWd1whipbT681JmTINKw0ZwOUBZm7+Gjs64DHuX34mmw8vJL2bFAaNacaW72zYiTJxSHi5abUuOi5nsfg==", + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/thread-stream/-/thread-stream-3.1.0.tgz", + "integrity": "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A==", + "license": "MIT", "dependencies": { "real-require": "^0.2.0" } @@ -1010,6 +1192,36 @@ "resolved": "https://registry.npmjs.org/tinyduration/-/tinyduration-3.3.0.tgz", "integrity": "sha512-sLR0iVUnnnyGEX/a3jhTA0QMK7UvakBqQJFLiibiuEYL6U1L85W+qApTZj6DcL1uoWQntYuL0gExoe9NU5B3PA==" }, + "node_modules/to-regex-range": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", + "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "is-number": "^7.0.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", + "dev": true, + "license": "ISC", + "bin": { + "nodetouch": "bin/nodetouch.js" + } + }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, "node_modules/undici-types": { "version": "5.26.5", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", @@ -1034,6 +1246,15 @@ "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==" }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "license": "MIT", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/web-encoding": { "version": "1.1.5", "resolved": "https://registry.npmjs.org/web-encoding/-/web-encoding-1.1.5.tgz", diff --git a/package.json b/package.json old mode 100644 new mode 100755 index 343e77e..6cebf0f --- a/package.json +++ b/package.json @@ -4,22 +4,30 @@ "private": true, "type": "module", "scripts": { - "start": "node ./src/index.js", - "start:debug": "node --nolazy --inspect-brk=0.0.0.0:9229 ./src/index.js" + "start": "node ./src/index.js $NODE_ARGS", + "dev": "nodemon --nolazy --inspect=0.0.0.0:9229 --max-old-space-size=1024 ./src/index.js $NODE_ARGS", + "dev:profiling": "nodemon --nolazy --inspect-brk=0.0.0.0:9229 --prof --max-old-space-size=1024 ./src/index.js $NODE_ARGS", + "dev:clinic:doctor": "clinic doctor $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS", + "dev:clinic:bubbleprof": "clinic bubbleprof $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS", + "dev:clinic:flame": "clinic flame $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS", + "dev:clinic:heapprofiler": "clinic heapprofiler $CLINIC_ARGS -- node ./src/index.js $NODE_ARGS" }, "engines": { - "node": ">=20 <21" + "node": ">=20 <24" }, "dependencies": { "dotenv": "^16.3.1", - "minio": "8.0.1", + "minio": "8.0.4", "kafkajs":"^2.2.4", - "pino": "^8.16.2", + "pino": "^9.6.0", "tinyduration": "^3.3.0", - "wretch": "^2.7.0" + "wretch": "^2.7.0", + "node-cron": "3.0.3", + "ms": "^2.1.3" }, "devDependencies": { "@types/node": "^20.9.0", - "pino-pretty": "^10.2.3" + "pino-pretty": "^13.0.0", + "nodemon":"^3.1.9" } } diff --git a/src/compactor.js b/src/compactor.js old mode 100644 new mode 100755 index b02ffdf..fccc977 --- a/src/compactor.js +++ b/src/compactor.js @@ -2,10 +2,9 @@ import { now, duration, formatDuration } from './utils/date.js'; import { logger } from './logger.js'; import { MinioClient } from './minio.js'; import { SimvaClient } from './simva.js'; -import { KafkaClient } from './kafka.js'; import { getState } from './state.js'; -import { createHash } from 'node:crypto'; import { binarySearch, diffArray } from './utils/misc.js'; +import { sha1sums } from './utils/sha.js'; /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */ /** @typedef {import('./simva.js').Activity} Activity */ @@ -28,7 +27,6 @@ export class Compactor { this.#opts = opts; this.#minio = new MinioClient(opts.minio); this.#simva = new SimvaClient(opts.simva); - this.#kafka = new KafkaClient(opts.kafka); this.shouldExit = false; this.status = { processing: false, @@ -43,9 +41,6 @@ export class Compactor { /** @type {MinioClient} */ #minio; - /** @type {KafkaClient} */ - #kafka; - /** @type {SimvaClient} */ #simva; @@ -70,6 +65,7 @@ export class Compactor { } catch (e) { logger.error(e); logger.error('####### ERROR WHILE PROCESSING THE TRACES. !!!!!!'); + throw e; } this.status.processing = false; } else { @@ -92,13 +88,10 @@ export class Compactor { async #checkConsistency() { let state = await getState(this.#opts, this.#minio); - let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] }); + let activities = await this.#simva.getActivities({ type: ['gameplay', 'manual', 'limesurvey', 'miniokafka', 'rageminio'] }); logger.info(`Known %d activities, received %d`, state.size, activities.length); - const outputDir = this.#opts.minio.outputs_dir; - const tracesFilename = this.#opts.minio.traces_file; - this.status.total = activities.length; const inconsistent = []; for(let idx=0; idx < activities.length; idx++) { @@ -117,11 +110,11 @@ export class Compactor { } let consistent = await activityState.checkConsistency(); - const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`; - if (! await this.#minio.fileExists(remotePath) ) { - logger.warn('Compact file for activity \'%s\' not found: %s', activityState.activityId, remotePath); - consistent = false; - } + const remotePath = activityState.remoteOutputPath; + if (! await this.#minio.fileExists(remotePath) ) { + logger.warn('Compact file for activity \'%s\' not found: %s', activityState.activityId, remotePath); + consistent = false; + } if (!consistent) { inconsistent.push(activity._id); } @@ -139,13 +132,14 @@ export class Compactor { async #compactActivities() { let state = await getState(this.#opts, this.#minio); - let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] }); + let activities = await this.#simva.getActivities({ type: ['gameplay', 'manual', 'limesurvey', 'miniokafka', 'rageminio'] }); logger.info(`Known %d activities, received %d`, state.size, activities.length); - await this.#garbageCollectActivities(state, activities); + let activitiesToGo = await this.#garbageCollectActivities(state, activities); this.status.total = activities.length; + let leastoneUpdated=false; for(let idx=0; idx < activities.length; idx++) { if (this.shouldExit) { break; @@ -160,16 +154,19 @@ export class Compactor { logger.info(`New activity: %s`, activity._id); activityState = await state.create(activity._id); } - const updated = await this.#updateActivityTraces(activityState); if (!updated) continue; + + leastoneUpdated=updated; await this.#distributeTrace(activityState); if (activities.length % 5) { await state.save(); } } - await state.save(); + if(leastoneUpdated) { + await state.save(); + } } /** @@ -200,12 +197,16 @@ export class Compactor { } catch(error) { logger.error('Could not remove activity: %s', activityId); logger.error(error); + throw error; } } + logger.info('Activities to removed OK.'); } - + logger.info('Starting collecting state garbage.'); // Garbage collect state files in activities - await state.garbageCollect(); + const activityToPass = await state.garbageCollect(); + logger.info('Collecting state garbage finished.'); + return activityToPass; } /** @@ -214,14 +215,8 @@ export class Compactor { * @returns {Promise} false if nothing new */ async #updateActivityTraces(activityState) { - let traceFiles = (await this.#minio.getTraces(activityState.activityId)).map((o) => o.name); - traceFiles.sort(); - const hash = createSha1(); - for(const traceFile of traceFiles) { - hash.update(traceFile); - hash.update('\n'); - } - const sha1 = hash.digest('hex'); + let traceFiles = (await this.#minio.getTraces(activityState.activityId)).map((o) => o.name).sort(); + const sha1 = sha1sums(traceFiles); if (sha1 === activityState.currentSha1) { logger.debug(`Nothing to do for activity %s`, activityState.activityId); return false; @@ -232,15 +227,29 @@ export class Compactor { const { added: filesToAdd } = diffArray(activityFiles, traceFiles); const nowDate = now(); const elapsedTime = duration(activityState.lastUpdate, nowDate); - if (filesToAdd.length < this.#opts.batchSize && elapsedTime < this.#opts.maxDelay) { + let filesToConsume=[]; + for(let index in filesToAdd) { + let file=filesToAdd[index]; + logger.debug(file); + if(activityFiles.includes(file)) { + logger.warn("Already consumed: %s", file); + } else { + filesToConsume.push(file); + } + } + if (filesToConsume.length < this.#opts.batchSize && elapsedTime < this.#opts.maxDelay) { const durationStr = formatDuration(elapsedTime); - logger.debug(`Update postponed elapsedTime=%s, batchSize=%d for activity %s`, durationStr, filesToAdd.length, activityState.activityId); + logger.debug(`Update postponed elapsedTime=%s, batchSize=%d for activity %s`, durationStr, filesToConsume.length, activityState.activityId); return false; } logger.info(`Compacting activity %s`, activityState.activityId); - await activityState.update(filesToAdd, nowDate, sha1); - return true; + if(filesToConsume.length > 0) { + await activityState.update(filesToConsume, nowDate, sha1); + return true; + } else { + return false; + } } /** @@ -249,154 +258,124 @@ export class Compactor { */ async #distributeTrace(activityState) { const localStatePath = activityState.localStatePath; - const outputDir = this.#opts.minio.outputs_dir; - const tracesFilename = this.#opts.minio.traces_file; - const remotePath = `${outputDir}/${activityState.activityId}/${tracesFilename}`; - await this.#minio.copyToRemoteFile(localStatePath, remotePath); + const remoteStatePath = activityState.remoteStatePath; + const remotePath = activityState.remoteOutputPath; + try { + //const metadata = { + // "Content-Type": "application/json", + // "Version": "1" + //}; + //await this.#minio.copyToRemoteFile(localStatePath, remotePath, metadata); + await this.#minio.copyWithinMinIO(remoteStatePath, remotePath); + logger.info("Object copied successfully!"); + } catch (error) { + logger.error("Copy compacted file failed:"); + logger.error(error); + throw error; + } logger.info(`Copied compacted file for activity %s`, activityState.activityId); } + async processConsistencyAndGarbage() { + logger.info('Check consistency'); + await this.#checkConsistency(); + logger.info('Start compaction'); + await this.#compactActivities(); + } + // Method to process messages (acts as the callback for KafkaClient) /** * @param {any} message */ async processMessage(message) { - try { - // Log the received message - logger.info('Received message:'); - logger.info(message.value); - - let state = await getState(this.#opts, this.#minio); - let activities = await this.#simva.getActivities({ type: ['gameplay', 'miniokafka', 'rageminio'] }); - await this.#garbageCollectActivities(state, activities); - // Set up the delimiter and the required bucket and path values - let delimiter = '/'; - let bucket = this.#opts.minio.bucket; - - // Build the path to traces topic - let tracestopicspath = `${this.#opts.minio.topics_dir}${delimiter}${this.#opts.minio.traces_topic}${delimiter}_id=`; + // Log the received message + logger.debug('Received message:'); + logger.debug(message.value); + + let state = await getState(this.#opts, this.#minio); + // Set up the delimiter and the required bucket and path values + let delimiter = '/'; + let bucket = this.#opts.minio.bucket; - // Log the constructed path - logger.info(`Trace topic path: ${tracestopicspath}`); + // Build the path to traces topic + let tracestopicspath = `${this.#opts.minio.topics_dir}${delimiter}${this.#opts.minio.traces_topic}${delimiter}_id=`; + + // Log the constructed path + logger.debug(`Trace topic path: ${tracestopicspath}`); + + // Parse the message value (assuming it's a JSON string) + let ev = JSON.parse(message.value); + let key = ev.Key; - // Parse the message value (assuming it's a JSON string) - let ev = JSON.parse(message.value); - let key = ev.Key; - - // Log the key extracted from the message - logger.info(`Received Key: ${key}`); + // Log the key extracted from the message + logger.debug(`Received Key: ${key}`); + + // Remove the bucket and trace topic path from the key to get the key value + let keyvalue = key.replace(`${bucket}${delimiter}${tracestopicspath}`, ""); - // Remove the bucket and trace topic path from the key to get the key value - let keyvalue = key.replace(`${bucket}${delimiter}${tracestopicspath}`, ""); - - // Log the key value after removal - logger.info(`Key value without bucket and path: ${keyvalue}`); + // Log the key value after removal + logger.debug(`Key value without bucket and path: ${keyvalue}`); + + // Split the key value to extract activityId and filename + let added = keyvalue.split(delimiter); - // Split the key value to extract activityId and filename - let added = keyvalue.split(delimiter); - - // Initialize variables for activityId and filename - let activityId = null; - let filename = null; - let keyWithoutBucket = null; + // Initialize variables for activityId and filename + let activityId = null; + let filename = null; + let keyWithoutBucket = null; + + // If the split key has exactly 2 parts, extract activityId and filename + if (added.length !== 2) { + throw new Error('Key format is unexpected. Unable to extract activityId and filename.'); + } + activityId = added[0]; + filename = added[1]; + keyWithoutBucket = `${tracestopicspath}${activityId}${delimiter}${filename}`; + + // Log the extracted values + logger.debug(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`); - // If the split key has exactly 2 parts, extract activityId and filename - if (added.length === 2) { - activityId = added[0]; - filename = added[1]; - keyWithoutBucket = `${tracestopicspath}${activityId}${delimiter}${filename}`; - - // Log the extracted values - logger.info(`activityId: ${activityId}, filename: ${filename}, key: ${key}, keyWithoutBucket: ${keyWithoutBucket}`); - - // ActivityState - let activityState = state.get(activityId); - if (activityState === undefined) { - logger.info(`New activity: %s`, activityId); - activityState = await state.create(activityId); - } - logger.info(activityState); - // compute which files need to be appended - const activityFiles = (await activityState.files()).sort(); - logger.info(activityFiles); - - if(activityFiles.includes(keyWithoutBucket)) { - logger.warn("Already consumed.") - } else { - try { - let positionvalue=-binarySearch(activityFiles, keyWithoutBucket, true, (a,b)=> { - if(typeof a == "string" && typeof b == "string" ) { - return a.localeCompare(b); - } else { - return -1; - } - })-1; - var nextposition=activityFiles.length; - logger.info(keyWithoutBucket); - logger.info("positionvalue:"); - logger.info(positionvalue); - logger.info("nextposition:"); - logger.info(nextposition); - if(positionvalue < nextposition) { - logger.warn("Not ordered. Should have been consumed before.") - } - await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket); - logger.info(activityState); - await this.#distributeTrace(activityState); - logger.info(activityState); - await state.save(); - } catch(e) { - logger.info(e); - } - } - } else { - logger.warn('Key format is unexpected. Unable to extract activityId and filename.'); - } - } catch(e) { - logger.debug('Error processing message:'); - logger.debug(e); + // ActivityState + let activityState = state.get(activityId); + if (activityState === undefined) { + logger.debug(`New activity: %s`, activityId); + activityState = await state.create(activityId); + } + logger.debug(activityState); + // compute which files need to be appended + const activityFiles = (await activityState.files()); + logger.debug(activityFiles); + + if(activityFiles.includes(keyWithoutBucket)) { + logger.warn("Already consumed: %s", keyWithoutBucket); + return; } + const newActivityFiles= await activityState.insertOrdered([keyWithoutBucket]); + const sha1 = sha1sums(newActivityFiles); + await this.#updateActivityTracesFromPath(activityState, keyWithoutBucket, sha1); + logger.debug(activityState); + + await this.#distributeTrace(activityState); + logger.debug(activityState); + + await state.save(); } /** * Update Activity Traces From Path * @param {ActivityCompactionState} activityState * @param {string} keyPath + * @param {string} sha1 * @returns {Promise} false if nothing new */ - async #updateActivityTracesFromPath(activityState, keyPath) { - const hash = createSha1(); - hash.update(keyPath); - hash.update('\n'); - const sha1 = hash.digest('hex'); + async #updateActivityTracesFromPath(activityState, keyPath, sha1) { const nowDate = now(); const filesToAdd = [keyPath]; logger.info(`Compacting activity %s`, activityState.activityId); - logger.info(filesToAdd); + logger.debug(filesToAdd); await activityState.update(filesToAdd, nowDate, sha1); return true; } - - // Method to start consuming messages using KafkaClient - async startKafkaConsumer() { - try { - logger.info('Compactor starting Kafka consumption...'); - // Start Kafka consumption and pass the processMessage as a callback - await this.#kafka.consumeLatestMessages(this.processMessage.bind(this)); - } catch (error) { - console.error('Error starting Compactor:', error); - } - } - - // Method to stop consuming messages - async stopKafkaConsumer() { - try { - await this.#kafka.disconnect(); - logger.info('Compactor stopped Kafka consumption.'); - } catch (error) { - console.error('Error stopping Compactor:', error); - } - } /** * Get MinioClient @@ -414,12 +393,3 @@ export class Compactor { return this.#opts; } } - -/** - * Create sha1 hash function. - * - * @returns - */ -function createSha1() { - return createHash('sha1'); -} diff --git a/src/config.js b/src/config.js old mode 100644 new mode 100755 index 006f6ff..4dcc3e5 --- a/src/config.js +++ b/src/config.js @@ -1,9 +1,10 @@ import 'dotenv/config'; import * as inspector from 'inspector'; +import ms from "ms"; /** * @typedef CompactorOptions - * @property {string} concatEventPolicy + * @property {boolean} concatEventPolicy * @property {number} batchSize * @property {number} maxDelay * @property {number} refreshInterval @@ -20,14 +21,14 @@ import * as inspector from 'inspector'; /** @type {CompactorOptions} */ export const config = { - concatEventPolicy: process.env.CONCAT_EVENT_POLICY || 'true', // if true minio-events else previous version of trace allocator + concatEventPolicy: process.env.CONCAT_EVENT_POLICY !== undefined ? (process.env.CONCAT_EVENT_POLICY.toLocaleLowerCase() === 'false' ? false : true) : true, // if true minio-events else previous version of trace allocator batchSize: process.env.BATCH_SIZE !== undefined ? parseInt(process.env.BATCH_SIZE) : 500, - maxDelay: process.env.MAX_DELAY !== undefined ? parseInt(process.env.MAX_DELAY) : 5*60*1000, - refreshInterval: process.env.REFRESH_INTERVAL !== undefined ? parseInt(process.env.REFRESH_INTERVAL) : 10*60*1000, + maxDelay: process.env.MAX_DELAY !== undefined ? ms(process.env.MAX_DELAY) : ms("5min"), + refreshInterval: process.env.REFRESH_INTERVAL !== undefined ? ms(process.env.REFRESH_INTERVAL) : ms("10min"), localStatePath: process.env.LOCAL_STATE || new URL('../state', import.meta.url).pathname, remoteStatePath: process.env.REMOTE_STATE || 'state', removeDryRun: process.env.REMOVE_DRY_RUN !== undefined ? (process.env.REMOVE_DRY_RUN.toLocaleLowerCase() === 'false' ? false : true) : true, - gcInterval: process.env.GC_INTERVAL !== undefined ? parseInt(process.env.GC_INTERVAL) : 10*24*60*60*1000, + gcInterval: process.env.GC_INTERVAL !== undefined ? ms(process.env.GC_INTERVAL) : ms("2h"), copyInsteadRename: process.env.COPY_INSTEAD_RENAME !== undefined ? (process.env.COPY_INSTEAD_RENAME.toLocaleLowerCase() === 'false' ? false : true) : true, tryRecovery: process.env.TRY_RECOVERY !== undefined ? (process.env.TRY_RECOVERY.toLocaleLowerCase() === 'false' ? false : true) : false, minio: { @@ -51,7 +52,7 @@ export const config = { }, kafka: { clientId: process.env.SIMVA_KAFKA_CLIENTID || 'my-client-id', - brokers: [ process.env.SIMVA_KAFKA_BROKER ] || ['localhost:9092'], + brokers: process.env.SIMVA_KAFKA_BROKER !== undefined ? [ process.env.SIMVA_KAFKA_BROKER ] : ['localhost:9092'], groupId: process.env.SIMVA_KAFKA_GROUPID || 'my-group-id', topic: process.env.SIMVA_KAFKA_MINIO_TOPIC || 'minio-events' } diff --git a/src/index.js b/src/index.js old mode 100644 new mode 100755 index 5f19639..a2e6ff7 --- a/src/index.js +++ b/src/index.js @@ -1,90 +1,108 @@ import { config, isInDebugMode } from './config.js'; import { logger } from './logger.js'; - +import cron from 'node-cron'; import { Compactor } from './compactor.js'; -import { getState } from './state.js'; - +import { convertTimeToCron } from "./utils/date.js"; +import { KafkaClient } from './kafka.js'; +import "./profiling.js"; +import ms from 'ms'; logger.debug('Current config: %o', config); const compactor = new Compactor(config); const run = compactor.compact.bind(compactor); -const MAX_WAIT_TIME_ON_EXIT = 30*1000; - let intervalId; -if (config.concatEventPolicy === "true") { +if (config.concatEventPolicy) { + const kafka = new KafkaClient(config.kafka); await startKafkaProcess(); -} else { + async function startKafkaProcess() { + // Schedule a task to run every x + let gcIntervalInMin=Math.round(config.gcInterval/(1000*60)); + const cronTime = convertTimeToCron(gcIntervalInMin); + logger.info(cronTime); + const task = async () => { + logger.info("Compactor starting process garbage scheduled task running every %s minutes", gcIntervalInMin); + try { + await compactor.processConsistencyAndGarbage(); + } catch(e) { + logger.error(e); + } + }; + + await task(); + + cron.schedule(cronTime, task); + + try { + logger.info('Compactor starting Kafka consumption...'); + // Start Kafka consumption and pass the processMessage as a callback + await kafka.startKafkaConsumer(compactor.processMessage.bind(compactor)); + } catch (error) { + logger.error('Error starting Compactor: %o', error); + } + } + // Method to stop consuming messages + async function stopKafkaConsumer() { + try { + await kafka.disconnect(); + logger.info('Compactor stopped Kafka consumption.'); + } catch (error) { + logger.error('Error stopping Compactor:: %o', error); + } + } +} else { + const MAX_WAIT_TIME_ON_EXIT = ms("30 sec"); await startPrevVersionProcess(); -} - -async function startKafkaProcess() { - let state = await getState(compactor.getOpts(), compactor.getMinioClient()); - if(state.size === 0) { + async function startPrevVersionProcess() { try { // Run the compact function immediately at launch logger.info('Running compactor at launch...'); await run(); + + // Set an interval to run compact periodically if not in debug mode + if (!isInDebugMode()) { + intervalId = setInterval(run, config.refreshInterval); + } else { + await run(); + } + logger.info('Compactor initialized.'); } catch (error) { - logger.error('Error during compactor initialization:', error); + logger.error('Error during compactor initialization:: %o', error); } + + process.on('SIGTERM', () => { + logger.info('SIGTERM signal received: terminating'); + if (intervalId !== undefined) { + clearInterval(intervalId); + } + compactor.shouldExit = true; + }); + + process.on('SIGINT', () => { + logger.info('SIGINT signal received'); + if (intervalId !== undefined) { + clearInterval(intervalId); + } + compactor.shouldExit = true; + setTimeout(() => { + logger.info('Exiting'); + process.exit(); + }, MAX_WAIT_TIME_ON_EXIT); + }); + + + process.on('SIGUSR2',function(){ + logger.info("SIGUSR2 signal received"); + if (!compactor.status.processing) { + setImmediate(run); + logger.info(`Force run`); + } + const status = compactor.status; + const elapsedTime = compactor.elapsedTime; + logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`); + }); } - - // Start consuming messages - (async () => { - await compactor.startKafkaConsumer(); - })(); } - -async function startPrevVersionProcess() { - try { - // Run the compact function immediately at launch - logger.info('Running compactor at launch...'); - await run(); - - // Set an interval to run compact periodically if not in debug mode - if (!isInDebugMode()) { - intervalId = setInterval(run, config.refreshInterval); - } else { - await run(); - } - - logger.info('Compactor initialized.'); - } catch (error) { - logger.error('Error during compactor initialization:', error); - } -} -process.on('SIGTERM', () => { - logger.info('SIGTERM signal received: terminating'); - if (intervalId !== undefined) { - clearInterval(intervalId); - } - compactor.shouldExit = true; -}); - -process.on('SIGINT', () => { - logger.info('SIGINT signal received'); - if (intervalId !== undefined) { - clearInterval(intervalId); - } - compactor.shouldExit = true; - setTimeout(() => { - logger.info('Exiting'); - process.exit(); - }, MAX_WAIT_TIME_ON_EXIT); -}); - - -process.on('SIGUSR2',function(){ - logger.info("SIGUSR2 signal received"); - if (!compactor.status.processing) { - setImmediate(run); - logger.info(`Force run`); - } - const status = compactor.status; - const elapsedTime = compactor.elapsedTime; - logger.info(`Status: ${status.current} / ${status.total}, elapsedTime: ${elapsedTime}`); -}); diff --git a/src/kafka.js b/src/kafka.js old mode 100644 new mode 100755 index c6b5068..4b54870 --- a/src/kafka.js +++ b/src/kafka.js @@ -28,11 +28,12 @@ export class KafkaClient { logger.info(`Subscribed to topic: ${this.topic}`); } catch (error) { console.error('Error connecting to Kafka:', error); + throw error; } } // Run the consumer and process messages - async consumeLatestMessages(onMessage) { + async startKafkaConsumer(onMessage) { try { await this.connect(); @@ -59,7 +60,8 @@ export class KafkaClient { } }); } catch (error) { - console.error('Error consuming messages:', error); + console.error('Error starting Kafka messages consumer :', error); + throw error; } } @@ -70,6 +72,7 @@ export class KafkaClient { logger.info('Kafka consumer disconnected'); } catch (error) { console.error('Error disconnecting from Kafka:', error); + throw error; } } } diff --git a/src/logger.js b/src/logger.js old mode 100644 new mode 100755 index 9ef321c..0b0be2b --- a/src/logger.js +++ b/src/logger.js @@ -1,9 +1,13 @@ import { config } from './config.js'; import pino from 'pino'; +import path, { join } from 'path'; import { now } from './utils/date.js'; +import { fileURLToPath } from 'url'; -const logsFolder = new URL('../logs', import.meta.url); -const logFile = `${logsFolder.pathname}/${now().toISOString()}.log`; +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const logsFolder =process.env.LOG_FOLDER || path.join(__dirname, '../logs'); +const logFile = `${logsFolder}/${now().toISOString()}.log`; /** @type {{targets:import('pino').TransportTargetOptions[]}} */ let transport = { @@ -13,32 +17,21 @@ let transport = { level: (process.env.LOG_LEVEL || 'info').toLowerCase(), options: { destination: logFile, - mkdir: true + singleLine: true } } ] }; if (process.env.NODE_ENV !== 'production') { - transport.targets.push( - { - target: 'pino-pretty', - level: (process.env.LOG_LEVEL || 'info').toLowerCase(), - options: { - ignore: 'pid,hostname' - } - } - ); -} else { transport.targets.push( { - target: 'pino/file', + target: 'pino-pretty', level: (process.env.LOG_LEVEL || 'info').toLowerCase(), options: { - singleLine: true, ignore: 'pid,hostname' } } -); + ); } /** @type {import('pino').LoggerOptions} */ diff --git a/src/minio.js b/src/minio.js old mode 100644 new mode 100755 index bc227cf..b79e9a2 --- a/src/minio.js +++ b/src/minio.js @@ -32,6 +32,18 @@ import { logger } from './logger.js'; * @property {string} versionId versionId of the object. */ +/** + * @typedef MultipartUploadResult + * @property {string} uploadId uploadId of the object. + * @property {string} key key path of the object. + */ + +/** + * @typedef MetadataObject + * @property {string} Version + * @property {string} Content-Type + */ + export class MinioClient { /** @@ -69,6 +81,13 @@ export class MinioClient { return files; } + /** + * @param {string} filePath + */ + async getMetadataObject(filePath) { + return this.#minio.statObject(this.#opts.bucket, filePath); + } + /** * * @param {string} file @@ -97,10 +116,12 @@ export class MinioClient { * * @param {string} remotePath * @param {string} localPath - * @returns {Promise} + * @param {MetadataObject} [metadata] + * @returns {Promise} */ - async copyToRemoteFile(localPath, remotePath) { - return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath); + async copyToRemoteFile(localPath, remotePath, metadata) { + logger.debug(`Coping file ${localPath} to remote ${remotePath}`); + return this.#minio.fPutObject(this.#opts.bucket, remotePath, localPath, metadata); } /** @@ -128,6 +149,7 @@ export class MinioClient { * @returns {Promise} */ async removeRemoteFile(path) { + logger.debug(`removeRemoteFile file ${path}`); return this.#minio.removeObject(this.#opts.bucket, path); } @@ -152,6 +174,21 @@ export class MinioClient { return ! nextValue.done; } + + /** + * + * @param {string} sourcePath + * @param {string} destinationPath + * @returns {Promise} + */ + async copyWithinMinIO(sourcePath, destinationPath) { + logger.debug(`Coping file into remote from ${sourcePath} to ${destinationPath}`); + this.#minio.copyObject( + this.#opts.bucket, + destinationPath, + `/${this.#opts.bucket}/${sourcePath}` + ); + } } function streamToString(stream) { diff --git a/src/profiling.js b/src/profiling.js new file mode 100755 index 0000000..00e3cca --- /dev/null +++ b/src/profiling.js @@ -0,0 +1,10 @@ +import path from 'path'; +import { logger } from './logger.js'; +import { fileURLToPath } from 'url'; +import v8 from 'v8'; +import process from 'node:process'; +import ms from 'ms'; + +if(process.env.NODE_ENV == "development" && process.env.ENABLE_DEBUG_PROFILING == "true") { + logger.info("Profiling in progress..."); +} \ No newline at end of file diff --git a/src/prueba.js b/src/prueba.js old mode 100644 new mode 100755 diff --git a/src/simva.js b/src/simva.js old mode 100644 new mode 100755 diff --git a/src/state.js b/src/state.js old mode 100644 new mode 100755 index 4cb501f..c8aa785 --- a/src/state.js +++ b/src/state.js @@ -2,14 +2,16 @@ import { MinioClient } from './minio.js'; import { logger } from './logger.js'; import { join } from 'node:path'; import { copyNoOverwrite, ensureDirectoryStructureExists, fileExists, forceRemove, listFiles, mktempPath, rename, withFile } from './utils/file.js'; +import { areArraysEqual, isStringArray, getFirstAndLastX } from './utils/array.js'; import { duration, epoch, formatDuration, now, parseDate } from './utils/date.js'; -import { diffArray, diffSet } from './utils/misc.js'; +import { binarySearch, diffArray, diffSet } from './utils/misc.js'; /** @typedef {import('./config.js').CompactorOptions} CompactorOptions */ /** * @typedef SerializedCompactorState * @property {string} lastGC + * @property {string} version * @property {Map} states */ @@ -25,7 +27,6 @@ export class ActivityCompactionState { this.activityId = activityId; this.#opts = opts; this.#minio = minio; - this.owners = []; this.currentSha1 = null; this.lastUpdate = epoch(); } @@ -39,9 +40,6 @@ export class ActivityCompactionState { /** @type {string} */ activityId; - /** @type {string[]} */ - owners; - /** @type {string} */ currentSha1; @@ -68,39 +66,130 @@ export class ActivityCompactionState { return files; } + /** + * + * @param {string[]} files + * @returns {Promise} + */ + async insertOrdered(files) { + let activityFiles = await this.files(); + for(let i in files) { + let file = files[i]; + let positionvalue=-binarySearch(activityFiles, file, true, (a,b)=> { + if(typeof a == "string" && typeof b == "string" ) { + return a.localeCompare(b); + } else { + return -1; + } + })-1; + + const nextposition = activityFiles.length; + logger.debug(file); + logger.debug("positionvalue:"); + logger.debug(positionvalue); + logger.debug("nextposition:"); + logger.debug(nextposition); + + if(positionvalue < nextposition) { + logger.warn("Not ordered. Should have been consumed before.") + activityFiles.splice(positionvalue, 0, file); + } else { + activityFiles.push(file); + } + logger.debug(activityFiles); + } + return activityFiles; + } + async garbageCollect() { if (this.currentSha1 === undefined) { - return; + return 0; } const localHashesAndFiles = await this.#listLocalHashes(); const remoteHashesAndFiles = await this.#listRemoteHashes(); - + localHashesAndFiles.forEach((value, key) => { + logger.debug(`${key}: ${value.join(", ")}`); + }); + remoteHashesAndFiles.forEach((value, key) => { + logger.debug(`${key}: ${value.join(", ")}`); + }); const localHashes = new Set(localHashesAndFiles.keys()); const remoteHashes = new Set(remoteHashesAndFiles.keys()); const setsDiff = diffSet(localHashes, remoteHashes); - if (setsDiff.added.length > 0 || setsDiff.removed.length > 0) { - logger.warn('Local and remote folders for activity not synced, garbage collection skipped: %s', this.activityId); - return; - } - + logger.debug(setsDiff); /** @type {string[]} */ let localFilesToRemove = []; /** @type {string[]} */ let remoteFilesToRemove = []; + if (setsDiff.added.length > 0 || setsDiff.removed.length > 0) { + logger.warn('Local and remote folders for activity not synced for activity %s', this.activityId); + if(setsDiff.added.includes(this.currentSha1)) { + logger.warn('%s current local file not present: %s', this.currentSha1, this.activityId); + logger.info('garbage collection skipped: %s', this.activityId); + return; + //this.#minio.copyFromRemoteFile(this.#filesStateRemotePath(), this.#filesStateLocalPath()); + //this.#minio.copyFromRemoteFile(this.#stateRemotePath(), this.#stateLocalPath()); + //setsDiff.added= setsDiff.added.filter(sha1 => (sha1 !== this.currentSha1)); + } else if(setsDiff.removed.includes(this.currentSha1)) { + logger.warn('%s current remote file not present: %s', this.currentSha1, this.activityId); + logger.info('garbage collection skipped: %s', this.activityId); + return; + //this.#minio.copyToRemoteFile(this.#filesStateLocalPath(), this.#filesStateRemotePath()); + //this.#minio.copyToRemoteFile(this.#stateLocalPath(), this.#stateRemotePath()); + //setsDiff.removed= setsDiff.removed.filter(sha1 => (sha1 !== this.currentSha1)); + } else { + let localFileContent= (await this.#loadLocalState()); + localFileContent.push(""); + logger.debug("localFileContent:"); + logger.debug(getFirstAndLastX(localFileContent, 3)); + let remoteFileContent= (await this.#minio.getFile(this.#stateRemotePath())).split('\n'); + logger.debug("remoteFileContent:"); + logger.debug(getFirstAndLastX(remoteFileContent, 3)); + if(areArraysEqual(localFileContent, remoteFileContent)) { + logger.info("Local file is the same that the remote file."); + } else { + logger.warn("Remote file is different that the local file."); + logger.warn('garbage collection skipped: %s', this.activityId); + return; + //this.#minio.copyFromRemoteFile(this.#filesStateRemotePath(), this.#filesStateLocalPath()); + //this.#minio.copyFromRemoteFile(this.#stateRemotePath(), this.#stateLocalPath()); + } + } + if(setsDiff.added.length > 0) { + setsDiff.added.forEach((hash) => { + const remoteFiles = remoteHashesAndFiles.get(hash); + if(isStringArray(remoteFiles)) { + remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles); + } + }); + } + + if(setsDiff.removed.length > 0) { + setsDiff.removed.forEach((hash) => { + const localFiles = localHashesAndFiles.get(hash); + if(isStringArray(localFiles)) { + localFilesToRemove = localFilesToRemove.concat(localFiles); + } + }); + } + } + localHashes.forEach((hash) => { if(hash !== this.currentSha1) { const localFiles = localHashesAndFiles.get(hash); - localFilesToRemove = localFilesToRemove.concat(localFiles); + if(isStringArray(localFiles)) { + localFilesToRemove = localFilesToRemove.concat(localFiles); + } const remoteFiles = remoteHashesAndFiles.get(hash); - remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles); + if(isStringArray(remoteFiles)) { + remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles); + } } }); - setsDiff.added.forEach((hash) => { - const remoteFiles = remoteHashesAndFiles.get(hash); - remoteFilesToRemove = remoteFilesToRemove.concat(remoteFiles); - }); + await this.#removeRemoteFiles(remoteFilesToRemove); await this.#removeLocalFiles(localFilesToRemove); + return 0; } /** @@ -113,13 +202,14 @@ export class ActivityCompactionState { const activityPath = join(this.#opts.localStatePath, this.activityId); const files = await listFiles(activityPath); for(const file of files) { + const filePath= join(activityPath, file); const chunks = file.split('-'); const hash = chunks[0]; const entry = hashes.get(hash); if (entry) { - entry.push(file); + entry.push(filePath); } else { - hashes.set(hash, [file]); + hashes.set(hash, [filePath]); } } if (this.currentSha1 !== undefined && hashes.size < 2 ) { @@ -139,7 +229,7 @@ export class ActivityCompactionState { const files = await this.#minio.listFiles(remotePath); for(const file of files) { const chunks = file.name.split('-'); - const hash = chunks[0]; + const hash = chunks[0].replace(remotePath, ""); const entry = hashes.get(hash); if (entry) { entry.push(file.name); @@ -155,7 +245,6 @@ export class ActivityCompactionState { * @param {string[]} files */ async #removeRemoteFiles(files) { - files = files.map((file) => `${this.#opts.remoteStatePath}/${this.activityId}/${file}`); if (this.#opts.removeDryRun) { logger.debug('DRY RUN - Removed remote files:\n %s', files.join(',\n')); } else { @@ -170,12 +259,11 @@ export class ActivityCompactionState { */ async #removeLocalFiles(files) { for(const file of files) { - const localPath = join(this.#opts.localStatePath, this.activityId, file); if (this.#opts.removeDryRun) { - logger.debug('DRY RUN - Removed local file: %s', localPath); + logger.debug('DRY RUN - Removed local file: %s', file); } else { - await forceRemove(localPath); - logger.debug('Removed local file: %s', localPath); + await forceRemove(file); + logger.debug('Removed local file: %s', file); } } } @@ -238,6 +326,17 @@ export class ActivityCompactionState { return path; } + /** + * + * @returns + */ + #outputRemotePath() { + const path = join(this.#opts.minio.outputs_dir, this.activityId,this.#opts.minio.traces_file); + return path; + } + + + async #copyFromRemoteFilesState() { const remotePath = this.#filesStateRemotePath(); const localPath = this.#filesStateLocalPath(); @@ -296,6 +395,28 @@ export class ActivityCompactionState { await rename(tmpPath, statePath, this.#opts.copyInsteadRename); } + + /** + * + * @returns + */ + async #loadLocalState() { + const statePath = this.#stateLocalPath(); + const withState = withFile(statePath); + + /** @type {string[]} */ + const files = await withState(async (state) => { + /** @type {string[]} */ + const files=[]; + for await (const line of state.readLines()) { + files.push(line); + } + return files; + }, false); + + return files; + } + /** * * @param {string} [sha1] @@ -303,7 +424,7 @@ export class ActivityCompactionState { */ #stateLocalPath(sha1) { sha1 = sha1 || this.currentSha1; - const path = join(this.#opts.localStatePath, this.activityId, `${sha1}-state.txt`); + const path = join(this.#opts.localStatePath, this.activityId, `${sha1}-state.json`); return path; } @@ -314,6 +435,20 @@ export class ActivityCompactionState { return this.#stateLocalPath(); } + /** + * @returns + */ + get remoteStatePath() { + return this.#stateRemotePath(); + } + + /** + * @returns + */ + get remoteOutputPath() { + return this.#outputRemotePath(); + } + /** * * @param {string} [sha1] @@ -321,7 +456,7 @@ export class ActivityCompactionState { */ #stateRemotePath(sha1) { sha1 = sha1 || this.currentSha1; - return `${this.#opts.remoteStatePath}/${this.activityId}/${sha1}-state.txt`; + return `${this.#opts.remoteStatePath}/${this.activityId}/${sha1}-state.json`; } /** @@ -333,16 +468,11 @@ export class ActivityCompactionState { async #saveLocalFilesState(filesToAdd, sha1) { const filesStatePath = this.#filesStateLocalPath(sha1); const tmpPath = mktempPath(); - if (this.currentSha1 !== null && this.currentSha1 !== sha1) { - const currentFilesStatePath = this.#filesStateLocalPath(); - await copyNoOverwrite(currentFilesStatePath, tmpPath); - } + const stateFiles = (await this.insertOrdered(filesToAdd)).join('\r\n'); const withFilesState = withFile(tmpPath, 'a'); - await withFilesState(async (filesState) => { - for (const line of filesToAdd) { - await filesState.write(line+'\n'); - } - }); + await withFilesState(async (file) => { + await file.writeFile(stateFiles); + }) await rename(tmpPath, filesStatePath, this.#opts.copyInsteadRename); } @@ -363,7 +493,11 @@ export class ActivityCompactionState { async #copyToRemoteState(sha1) { const localPath = this.#stateLocalPath(sha1); const remotePath = this.#stateRemotePath(sha1); - await this.#minio.copyToRemoteFile(localPath, remotePath); + let metadata={ + "Content-Type": "application/json", + "Version": "1" + }; + await this.#minio.copyToRemoteFile(localPath, remotePath, metadata); } async checkConsistency() { @@ -375,6 +509,43 @@ export class ActivityCompactionState { let consistent = true; const localStatePath = this.#stateLocalPath(); const localFilesStatePath = this.#filesStateLocalPath(); + + const remoteStatePath = this.#stateRemotePath(); + const remoteFilesStatePath = this.#filesStateRemotePath(); + + if (! await fileExists(localStatePath)) { + let previousLocalStatePath=localStatePath.replace(".json",".txt"); + if(await fileExists(previousLocalStatePath)) { + try { + await rename(previousLocalStatePath, localStatePath, this.#opts.copyInsteadRename); + let metadata= { + "Content-Type": "application/json", + "Version": "1" + }; + await this.#minio.copyToRemoteFile(localStatePath, remoteStatePath, metadata); + } catch(error) { + logger.error("Copy failed:"); + logger.error(error); + throw error; + } + } else { + let previousRemoteStatePath=remoteStatePath.replace(".json",".txt"); + if(await this.#minio.fileExists(previousRemoteStatePath)) { + try { + await this.#minio.copyFromRemoteFile(previousRemoteStatePath, localStatePath); + let metadata= { + "Content-Type": "application/json", + "Version": "1" + }; + await this.#minio.copyToRemoteFile(localStatePath, remoteStatePath, metadata); + } catch(error) { + logger.error("Copy failed:"); + logger.error(error); + throw error; + } + } + } + } if (! await fileExists(localStatePath)) { logger.warn('Local state file for activity \'%s\' not found: %s', this.activityId, localStatePath); consistent = false; @@ -383,8 +554,7 @@ export class ActivityCompactionState { logger.warn('Local files state for activity \'%s\' not found: %s', this.activityId, localFilesStatePath); consistent = false; } - const remoteStatePath = this.#stateRemotePath(); - const remoteFilesStatePath = this.#filesStateRemotePath(); + if (! await this.#minio.fileExists(remoteStatePath)) { logger.warn('Remote state file for activity \'%s\' not found: %s', this.activityId, remoteStatePath); consistent = false; @@ -410,6 +580,7 @@ export class CompactorState { this.#minio = minio; this.#states = new Map(); this.#lastGC = null; + this.#version = null; } /** @type {CompactorOptions} opts */ #opts; @@ -423,12 +594,33 @@ export class CompactorState { /** @type {Date} */ #lastGC; + /** @type {Number} */ + #version; + async init() { - let loaded = await this.#loadLocalState(); - if (loaded) return; - loaded = await this.#loadRemoteState(); - if (!loaded) { + logger.debug("Loading Local State...") + let localStateLoaded = await this.#loadLocalState(false); + let localStateVersion=this.#version; + logger.debug("Loading Remote State..."); + let remoteStateLoaded = await this.#loadRemoteState(true); + let remoteStateVersion=this.#version; + if (!localStateLoaded && !remoteStateLoaded) { logger.warn('Seems that we are running for the first time'); + return; + } + logger.debug(`Local version : ${localStateVersion } - Remote version : ${remoteStateVersion }`) + if(remoteStateVersion > localStateVersion) { + logger.debug("The version in remote is more uptodate. Taking this version."); + await this.#loadRemoteState(false); + return; + } else if(remoteStateVersion < localStateVersion) { + logger.debug("The version in local is more uptodate. Taking this version."); + await this.#loadLocalState(false); + await this.#copyStateToRemote(); + return; + } else { + logger.debug("The versions in local and in remote are the same."); + return; } } @@ -445,42 +637,71 @@ export class CompactorState { } async garbageCollect() { + let activityToPass=[]; const nowDate = now(); const lastGC = this.#lastGC ?? epoch(); const elapsedTimeSinceLastGC = duration(lastGC, nowDate); if (elapsedTimeSinceLastGC < this.#opts.gcInterval) { - return; + return []; } logger.debug('Garbage collection started'); for(const activity of this.#states.values()) { try { - await activity.garbageCollect(); + let result = await activity.garbageCollect(); + logger.info("activity.garbageCollect : " + result); + switch(result) { + case 1: + logger.warn("Error during garbage collection."); + if(this.#opts.tryRecovery) { + logger.warn("Removing activity :" + activity.activityId); + await activity.clear(); + await this.remove(activity.activityId); + } + activityToPass.push(activity.activityId); + break; + default: + logger.info("Everything ok during garbage collection."); + } } catch (error) { logger.error('Could not garbage collect: %s', activity.activityId); logger.error(error); + throw error; } } const finishTime = now(); logger.info('Garbage collection finished, took: %s', formatDuration(duration(nowDate, finishTime))); this.#lastGC = finishTime; + return activityToPass; } /** + * @param {boolean } loadTemp * @return {Promise} true if config has been loaded */ - async #loadLocalState() { - const path = this.#localPath; - const withState = withFile(path); - const result = await withState(async (file) => { - const content = await file.readFile('utf-8'); - await this.#initState(content); - return true; - }, false); - if (result !== undefined) { - return result; + async #loadLocalState(loadTemp) { + try { + let path; + if(loadTemp) { + path = this.#localTempPath; + } else { + path = this.#localPath; + } + const withState = withFile(path); + const result = await withState(async (file) => { + const content = await file.readFile('utf-8'); + await this.#initState(content); + return true; + }, false); + if (result !== undefined) { + return result; + } + return false; + } catch(e) { + logger.warn(e); + return false; } - return false; + } get #localPath () { @@ -488,17 +709,29 @@ export class CompactorState { return path; } + get #localTempPath () { + const path = join(this.#opts.localStatePath, `temp_${STATE_FILENAME}`); + return path; + } + /** + * @param {boolean} loadTemp * @return {Promise} true if config has been loaded */ - async #loadRemoteState() { + async #loadRemoteState(loadTemp) { + let path; + if(loadTemp) { + path = this.#localTempPath; + } else { + path = this.#localPath; + } try { - await this.#minio.copyFromRemoteFile(this.#remotePath, this.#localPath); - return this.#loadLocalState(); + await this.#minio.copyFromRemoteFile(this.#remotePath, path); + return this.#loadLocalState(loadTemp); } catch (e) { logger.warn(e); + return false; } - return false; } get #remotePath() { @@ -518,14 +751,18 @@ export class CompactorState { } catch (error) { logger.error('Could not initialize activity: ', activity.activityId); logger.error(error); + throw error; } } this.#lastGC = serializedState.lastGC !== null ? new Date(Date.parse(serializedState.lastGC)) : null; + this.#version = serializedState.version !== null ? (serializedState.version !== "NaN" ? parseInt(serializedState.version) : 0) : 0 ; } async save() { + this.#version =this.#version+1; /** @type {SerializedCompactorState} */ const serializedState = { + version: this.#version.toString(), states: this.#states, lastGC: this.#lastGC !== null ? this.#lastGC.toISOString() : null } @@ -618,8 +855,7 @@ function replacer(key, value) { value: { activityId : value.activityId, lastUpdate : value.lastUpdate.toISOString(), - currentSha1 : value.currentSha1, - owners : value.owners + currentSha1 : value.currentSha1 } } } @@ -642,7 +878,6 @@ function withContextReviver(opts, minio) { const activityState = new ActivityCompactionState(value.activityId, opts, minio); activityState.lastUpdate = parseDate(value.lastUpdate); activityState.currentSha1 = value.currentSha1; - activityState.owners = value.owners; return activityState; } } diff --git a/src/utils/array.js b/src/utils/array.js new file mode 100755 index 0000000..39deab8 --- /dev/null +++ b/src/utils/array.js @@ -0,0 +1,35 @@ +/** + * Compare two arrays of strings. + * + * @param {string[]} arr1 + * @param {string[]} arr2 + */ +export function areArraysEqual(arr1, arr2) { + return arr1.length === arr2.length && arr1.every((val, index) => val === arr2[index]); +} + +/** + * Check if the given `value` is an array of strings. + * + * @param {Object} value + */ +export function isStringArray(value) { + return Array.isArray(value) && value.every(item => typeof item === "string"); +} + +/** + * Print the first and last `x` elements of an array. + * + * @param {string[]} arr + * @param {number} x + */ +export function getFirstAndLastX(arr, x) { + if (arr.length <= 2 * x) { + return arr; // If the array is small, print it all + } + + const firstX = arr.slice(0, x); // First X elements + const lastX = arr.slice(-x); // Last X elements + + return [...firstX, "...", ...lastX]; // Using "..." as a placeholder +} \ No newline at end of file diff --git a/src/utils/date.js b/src/utils/date.js old mode 100644 new mode 100755 index c48b445..3ade916 --- a/src/utils/date.js +++ b/src/utils/date.js @@ -79,4 +79,62 @@ export function formatDuration(durationMillis) { duration.seconds = value; return Duration.serialize(duration); +} + +/** + * + * @param {number} durationMin + * @returns {string} + */ +export function convertTimeToCron(durationMin) { + if (durationMin <= 0) { + throw new Error("Time should be greater than 0."); + } + + if (durationMin < 60) { + return `*/${durationMin} * * * *`; // Runs every N minutes + } + + const hours = Math.floor(durationMin / 60); + const extraMinutes = durationMin % 60; + + if (hours < 24) { + let cronExpressions = [`0 */${hours} * * *`]; // Runs every X hours + if (extraMinutes > 0) { + cronExpressions.push(`${extraMinutes} */${hours} * * *`); // Extra minute offset + } + return cronExpressions.join("\n"); + } + + const days = Math.floor(hours / 24); + const extraHours = hours % 24; + + if (days < 7) { + let cronExpressions = [`0 0 */${days} * *`]; // Runs every X days + if (extraHours > 0) { + cronExpressions.push(`0 */${extraHours} * * *`); // Extra hourly offset + } + return cronExpressions.join("\n"); + } + + const weeks = Math.floor(days / 7); + const extraDays = days % 7; + + if (weeks < 4) { + let cronExpressions = [`0 0 * */${weeks} *`]; // Runs every X weeks + if (extraDays > 0) { + cronExpressions.push(`0 0 */${extraDays} * *`); // Extra daily offset + } + return cronExpressions.join("\n"); + } + + const months = Math.floor(weeks / 4); + const extraWeeks = weeks % 4; + + let cronExpressions = [`0 0 1 */${months} *`]; // Runs every X months + if (extraWeeks > 0) { + cronExpressions.push(`0 0 * */${extraWeeks} *`); // Extra weekly offset + } + + return cronExpressions.join("\n"); } \ No newline at end of file diff --git a/src/utils/errors.js b/src/utils/errors.js old mode 100644 new mode 100755 diff --git a/src/utils/file.js b/src/utils/file.js old mode 100644 new mode 100755 index 0b0bbbb..561024a --- a/src/utils/file.js +++ b/src/utils/file.js @@ -276,4 +276,4 @@ export async function rename(source, dest, useCopy = false) { } catch (error) { throw wrapError(error, `Could not rename '${source}' to '${dest}'`); } -} \ No newline at end of file +} diff --git a/src/utils/misc.js b/src/utils/misc.js old mode 100644 new mode 100755 diff --git a/src/utils/sha.js b/src/utils/sha.js new file mode 100644 index 0000000..329db0c --- /dev/null +++ b/src/utils/sha.js @@ -0,0 +1,51 @@ +import { createHash } from 'node:crypto'; + +/** + * @overload + * + * @param {string} str + * + * @returns {string} + */ +/** + * @overload + * + * @param {string[]} strings + * + * @returns {string} + */ +/** + * + * @param {string | string[]} strings + * + * @returns {string} + */ +export function sha1sums(strings) { + /** @type {string[]} */ + let str; + if (Array.isArray(strings)) { + str = strings; + } else if (typeof strings === 'string') { + str = [ strings ] + } else { + throw new TypeError(`'strings' must be string or string[]`); + } + + const hash = createSha1(); + for(const s of strings) { + hash.update(s); + hash.update('\n'); + } + const sha1 = hash.digest('hex'); + return sha1; +} + + +/** + * Create sha1 hash function. + * + * @returns + */ +function createSha1() { + return createHash('sha1'); +}