Tests refactoring and reanimate test_keeper.py

.github/workflows/run_tests.yaml

@@ -47,7 +47,7 @@ jobs:
         with:
           driver: docker
           container-runtime: containerd
-          kubernetes-version: v1.33.5
+          kubernetes-version: v1.34.3
           cpus: max
           memory: max
 
@@ -108,17 +108,27 @@ jobs:
             test_mode="--test-to-end"
           fi
 
-          ~/venv/qa/bin/python3 ./tests/regression.py --only=/regression/e2e.test_operator/${ONLY} $test_mode --trim-results on -o short --no-colors --native --log ./tests/raw.log
+          for test_file in ./tests/e2e/test_operator*.py; do
+            name=$(basename "$test_file" .py | sed 's/^test_//')
+            run_cmd="~/venv/qa/bin/python3 ./tests/regression.py --only=/regression/e2e?test_${name}/${ONLY} $test_mode --no-colors --trim-results on --debug --native --log ./tests/raw_${name}.log && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform compact ./tests/raw_${name}.log ./tests/compact_${name}.log.txt && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform nice ./tests/raw_${name}.log ./tests/nice_${name}.log.txt && "
+            run_cmd+="~/venv/qa/bin/tfs --no-colors transform short ./tests/raw_${name}.log ./tests/short_${name}.log.txt && "
+            run_cmd+="bash -xec '~/venv/qa/bin/tfs --no-colors report results -a '${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/' ./tests/raw_${name}.log - --confidential --copyright 'Altinity Inc.' --logo ./tests/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./tests/report_${name}.html'"
+
+            run_tests+=(
+              "${run_cmd}"
+            )
+          done          
+          printf "%s\n" "${run_tests[@]}" | xargs -P 2 -I {} bash -xec '{}'
+
+          ls -la ./tests/*.html
           test_result=$?
-          ~/venv/qa/bin/tfs --no-colors transform compact ./tests/raw.log ./tests/compact.log
-          ~/venv/qa/bin/tfs --no-colors transform nice ./tests/raw.log ./tests/nice.log.txt
-          ~/venv/qa/bin/tfs --no-colors transform short ./tests/raw.log ./tests/short.log.txt
-          ~/venv/qa/bin/tfs --no-colors report results -a "${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}/" ./tests/raw.log - --confidential --copyright "Altinity Inc." --logo ./tests/altinity.png | ~/venv/qa/bin/tfs --debug --no-colors document convert > ./tests/report.html
 
           echo "test_result=$test_result" >> $GITHUB_OUTPUT
           exit "$test_result"
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         with:
           name: testflows-logs
           path: |
@@ -127,7 +137,7 @@ jobs:
           if-no-files-found: error
           retention-days: 7
 
-      - uses: actions/upload-artifact@v4
+      - uses: actions/upload-artifact@v6
         with:
           name: testflows-report
           path: |

.gitignore

@@ -44,6 +44,8 @@ venv
 
 # Tests cached files
 tests/image/cache
+tests/*.log.txt
+tests/*.html
 
 # Skip tmp folder
 /tmp/

Vagrantfile

@@ -121,15 +121,13 @@ Vagrant.configure(2) do |config|
     # docker
     curl -fsSL https://download.docker.com/linux/ubuntu/gpg | apt-key add -
     add-apt-repository "deb https://download.docker.com/linux/ubuntu $(lsb_release -cs) test"
-    apt-get install --no-install-recommends -y docker-ce pigz
+    apt-get install --no-install-recommends -y docker-ce docker-ce-cli containerd.io docker-compose-plugin pigz
 
-    # docker compose
     apt-get install -y --no-install-recommends python3-distutils
     curl -sL https://bootstrap.pypa.io/get-pip.py -o /tmp/get-pip.py
     python3 /tmp/get-pip.py
 
-    pip3 install -U setuptools
-    pip3 install -U docker-compose
+    pip3 install -U -r ./tests/image/requirements.txt
 
     # k9s CLI
     K9S_VERSION=$(curl -sL https://github.com/derailed/k9s/releases/latest -H "Accept: application/json" | jq -r .tag_name)
@@ -170,7 +168,8 @@ Vagrant.configure(2) do |config|
 #    K8S_VERSION=${K8S_VERSION:-1.23.1}
 #    K8S_VERSION=${K8S_VERSION:-1.24.8}
 #    K8S_VERSION=${K8S_VERSION:-1.25.4}
-    K8S_VERSION=${K8S_VERSION:-1.31.1}
+#    K8S_VERSION=${K8S_VERSION:-1.31.1}
+    K8S_VERSION=${K8S_VERSION:-1.31.2}
     export VALIDATE_YAML=true
 
     killall kubectl || true

config/chi/templates.d/001-templates.json.example

@@ -29,7 +29,7 @@
             "containers" : [
               {
                 "name": "clickhouse",
-                "image": "clickhouse/clickhouse-server:23.8",
+                "image": "clickhouse/clickhouse-server:24.8",
                 "ports": [
                   {
                     "name": "http",

deploy/builder/templates-config/chi/templates.d/001-templates.json.example

@@ -29,7 +29,7 @@
             "containers" : [
               {
                 "name": "clickhouse",
-                "image": "clickhouse/clickhouse-server:23.8",
+                "image": "clickhouse/clickhouse-server:24.8",
                 "ports": [
                   {
                     "name": "http",

deploy/clickhouse-keeper/clickhouse-keeper-manually/clickhouse-keeper-1-node-256M-for-test-only.yaml

@@ -382,13 +382,13 @@ spec:
       containers:
         - name: clickhouse-keeper
           imagePullPolicy: Always
-          image: "clickhouse/clickhouse-keeper:latest-alpine"
+          image: "clickhouse/clickhouse-keeper:24.8"
           resources:
             requests:
               memory: "256M"
               cpu: "1"
             limits:
-              memory: "4Gi"
+              memory: "1Gi"
               cpu: "2"
           volumeMounts:
             - name: clickhouse-keeper-settings

deploy/clickhouse-keeper/clickhouse-keeper-manually/clickhouse-keeper-3-nodes-256M-for-test-only.yaml

@@ -382,13 +382,13 @@ spec:
       containers:
         - name: clickhouse-keeper
           imagePullPolicy: Always
-          image: "clickhouse/clickhouse-keeper:latest-alpine"
+          image: "clickhouse/clickhouse-keeper:24.8"
           resources:
             requests:
               memory: "256M"
               cpu: "1"
             limits:
-              memory: "4Gi"
+              memory: "1Gi"
               cpu: "2"
           volumeMounts:
             - name: clickhouse-keeper-settings

deploy/helm/clickhouse-operator/templates/generated/ClusterRoleBinding-clickhouse-operator-kube-system.yaml

@@ -20,7 +20,6 @@ subjects:
   - kind: ServiceAccount
     name: {{ include "altinity-clickhouse-operator.serviceAccountName" . }}
     namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
-
 # Template Parameters:
 #
 # NAMESPACE=kube-system

deploy/helm/clickhouse-operator/templates/generated/ServiceAccount-clickhouse-operator.yaml

@@ -13,7 +13,6 @@ metadata:
   namespace: {{ include "altinity-clickhouse-operator.namespace" . }}
   labels: {{ include "altinity-clickhouse-operator.labels" . | nindent 4 }}
   annotations: {{ include "altinity-clickhouse-operator.annotations" . | nindent 4 }}{{ if .Values.serviceAccount.annotations }}{{ toYaml .Values.serviceAccount.annotations | nindent 4 }}{{ end }}
-
 # Template Parameters:
 #
 # NAMESPACE=kube-system

deploy/helm/clickhouse-operator/values.yaml

@@ -724,7 +724,7 @@ configs:
                   "containers" : [
                     {
                       "name": "clickhouse",
-                      "image": "clickhouse/clickhouse-server:23.8",
+                      "image": "clickhouse/clickhouse-server:24.8",
                       "ports": [
                         {
                           "name": "http",

deploy/operator/clickhouse-operator-install-ansible.yaml

@@ -5306,7 +5306,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/operator/clickhouse-operator-install-bundle-v1beta1.yaml

@@ -5499,7 +5499,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/operator/clickhouse-operator-install-bundle.yaml

@@ -5565,7 +5565,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/operator/clickhouse-operator-install-template-v1beta1.yaml

@@ -5246,7 +5246,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/operator/clickhouse-operator-install-template.yaml

@@ -5299,7 +5299,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/operator/clickhouse-operator-install-tf.yaml

@@ -5306,7 +5306,7 @@ data:
                 "containers" : [
                   {
                     "name": "clickhouse",
-                    "image": "clickhouse/clickhouse-server:23.8",
+                    "image": "clickhouse/clickhouse-server:24.8",
                     "ports": [
                       {
                         "name": "http",

deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-1-node-1GB-for-tests-only.yaml

@@ -82,7 +82,7 @@ spec:
       containers:
         - name: kubernetes-zookeeper
           imagePullPolicy: IfNotPresent
-          image: "docker.io/zookeeper:3.8.4"
+          image: "docker.io/zookeeper:latest"
           ports:
             - containerPort: 2181
               name: client

deploy/zookeeper/zookeeper-manually/quick-start-persistent-volume/zookeeper-3-nodes-1GB-for-tests-only.yaml

@@ -82,7 +82,7 @@ spec:
       containers:
         - name: kubernetes-zookeeper
           imagePullPolicy: IfNotPresent
-          image: "docker.io/zookeeper:3.8.4"
+          image: "docker.io/zookeeper:latest"
           ports:
             - containerPort: 2181
               name: client

docs/chi-examples/99-clickhouseinstallation-max.yaml

@@ -674,7 +674,7 @@ spec:
         spec:
           containers:
             - name: clickhouse
-              image: clickhouse/clickhouse-server:23.8
+              image: clickhouse/clickhouse-server:24.8
               volumeMounts:
                 - name: default-volume-claim
                   mountPath: /var/lib/clickhouse
@@ -686,7 +686,7 @@ spec:
                   memory: "64Mi"
                   cpu: "100m"
             - name: clickhouse-log
-              image: clickhouse/clickhouse-server:23.8
+              image: clickhouse/clickhouse-server:24.8
               command:
                 - "/bin/sh"
                 - "-c"
@@ -704,7 +704,7 @@ spec:
         spec:
           containers:
             - name: clickhouse
-              image: clickhouse/clickhouse-server:23.8
+              image: clickhouse/clickhouse-server:24.8
               volumeMounts:
                 - name: default-volume-claim
                   mountPath: /var/lib/clickhouse
@@ -716,7 +716,7 @@ spec:
                   memory: "64Mi"
                   cpu: "100m"
             - name: clickhouse-log
-              image: clickhouse/clickhouse-server:23.8
+              image: clickhouse/clickhouse-server:24.8
               command:
                 - "/bin/sh"
                 - "-c"

go.mod

@@ -1,6 +1,6 @@
 module github.com/altinity/clickhouse-operator
 
-go 1.25.4
+go 1.25
 
 replace (
 	github.com/emicklei/go-restful/v3 => github.com/emicklei/go-restful/v3 v3.10.0

tests/README.md

@@ -10,7 +10,7 @@ To execute tests, you will need:
 * TestFlows Python library (`pip3 install -r ./tests/image/requirements.txt`)
 * To run tests in docker container (approximately 2 times slower, but does not require any additional configuration):
     - `docker` 
-    - `docker-compose`
+    - `docker compose`
     - `python3`
 * To run tests natively on your machine:
     - `kubectl`
@@ -37,7 +37,7 @@ To execute the test suite (that currently involves only operator tests, not test
 ```bash
 pip3 install -U -r ./tests/image/requirements.txt
 docker pull registry.gitlab.com/altinity-public/container-images/clickhouse-operator-test-runner:latest
-COMPOSE_HTTP_TIMEOUT=1800 python3 ./tests/regression.py --only "/regression/e2e.test_operator/*"
+COMPOSE_HTTP_TIMEOUT=1800 python3 ./tests/regression.py --only "/regression/e2e?test_operator/*"
 ```
 
 To execute tests natively (not in docker), you need to add `--native` parameter.
@@ -47,7 +47,7 @@ Tests running in parallel by default, to run it consistently, add `--parallel of
 If you need only one certain test, you may execute
 
 ```bash
-COMPOSE_HTTP_TIMEOUT=1800 python3 ./tests/regression.py --only "/regression/e2e.test_operator/test_009*"
+COMPOSE_HTTP_TIMEOUT=1800 python3 ./tests/regression.py --only "/regression/e2e?test_operator/test_009*"
 ```
 
 where `009` may be substituted by the number of the test you need. Tests --- numbers and names correspondence may be found in `tests/regression.py` and `tests/test_*.py` source code files.

tests/docker-compose/docker-compose.yml

@@ -21,7 +21,7 @@ services:
       - NET_ADMIN
 
   # dummy service which does nothing, but allows to postpone
-  # 'docker-compose up -d' till all dependecies will go healthy
+  # 'docker compose up -d' till all dependecies will go healthy
   all_services_ready:
     image: hello-world
     privileged: true

tests/e2e/clickhouse.py

@@ -27,12 +27,12 @@ def query(
     pwd_str = "" if pwd == "" else f"--password={pwd}"
     user_str = "" if user == "" else f"--user={user}"
 
-    for i in [1,2,3]: # re-tries for "Unknown stream id" error
+    for _ in [1,2,3]: # re-tries for "Unknown stream id" error
         if with_error:
             res = kubectl.launch(
                 f"exec {pod_name} -n {current().context.test_namespace} -c {container}"
                 f" --"
-                f" clickhouse-client -mn -h {host} --port={port} {user_str} {pwd_str} {advanced_params}"
+                f" clickhouse-client --receive_timeout={timeout} -mn -h {host} --port={port} {user_str} {pwd_str} {advanced_params}"
                 f' --query="{sql}"'
                 f" 2>&1",
                 timeout=timeout,
@@ -43,7 +43,7 @@ def query(
             res = kubectl.launch(
                 f"exec {pod_name} -n {current().context.test_namespace} -c {container}"
                 f" -- "
-                f"clickhouse-client -mn -h {host} --port={port} {user_str} {pwd_str} {advanced_params}"
+                f"clickhouse-client --receive_timeout={timeout} -mn -h {host} --port={port} {user_str} {pwd_str} {advanced_params}"
                 f'--query="{sql}"',
                 timeout=timeout,
                 ns=current().context.test_namespace,

tests/e2e/manifests/chi/test-011-secured-default-2.yaml

@@ -6,6 +6,8 @@ spec:
   useTemplates:
     - name: clickhouse-version
   configuration:
+    profiles:
+      readonly/receive_timeout: 60
     users:
       default/profile: readonly
     clusters:

tests/e2e/manifests/chi/test-022-broken-image.yaml

@@ -5,14 +5,14 @@ metadata:
 spec:
   defaults:
     templates:
-      podTemplate: v20.3
+      podTemplate: broken-image
   templates:
     podTemplates:
-    - name: v20.3
+    - name: broken-image
       spec:
         containers:
         - name: clickhouse-pod
-          image: clickhouse/clickhouse-server:24.3-broken
+          image: clickhouse/clickhouse-server:24.8-broken
   configuration:
     clusters:
     - name: default

tests/e2e/manifests/chi/test-046-2-clickhouse-operator-metrics.yaml

@@ -15,7 +15,7 @@ spec:
       spec:
         containers:
         - name: clickhouse-pod
-          image: clickhouse/clickhouse-server:24.3-broken
+          image: clickhouse/clickhouse-server:24.8-broken
   defaults:
     templates:
       podTemplate: clickhouse-new

tests/e2e/manifests/chi/test-cluster-for-alerts.yaml

@@ -18,6 +18,14 @@ spec:
       prometheus/metrics: true
       prometheus/events: true
       prometheus/asynchronous_metrics: true
+      # tune for low memory
+      mark_cache_size: 67108864
+      merge_tree/parts_to_throw_insert: 300
+      merge_tree/parts_to_delay_insert: 150
+#      merge_tree/merge_max_block_size: 1024
+#      merge_tree/max_bytes_to_merge_at_max_space_in_pool: 1073741824
+#      merge_tree/number_of_free_entries_in_pool_to_lower_max_size_of_merge: 0
+#      background_schedule_pool_size: 128
 
     zookeeper:
       nodes:

tests/e2e/manifests/chit/tpl-clickhouse-alerts.yaml

@@ -17,5 +17,5 @@ spec:
         spec:
           containers:
             - name: clickhouse-pod
-              image: clickhouse/clickhouse-server:24.3
+              image: clickhouse/clickhouse-server:latest
               imagePullPolicy: Always