diff --git a/.bazelrc b/.bazelrc index 86182129958..df1d3cc2fee 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,8 +1,15 @@ # Global bazelrc file, see https://docs.bazel.build/versions/master/guide.html#bazelrc. +# Use local Cache directory if building on a VM: +# On Chef VM, create a directory and comment in the following line: +# build --disk_cache= # Optional for multi-user cache: Make this directory owned by a group name e.g. "bazelcache" + # Use strict action env to prevent leaks of env vars. build --incompatible_strict_action_env +# Use cache +# build --disk_cache=/tmp/bazel/cache # must not be merged dev only settng + # Only pass through GH_API_KEY for stamped builds. # This is still not ideal as it still busts the cache of stamped builds. build:stamp --stamp diff --git a/.github/workflows/build_and_test.yaml b/.github/workflows/build_and_test.yaml index 4e29338249a..58d51489c78 100644 --- a/.github/workflows/build_and_test.yaml +++ b/.github/workflows/build_and_test.yaml @@ -36,7 +36,7 @@ jobs: image-base-name: "dev_image_with_extras" ref: ${{ needs.env-protect-setup.outputs.ref }} clang-tidy: - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 needs: [authorize, env-protect-setup, get-dev-image] container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} @@ -64,7 +64,7 @@ jobs: code-coverage: if: github.event_name == 'push' needs: [authorize, env-protect-setup, get-dev-image] - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} steps: @@ -88,7 +88,7 @@ jobs: ./ci/collect_coverage.sh -u -b main -c "$(git rev-parse HEAD)" -r pixie-io/pixie generate-matrix: needs: [authorize, env-protect-setup, get-dev-image] - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 container: image: ${{ needs.get-dev-image.outputs.image-with-tag }} outputs: @@ -120,7 +120,7 @@ jobs: bazel_tests_* build-and-test: needs: [authorize, env-protect-setup, get-dev-image, generate-matrix] - runs-on: oracle-16cpu-64gb-x86-64 + runs-on: oracle-vm-16cpu-64gb-x86-64 permissions: contents: read actions: read @@ -160,6 +160,10 @@ jobs: run: | # Github actions container runner creates a docker network without IPv6 support. We enable it manually. sysctl -w net.ipv6.conf.lo.disable_ipv6=0 + + # Our qemu builds require unprivileged user namespaces to run. + sysctl -w kernel.unprivileged_userns_clone=1 + sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 ./scripts/bazel_ignore_codes.sh test ${{ matrix.args }} --target_pattern_file=target_files/${{ matrix.tests }} \ 2> >(tee bazel_stderr) - name: Parse junit reports diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index d061afd4936..87ec25a7bdd 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -10,6 +10,166 @@ This document outlines the process for setting up the development environment fo ## Setting up the Environment +Decide first if you'd like a full buildsystem (on a VM) or a containerized dev environment. + +### VM as buildsystem + +This utilizes `chef` to setup all dependencies and is based on `ubuntu`. +> [!Important] +> The below description defaults to using a `minikube` on this VM for the developer to have an `all-in-one` setup. The VM type must support nested virtualization for `minikube` to work. Please confirm that the nested virtualization really is turned on before you continue, not all VM-types support it. +> If you `bring-your-own-k8s`, you may disregard this. + +```yaml +advancedMachineFeatures: + enableNestedVirtualization: true +``` + +The following specifics were tested on GCP on a Ubuntu 24.04 (May 2025). Please see the latest [packer file](https://github.com/pixie-io/pixie/blob/main/tools/chef/Makefile#L56) for the current supported Ubuntu version: The initial compilation is CPU intense and `16vcpu` were a good trade-off, a balanced disk of 500 GB seems convenient and overall `n2-standard-16` works well. + +> [!Warning] +> The first `full build` takes several hours and at least 160 Gb of space +> The first `vizier build` on these parameters takes approx. 1 hr and 45 Gb of space. + + + + + +#### 1) Install chef and some dependencies + +First, install `chef` to cook your `recipies`: + +```bash +curl -L https://chefdownload-community.chef.io/install.sh | sudo bash +``` +You may find it helpful to use a terminal manager like `screen` or `tmux`, esp to detach the builds. +```bash +sudo apt install -y screen git +``` + +In order to very significantly speed up your work, you may opt for a local cache directory. This can be shared between users of the VM, if both are part of the same group. +Create a cache dir under such as e.g. /tmp/bazel +```sh +sudo groupadd bazelcache +sudo usermod -aG bazelcache $USER +sudo mkdir -p +sudo chown -R :bazelcache +sudo chmod -R 2775 +``` + + +Now, on this VM, clone pixie (or your fork of it) + +```bash +git clone https://github.com/pixie-io/pixie.git +cd pixie/tools/chef +sudo chef-solo -c solo.rb -j node_workstation.json +sudo usermod -aG libvirt $USER +``` + +Make permanent the env loading via your bashrc +```sh +echo "source /opt/px_dev/pxenv.inc " >> ~/.bashrc +``` + + +#### 2) If using cache, tell bazel about it + +Edit the `` into the .bazelrc and put it into your homedir: +``` +# Global bazelrc file, see https://docs.bazel.build/versions/master/guide.html#bazelrc. + +# Use local Cache directory if building on a VM: +# On Chef VM, create a directory and comment in the following line: + build --disk_cache=/tmp/bazel/ # Optional for multi-user cache: Make this directory owned by a group name e.g. "bazelcache" +``` + +```sh +cp .bazelrc ~/. +``` + +#### 3) Create/Use a registry you control and login + +```sh +docker login ghcr.io/ +``` + +#### 4) Prepare your kubernetes + +> [!Important] +> The below description defaults to using a `minikube` on this VM for the developer to have an `all-in-one` setup. +> If you `bring-your-own-k8s`, please prepare your preferred setup and go to Step 5 + +If you added your user to the libvirt group (`sudo usermod -aG libvirt $USER`), starting the development environment on this VM will now work (if you did this interactively: you need to refresh your group membership, e.g. by logout/login). The following command will, amongst other things, start minikube +```sh +make dev-env-start +``` + +#### 5) Deploy a vanilla pixie + +First deploy the upstream pixie (`vizier`, `kelvin` and `pem`) using the hosted cloud. Follow [these instructions](https://docs.px.dev/installing-pixie/install-schemes/cli) to install the `px` command line interface and Pixie: +```sh +px auth login +``` + +Once logged in to pixie, we found that limiting the memory is useful, thus after login, set the deploy option like so: +```sh +px deploy -p=1Gi +``` +For reference and further information https://docs.px.dev/installing-pixie/install-guides/hosted-pixie/cosmic-cloud. + +Optional on `minikube`: + +You may encounter the following WARNING, which is related to the kernel headers missing on the minikube node (this is not your VM node). This is safe to ignore if Pixie starts up properly and your cluster is queryable from Pixie's [Live UI](https://docs.px.dev/using-pixie/using-live-ui). Please see [pixie-issue2051](https://github.com/pixie-io/pixie/issues/2051) for further details. +``` +ERR: Detected missing kernel headers on your cluster's nodes. This may cause issues with the Pixie agent. Please install kernel headers on all nodes. +``` + +#### 6) Skaffold deploy your changes + +Once you make changes to the source code, or switch to another source code version, use Skaffold to deploy (after you have the vanilla setup working on minikube) + +Ensure that you have commented in the bazelcache-directory into the bazel config (see Step 2). + + +Review the compilation-mode suits your purposes: +``` +cat skaffold/skaffold_vizier.yaml +# Note: You will want to stick with a sysroot based build (-p x86_64_sysroot or -p aarch64_sysroot), +# but you may want to change the --complication_mode setting based on your needs. +# opt builds remove assert/debug checks, while dbg builds work with debuggers (gdb). +# See the bazel docs for more details https://bazel.build/docs/user-manual#compilation-mode +- name: x86_64_sysroot + patches: + - op: add + path: /build/artifacts/context=./bazel/args + value: + - --config=x86_64_sysroot + - --compilation_mode=dbg +# - --compilation_mode=opt +``` + +Optional: you can make permanent your in the skaffold config: +```sh +skaffold config set default-repo +skaffold run -f skaffold/skaffold_vizier.yaml -p x86_64_sysroot +``` + +Check that your docker login token is still valid, then + +```sh +skaffold run -f skaffold/skaffold_vizier.yaml -p x86_64_sysroot --default-repo= +``` + + + +#### 7) Golden Image + +Once all the above is working and the first cache has been built, bake an image of your VM for safekeeping. + + + + +### Containerized Devenv To set up the developer environment required to start building Pixie's components, run the `run_docker.sh` script. The following script will run the Docker container and dump you out inside the docker container console from which you can run all the necessary tools to build, test, and deploy Pixie in development mode. 1. Since this script runs a Docker container, you must have Docker installed. To install it follow these instructions [here](https://docs.docker.com/get-docker/). @@ -138,3 +298,10 @@ You will be able to run any of the CLI commands using `bazel run`. - `bazel run //src/pixie_cli:px -- deploy` will be equivalent to `px deploy` - `bazel run //src/pixie_cli:px -- run px/cluster` is the same as `px run px/cluster` + + +# Using a Custom Pixie without Development Environment +This section is on deploying pixie when it is in a state where parts are official and parts are self-developped, without setting up the Development environment + +First, get yourself a kubernetes and have helm, kubectl and your favourite tools in your favourite places. + diff --git a/PLATFORM.md b/PLATFORM.md new file mode 100644 index 00000000000..e04091a0223 --- /dev/null +++ b/PLATFORM.md @@ -0,0 +1,5 @@ +# Using a Custom Pixie without Development Environment +This section is on deploying pixie when it is in a state where parts are official and parts are self-developped, without setting up the Development environment + +First, get yourself a kubernetes and have helm, kubectl and your favourite tools in your favourite places. + diff --git a/WORKSPACE b/WORKSPACE index ff0cd59c4e6..0e30d584323 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -204,7 +204,7 @@ bind( ) # gazelle:repo bazel_gazelle -# Gazelle depes need to be loaded last to make sure they don't override our dependencies. +# Gazelle deps need to be loaded last to make sure they don't override our dependencies. # The first one wins when it comes to package declaration. load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") diff --git a/bazel/container_images.bzl b/bazel/container_images.bzl index ab81087c1d6..e31088b2471 100644 --- a/bazel/container_images.bzl +++ b/bazel/container_images.bzl @@ -14,7 +14,7 @@ # # SPDX-License-Identifier: Apache-2.0 -load("@io_bazel_rules_docker//container:container.bzl", "container_pull") +load("@io_bazel_rules_docker//container:container.bzl", "container_pull", "container_image", "container_layer") # When adding an image here, first add it to scripts/regclient/regbot_deps.yaml # Once that is in, trigger the github workflow that mirrors the required image @@ -367,3 +367,12 @@ def stirling_test_images(): repository = "golang_1_22_grpc_server_with_buildinfo", digest = "sha256:67adba5e8513670fa37bd042862e7844f26239e8d2997ed8c3b0aa527bc04cc3", ) + + # ClickHouse server image for testing. + # clickhouse/clickhouse-server:25.7-alpine + container_pull( + name = "clickhouse_server_base_image", + registry = "docker.io", + repository = "clickhouse/clickhouse-server", + digest = "sha256:60c53a520a1caad6555eb6772a8a9c91bb09774c1c7ec87e3371ea3da254eeab", + ) diff --git a/bazel/external/clickhouse_cpp.BUILD b/bazel/external/clickhouse_cpp.BUILD new file mode 100644 index 00000000000..625dfb16ee1 --- /dev/null +++ b/bazel/external/clickhouse_cpp.BUILD @@ -0,0 +1,64 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@rules_foreign_cc//foreign_cc:defs.bzl", "cmake") + +licenses(["notice"]) + +exports_files(["LICENSE"]) + +filegroup( + name = "all", + srcs = glob(["**"]), +) + +cmake( + name = "clickhouse_cpp", + build_args = [ + "--", # <- Pass remaining options to the native tool. + "-j`nproc`", + "-l`nproc`", + ], + cache_entries = { + "BUILD_BENCHMARK": "OFF", + "BUILD_TESTS": "OFF", + "BUILD_SHARED_LIBS": "OFF", + "CMAKE_BUILD_TYPE": "Release", + "WITH_OPENSSL": "OFF", # Disable OpenSSL for now + "WITH_SYSTEM_ABSEIL": "OFF", # Use bundled abseil + "WITH_SYSTEM_LZ4": "OFF", # Use bundled for now + "WITH_SYSTEM_CITYHASH": "OFF", # Use bundled for now + "WITH_SYSTEM_ZSTD": "OFF", # Use bundled for now + "CMAKE_POSITION_INDEPENDENT_CODE": "ON", + }, + lib_source = ":all", + out_static_libs = [ + "libclickhouse-cpp-lib.a", + "liblz4.a", + "libcityhash.a", + "libzstdstatic.a", + "libabsl_int128.a", + ], + targets = [ + "clickhouse-cpp-lib", + "lz4", + "cityhash", + "zstdstatic", + "absl_int128", + ], + visibility = ["//visibility:public"], + working_directory = "", +) \ No newline at end of file diff --git a/bazel/repositories.bzl b/bazel/repositories.bzl index 55d23e61323..1ce7f3db77e 100644 --- a/bazel/repositories.bzl +++ b/bazel/repositories.bzl @@ -147,6 +147,7 @@ def _cc_deps(): _bazel_repo("com_github_ariafallah_csv_parser", build_file = "//bazel/external:csv_parser.BUILD") _bazel_repo("com_github_arun11299_cpp_jwt", build_file = "//bazel/external:cpp_jwt.BUILD") _bazel_repo("com_github_cameron314_concurrentqueue", build_file = "//bazel/external:concurrentqueue.BUILD") + _bazel_repo("com_github_clickhouse_clickhouse_cpp", build_file = "//bazel/external:clickhouse_cpp.BUILD") _bazel_repo("com_github_cyan4973_xxhash", build_file = "//bazel/external:xxhash.BUILD") _bazel_repo("com_github_nlohmann_json", build_file = "//bazel/external:nlohmann_json.BUILD") _bazel_repo("com_github_packetzero_dnsparser", build_file = "//bazel/external:dnsparser.BUILD") diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 3252648e6d7..3e93af8621e 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -71,6 +71,11 @@ REPOSITORY_LOCATIONS = dict( strip_prefix = "concurrentqueue-1.0.3", urls = ["https://github.com/cameron314/concurrentqueue/archive/refs/tags/v1.0.3.tar.gz"], ), + com_github_clickhouse_clickhouse_cpp = dict( + sha256 = "1029a1bb0da8a72db1662a0418267742e66c82bb3e6b0ed116623a2fa8c65a58", + strip_prefix = "clickhouse-cpp-22dc9441cd807156511c6dcf97b1b878bd663d77", + urls = ["https://github.com/ClickHouse/clickhouse-cpp/archive/22dc9441cd807156511c6dcf97b1b878bd663d77.tar.gz"], + ), com_github_cyan4973_xxhash = dict( sha256 = "952ebbf5b11fbf59ae5d760a562d1e9112278f244340ad7714e8556cbe54f7f7", strip_prefix = "xxHash-0.7.3", diff --git a/ci/github/bazelrc b/ci/github/bazelrc index f4b0cdb5ac0..8de37643b0c 100644 --- a/ci/github/bazelrc +++ b/ci/github/bazelrc @@ -5,9 +5,6 @@ common --color=yes # a given run. common --keep_going -# Always use remote exec -build --config=remote - build --build_metadata=HOST=github-actions build --build_metadata=USER=github-actions build --build_metadata=REPO_URL=https://github.com/pixie-io/pixie diff --git a/k8s/cloud/dev/plugin_db_updater_job.yaml b/k8s/cloud/dev/plugin_db_updater_job.yaml index d92d7d544f5..769e5f6bd55 100644 --- a/k8s/cloud/dev/plugin_db_updater_job.yaml +++ b/k8s/cloud/dev/plugin_db_updater_job.yaml @@ -62,7 +62,7 @@ spec: name: pl-service-config key: PL_PLUGIN_SERVICE - name: PL_PLUGIN_REPO - value: "pixie-io/pixie-plugin" + value: "k8sstormcenter/pixie-plugin" - name: PL_GH_API_KEY valueFrom: secretKeyRef: diff --git a/k8s/cloud/overlays/plugin_job/plugin_job.yaml b/k8s/cloud/overlays/plugin_job/plugin_job.yaml index 228efbda87d..ab51bd9db20 100644 --- a/k8s/cloud/overlays/plugin_job/plugin_job.yaml +++ b/k8s/cloud/overlays/plugin_job/plugin_job.yaml @@ -55,7 +55,7 @@ spec: name: pl-service-config key: PL_PLUGIN_SERVICE - name: PL_PLUGIN_REPO - value: "pixie-io/pixie-plugin" + value: "k8sstormcenter/pixie-plugin" # The alpine based image contains a shell and is needed for this command to work. # yamllint disable-line rule:line-length - image: gcr.io/cloud-sql-connectors/cloud-sql-proxy:2.11.3-alpine@sha256:4885fd3e6362ba22abff1804a7f5e75cec5fafbeb4e41be8b0059ecad94a16f1 diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml new file mode 100644 index 00000000000..c407804b6c1 --- /dev/null +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -0,0 +1,73 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: adaptive-export +spec: + replicas: 1 + selector: + matchLabels: + name: adaptive-export + template: + metadata: + labels: + name: adaptive-export + plane: control + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: kubernetes.io/os + operator: Exists + - key: kubernetes.io/os + operator: In + values: + - linux + - matchExpressions: + - key: beta.kubernetes.io/os + operator: Exists + - key: beta.kubernetes.io/os + operator: In + values: + - linux + serviceAccountName: pl-adaptive-export-service-account + containers: + - name: adaptive-export + image: vizier-adaptive_export_image:latest + env: + - name: PL_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: PIXIE_API_KEY + valueFrom: + secretKeyRef: + name: pl-adaptive-export-secrets + key: pixie-api-key + - name: CLICKHOUSE_DSN + valueFrom: + secretKeyRef: + name: pl-adaptive-export-secrets + key: clickhouse-dsn + - name: VERBOSE + value: "true" + - name: DETECTION_INTERVAL_SEC + value: "10" + - name: DETECTION_LOOKBACK_SEC + value: "30" + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + seccompProfile: + type: RuntimeDefault + securityContext: + runAsUser: 10100 + runAsGroup: 10100 + fsGroup: 10100 + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault diff --git a/k8s/vizier/bootstrap/adaptive_export_role.yaml b/k8s/vizier/bootstrap/adaptive_export_role.yaml new file mode 100644 index 00000000000..33887150f37 --- /dev/null +++ b/k8s/vizier/bootstrap/adaptive_export_role.yaml @@ -0,0 +1,64 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: pl-adaptive-export-service-account +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: pl-adaptive-export-role +rules: +- apiGroups: + - "" + resources: + - secrets + verbs: + - get + - list +- apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: pl-adaptive-export-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: pl-adaptive-export-role +subjects: +- kind: ServiceAccount + name: pl-adaptive-export-service-account + namespace: pl +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: pl-adaptive-export-cluster-role +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: pl-adaptive-export-cluster-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: pl-adaptive-export-cluster-role +subjects: +- kind: ServiceAccount + name: pl-adaptive-export-service-account + namespace: pl diff --git a/k8s/vizier/bootstrap/adaptive_export_secrets.yaml b/k8s/vizier/bootstrap/adaptive_export_secrets.yaml new file mode 100644 index 00000000000..19be138743b --- /dev/null +++ b/k8s/vizier/bootstrap/adaptive_export_secrets.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: v1 +kind: Secret +metadata: + name: pl-adaptive-export-secrets +type: Opaque +stringData: + # Replace with your actual Pixie API key from https://work.withpixie.ai + pixie-api-key: "PIXIE_API_KEY_PLACEHOLDER" + # Replace with your ClickHouse DSN: clickhouse://user:password@host:port/database + clickhouse-dsn: "otelcollector:otelcollectorpass@hyperdx-hdx-oss-v2-clickhouse.click.svc.cluster.local:9000/default" diff --git a/k8s/vizier/bootstrap/kustomization.yaml b/k8s/vizier/bootstrap/kustomization.yaml index 714f5676426..e373c6bbfe3 100644 --- a/k8s/vizier/bootstrap/kustomization.yaml +++ b/k8s/vizier/bootstrap/kustomization.yaml @@ -15,3 +15,6 @@ resources: - cert_provisioner_role.yaml - cert_provisioner_job.yaml - vizier_crd_role.yaml +- adaptive_export_role.yaml +- adaptive_export_secrets.yaml +- adaptive_export_deployment.yaml diff --git a/skaffold/skaffold_vizier.yaml b/skaffold/skaffold_vizier.yaml index 2b6218a8c7d..33389dffb2e 100644 --- a/skaffold/skaffold_vizier.yaml +++ b/skaffold/skaffold_vizier.yaml @@ -8,37 +8,50 @@ build: bazel: target: //src/vizier/services/agent/pem:pem_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt - image: vizier-kelvin_image context: . bazel: target: //src/vizier/services/agent/kelvin:kelvin_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt - image: vizier-metadata_server_image context: . bazel: target: //src/vizier/services/metadata:metadata_server_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt - image: vizier-query_broker_server_image context: . bazel: target: //src/vizier/services/query_broker:query_broker_server_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt - image: vizier-cloud_connector_server_image context: . bazel: target: //src/vizier/services/cloud_connector:cloud_connector_server_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt - image: vizier-cert_provisioner_image context: . bazel: target: //src/utils/cert_provisioner:cert_provisioner_image.tar args: - - --compilation_mode=dbg + - --config=x86_64_sysroot + - --compilation_mode=opt + - image: vizier-adaptive_export_image + context: . + bazel: + target: //src/vizier/services/adaptive_export:adaptive_export_image.tar + args: + - --config=x86_64_sysroot + - --compilation_mode=opt tagPolicy: dateTime: {} local: @@ -138,9 +151,15 @@ profiles: path: /manifests/kustomize/paths value: - k8s/vizier/persistent_metadata/aarch64 +# Note: You will want to stick with a sysroot based build (-p x86_64_sysroot or -p aarch64_sysroot), +# but you may want to change the --complication_mode setting based on your needs. +# opt builds remove assert/debug checks, while dbg builds work with debuggers (gdb). +# See the bazel docs for more details https://bazel.build/docs/user-manual#compilation-mode - name: x86_64_sysroot patches: - op: add path: /build/artifacts/context=./bazel/args value: - --config=x86_64_sysroot + - --compilation_mode=dbg +# - --compilation_mode=opt diff --git a/src/api/go/pxapi/vizier.go b/src/api/go/pxapi/vizier.go index ef5b0bcdfcb..88c5404a583 100644 --- a/src/api/go/pxapi/vizier.go +++ b/src/api/go/pxapi/vizier.go @@ -20,6 +20,7 @@ package pxapi import ( "context" + "strings" "px.dev/pixie/src/api/go/pxapi/errdefs" "px.dev/pixie/src/api/proto/vizierpb" @@ -40,6 +41,7 @@ func (v *VizierClient) ExecuteScript(ctx context.Context, pxl string, mux TableM ClusterID: v.vizierID, QueryStr: pxl, EncryptionOptions: v.encOpts, + Mutation: strings.Contains(pxl, "import pxlog") || strings.Contains(pxl, "import pxtrace"), } origCtx := ctx ctx, cancel := context.WithCancel(ctx) diff --git a/src/carnot/BUILD.bazel b/src/carnot/BUILD.bazel index 664599ad9c0..b22fd1a949a 100644 --- a/src/carnot/BUILD.bazel +++ b/src/carnot/BUILD.bazel @@ -69,6 +69,7 @@ pl_cc_test( ":cc_library", "//src/carnot/exec:test_utils", "//src/carnot/udf_exporter:cc_library", + "//src/common/testing/event:cc_library", ], ) @@ -79,6 +80,7 @@ pl_cc_test( ":cc_library", "//src/carnot/exec:test_utils", "//src/carnot/udf_exporter:cc_library", + "//src/common/testing/event:cc_library", ], ) @@ -98,7 +100,23 @@ pl_cc_binary( pl_cc_binary( name = "carnot_executable", srcs = ["carnot_executable.cc"], + data = [ + "//src/stirling/source_connectors/socket_tracer/testing/container_images:clickhouse.tar", + ], + stamp = -1, + tags = [ + "requires_docker", + ], deps = [ ":cc_library", + "//src/common/testing:cc_library", + "//src/common/testing/test_utils:cc_library", + "//src/shared/version:cc_library", + "//src/shared/version:version_linkstamp", + "//src/stirling/source_connectors/socket_tracer:cc_library", + "//src/vizier/funcs:cc_library", + "//src/vizier/funcs/context:cc_library", + "//src/vizier/services/metadata/local:cc_library", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", ], ) diff --git a/src/carnot/carnot.cc b/src/carnot/carnot.cc index a466bb5194d..ff55ff0ec15 100644 --- a/src/carnot/carnot.cc +++ b/src/carnot/carnot.cc @@ -181,6 +181,8 @@ Status CarnotImpl::RegisterUDFsInPlanFragment(exec::ExecState* exec_state, plan: .OnUDTFSource(no_op) .OnEmptySource(no_op) .OnOTelSink(no_op) + .OnClickHouseSource(no_op) + .OnClickHouseExportSink(no_op) .Walk(pf); } @@ -378,9 +380,9 @@ Status CarnotImpl::ExecutePlan(const planpb::Plan& logical_plan, const sole::uui int64_t total_time_ns = stats->TotalExecTime(); int64_t self_time_ns = stats->SelfExecTime(); LOG(INFO) << absl::Substitute( - "self_time:$1\ttotal_time: $2\tbytes_output: $3\trows_output: $4\tnode_id:$0", + "self_time:$1\ttotal_time: $2\tbytes_input: $3\tbytes_output: $4\trows_input: $5\trows_output: $6\tnode_id:$0", node_name, PrettyDuration(self_time_ns), PrettyDuration(total_time_ns), - stats->bytes_output, stats->rows_output); + stats->bytes_input, stats->bytes_output, stats->rows_input, stats->rows_output); queryresultspb::OperatorExecutionStats* stats_pb = agent_operator_exec_stats.add_operator_execution_stats(); diff --git a/src/carnot/carnot_executable.cc b/src/carnot/carnot_executable.cc index 52a3d46cd7f..4f19570e782 100644 --- a/src/carnot/carnot_executable.cc +++ b/src/carnot/carnot_executable.cc @@ -16,9 +16,14 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include + +#include #include #include +#include #include +#include #include #include @@ -28,9 +33,40 @@ #include "src/carnot/exec/local_grpc_result_server.h" #include "src/carnot/funcs/funcs.h" #include "src/common/base/base.h" +#include "src/common/testing/test_environment.h" +#include "src/common/testing/test_utils/container_runner.h" #include "src/shared/types/column_wrapper.h" #include "src/shared/types/type_utils.h" #include "src/table_store/table_store.h" +#include "src/vizier/funcs/context/vizier_context.h" +#include "src/vizier/funcs/funcs.h" +#include "src/vizier/services/metadata/local/local_metadata_service.h" +#include "src/stirling/source_connectors/socket_tracer/http_table.h" + +// Example clickhouse test usage: +// The records inserted into clickhouse exist between -10m and -5m +// bazel run -c dbg src/carnot:carnot_executable -- --vmodule=clickhouse_source_node=1 --use_clickhouse=true --query="import px;df = px.DataFrame('http_events', clickhouse_dsn='default:test_password@localhost:9000/default', start_time='-10m', end_time='-9m'); px.display(df)" --output_file=$(pwd)/output.csv +// +// +// Test that verifies bug with Map operators isn't introduced +// bazel run -c dbg src/carnot:carnot_executable -- -v=1 --vmodule=clickhouse_source_node=1 --use_clickhouse=true --query="import px;df = px.DataFrame('http_events', clickhouse_dsn='default:test_password@localhost:9000/default', start_time='-10m', end_time='-9m'); df.time_ = df.event_time; df = df[['time_', 'req_path']]; px.display(df)" --output_file=$(pwd)/output.csv +// +// +// Testing existing ClickHouse table (kubescape_stix) table population and query: +// docker run -p 9000:9000 --network=host --env=CLICKHOUSE_PASSWORD=test_password clickhouse/clickhouse-server:25.7-alpine +// CREATE TABLE IF NOT EXISTS default.kubescape_stix ( +// timestamp String, +// pod_name String, +// namespace String, +// data String, +// hostname String, +// event_time DateTime64(3) +//) ENGINE = MergeTree() +//PARTITION BY toYYYYMM(event_time) +//ORDER BY (hostname, event_time); + +// bazel run -c dbg src/carnot:carnot_executable -- --vmodule=clickhouse_source_node=1 --use_clickhouse=true --start_clickhouse=false --query="import px;df = px.DataFrame('kubescape_stix', clickhouse_dsn='default:test_password@localhost:9000/default', start_time='-10m'); px.display(df)" --output_file=$(pwd)/output.csv + DEFINE_string(input_file, gflags::StringFromEnv("INPUT_FILE", ""), "The csv containing data to run the query on."); @@ -46,6 +82,12 @@ DEFINE_string(table_name, gflags::StringFromEnv("TABLE_NAME", "csv_table"), DEFINE_int64(rowbatch_size, gflags::Int64FromEnv("ROWBATCH_SIZE", 100), "The size of the rowbatches."); +DEFINE_bool(use_clickhouse, gflags::BoolFromEnv("USE_CLICKHOUSE", false), + "Whether to populate a ClickHouse database."); + +DEFINE_bool(start_clickhouse, gflags::BoolFromEnv("START_CLICKHOUSE", true), + "Whether to start a ClickHouse container with test data."); + using px::types::DataType; namespace { @@ -225,6 +267,264 @@ void TableToCsv(const std::string& filename, output_csv.close(); } +// ClickHouse container configuration +constexpr char kClickHouseImage[] = + "src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse.tar"; +constexpr char kClickHouseReadyMessage[] = "Ready for connections"; +constexpr int kClickHousePort = 9000; + +/** + * Sets up a ClickHouse client connection with retries. + */ +std::unique_ptr SetupClickHouseClient() { + clickhouse::ClientOptions client_options; + client_options.SetHost("localhost"); + client_options.SetPort(kClickHousePort); + client_options.SetUser("default"); + client_options.SetPassword("test_password"); + client_options.SetDefaultDatabase("default"); + + const int kMaxRetries = 10; + for (int i = 0; i < kMaxRetries; ++i) { + LOG(INFO) << "Attempting to connect to ClickHouse (attempt " << (i + 1) << "/" << kMaxRetries + << ")..."; + try { + auto client = std::make_unique(client_options); + client->Execute("SELECT 1"); + LOG(INFO) << "Successfully connected to ClickHouse"; + return client; + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to connect: " << e.what(); + if (i < kMaxRetries - 1) { + std::this_thread::sleep_for(std::chrono::seconds(2)); + } else { + LOG(FATAL) << "Failed to connect to ClickHouse after " << kMaxRetries << " attempts"; + } + } + } + return nullptr; +} + +/** + * Creates the http_events table in ClickHouse with proper schema and sample data. + */ +void PopulateHttpEventsTable(clickhouse::Client* client) { + try { + // Get current hostname for the data + char current_hostname[256]; + gethostname(current_hostname, sizeof(current_hostname)); + std::string hostname_str(current_hostname); + + // Insert sample data matching the stirling HTTP table schema (upid as String with high:low format) + auto time_col = std::make_shared(9); + auto upid_col = std::make_shared(); + auto remote_addr_col = std::make_shared(); + auto remote_port_col = std::make_shared(); + auto local_addr_col = std::make_shared(); + auto local_port_col = std::make_shared(); + auto trace_role_col = std::make_shared(); + auto encrypted_col = std::make_shared(); // Boolean + auto major_version_col = std::make_shared(); + auto minor_version_col = std::make_shared(); + auto content_type_col = std::make_shared(); + auto req_headers_col = std::make_shared(); + auto req_method_col = std::make_shared(); + auto req_path_col = std::make_shared(); + auto req_body_col = std::make_shared(); + auto req_body_size_col = std::make_shared(); + auto resp_headers_col = std::make_shared(); + auto resp_status_col = std::make_shared(); + auto resp_message_col = std::make_shared(); + auto resp_body_col = std::make_shared(); + auto resp_body_size_col = std::make_shared(); + auto latency_col = std::make_shared(); +#ifndef NDEBUG + auto px_info_col = std::make_shared(); +#endif + auto hostname_col = std::make_shared(); + auto event_time_col = std::make_shared(3); + + // Add sample rows + std::time_t now = std::time(nullptr); + LOG(INFO) << "Current time: " << now; + + // Add 10 records (5 with current hostname, 5 with different hostnames) + for (int i = 0; i < 10; ++i) { + time_col->Append((now - 600 + i * 60) * 1000000000LL); // Convert to nanoseconds + + // Generate upid as UINT128 in high:low string format + uint64_t upid_high = 1000 + i; + uint64_t upid_low = 2000 + i; + upid_col->Append(absl::StrFormat("%d:%d", upid_high, upid_low)); + + remote_addr_col->Append(absl::StrFormat("192.168.1.%d", 100 + i)); + remote_port_col->Append(50000 + i); + local_addr_col->Append("127.0.0.1"); + local_port_col->Append(8080); + + // trace_role: 1 = server, 2 = client (alternate) + trace_role_col->Append(i % 2 == 0 ? 1 : 2); + + // encrypted: false for most, true for some + encrypted_col->Append(i % 3 == 0 ? 1 : 0); + + major_version_col->Append(1); + minor_version_col->Append(1); + content_type_col->Append(i % 2 == 0 ? 1 : 0); // 1 = JSON, 0 = unknown + + req_headers_col->Append("Content-Type: application/json"); + req_method_col->Append(i % 2 == 0 ? "GET" : "POST"); + req_path_col->Append(absl::StrFormat("/api/v1/resource/%d", i)); + + std::string req_body = i % 2 == 0 ? "" : "{\"data\": \"test\"}"; + req_body_col->Append(req_body); + req_body_size_col->Append(req_body.size()); + + resp_headers_col->Append("Content-Type: application/json"); + resp_status_col->Append(200); + resp_message_col->Append("OK"); + + std::string resp_body = "{\"result\": \"success\"}"; + resp_body_col->Append(resp_body); + resp_body_size_col->Append(resp_body.size()); + + latency_col->Append(1000000 + i * 100000); +#ifndef NDEBUG + px_info_col->Append(""); +#endif + + // First 5 use current hostname, next 5 use different hostnames + if (i < 5) { + hostname_col->Append(hostname_str); + } else { + hostname_col->Append(absl::StrFormat("other-host-%d", i % 3)); + } + + event_time_col->Append((now - 600 + i * 60) * 1000LL); // Convert to milliseconds + } + + clickhouse::Block block; + block.AppendColumn("time_", time_col); + block.AppendColumn("upid", upid_col); + block.AppendColumn("remote_addr", remote_addr_col); + block.AppendColumn("remote_port", remote_port_col); + block.AppendColumn("local_addr", local_addr_col); + block.AppendColumn("local_port", local_port_col); + block.AppendColumn("trace_role", trace_role_col); + block.AppendColumn("encrypted", encrypted_col); + block.AppendColumn("major_version", major_version_col); + block.AppendColumn("minor_version", minor_version_col); + block.AppendColumn("content_type", content_type_col); + block.AppendColumn("req_headers", req_headers_col); + block.AppendColumn("req_method", req_method_col); + block.AppendColumn("req_path", req_path_col); + block.AppendColumn("req_body", req_body_col); + block.AppendColumn("req_body_size", req_body_size_col); + block.AppendColumn("resp_headers", resp_headers_col); + block.AppendColumn("resp_status", resp_status_col); + block.AppendColumn("resp_message", resp_message_col); + block.AppendColumn("resp_body", resp_body_col); + block.AppendColumn("resp_body_size", resp_body_size_col); + block.AppendColumn("latency", latency_col); + block.AppendColumn("hostname", hostname_col); + block.AppendColumn("event_time", event_time_col); + + client->Insert("http_events", block); + LOG(INFO) << "http_events table populated successfully with 10 records"; + } catch (const std::exception& e) { + LOG(FATAL) << "Failed to populate http_events table: " << e.what(); + } +} + +/** + * Checks if a table exists in ClickHouse. + */ +bool TableExists(clickhouse::Client* client, const std::string& table_name) { + try { + std::string query = absl::Substitute("EXISTS TABLE $0", table_name); + bool exists = false; + client->Select(query, [&exists](const clickhouse::Block& block) { + if (block.GetRowCount() > 0) { + auto result_col = block[0]->As(); + exists = result_col->At(0) == 1; + } + }); + return exists; + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to check if table " << table_name << " exists: " << e.what(); + return false; + } +} + +/** + * Populates the kubescape_stix table with sample STIX data if it exists. + */ +void PopulateKubescapeStixTable(clickhouse::Client* client) { + try { + // Check if table exists + if (!TableExists(client, "kubescape_stix")) { + LOG(INFO) << "kubescape_stix table does not exist, skipping population"; + return; + } + + LOG(INFO) << "Populating kubescape_stix table with sample data..."; + + // Get current hostname + char current_hostname[256]; + gethostname(current_hostname, sizeof(current_hostname)); + std::string hostname_str(current_hostname); + + // Create columns for the kubescape_stix table + auto timestamp_col = std::make_shared(); + auto pod_name_col = std::make_shared(); + auto namespace_col = std::make_shared(); + auto data_col = std::make_shared(); + auto hostname_col = std::make_shared(); + auto event_time_col = std::make_shared(3); + + // Add sample STIX data + std::time_t now = std::time(nullptr); + + // Add 5 sample records with different pods and namespaces + std::vector pod_names = {"web-pod-1", "api-pod-2", "db-pod-3", "cache-pod-4", "worker-pod-5"}; + std::vector namespaces = {"production", "staging", "development", "production", "staging"}; + + for (int i = 0; i < 5; ++i) { + // Timestamp as ISO 8601 string + std::time_t record_time = now - (300 - i * 60); // 5 minutes ago to 1 minute ago + char time_buf[30]; + std::strftime(time_buf, sizeof(time_buf), "%Y-%m-%dT%H:%M:%SZ", std::gmtime(&record_time)); + timestamp_col->Append(std::string(time_buf)); + + pod_name_col->Append(pod_names[i]); + namespace_col->Append(namespaces[i]); + + // Add unique STIX data for each record + std::string stix_data = absl::Substitute( + R"({"type":"bundle","id":"bundle--$0","objects":[{"type":"vulnerability","id":"vuln--$0","severity":"$1"}]})", + i, (i % 3 == 0 ? "high" : "medium")); + data_col->Append(stix_data); + + hostname_col->Append(hostname_str); + event_time_col->Append(record_time * 1000LL); // Convert to milliseconds + } + + // Create block and insert + clickhouse::Block block; + block.AppendColumn("timestamp", timestamp_col); + block.AppendColumn("pod_name", pod_name_col); + block.AppendColumn("namespace", namespace_col); + block.AppendColumn("data", data_col); + block.AppendColumn("hostname", hostname_col); + block.AppendColumn("event_time", event_time_col); + + client->Insert("kubescape_stix", block); + LOG(INFO) << "kubescape_stix table populated successfully with 5 records"; + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to populate kubescape_stix table: " << e.what(); + } +} + } // namespace int main(int argc, char* argv[]) { @@ -235,14 +535,67 @@ int main(int argc, char* argv[]) { auto query = FLAGS_query; auto rb_size = FLAGS_rowbatch_size; auto table_name = FLAGS_table_name; + auto use_clickhouse = FLAGS_use_clickhouse; + + // ClickHouse container and client (if enabled) + std::unique_ptr clickhouse_server; + std::unique_ptr clickhouse_client; + + std::shared_ptr table; + + if (use_clickhouse) { + + if (FLAGS_start_clickhouse) { + LOG(INFO) << "Starting ClickHouse container..."; + clickhouse_server = + std::make_unique(px::testing::BazelRunfilePath(kClickHouseImage), + "clickhouse_carnot", kClickHouseReadyMessage); + + std::vector options = { + absl::Substitute("--publish=$0:$0", kClickHousePort), + "--env=CLICKHOUSE_PASSWORD=test_password", + "--network=host", + }; + + auto status = clickhouse_server->Run(std::chrono::seconds{60}, options, {}, true, + std::chrono::seconds{300}); + if (!status.ok()) { + LOG(FATAL) << "Failed to start ClickHouse container: " << status.msg(); + } + } + + // Give ClickHouse time to initialize + LOG(INFO) << "Waiting for ClickHouse to initialize..."; + std::this_thread::sleep_for(std::chrono::seconds(5)); - auto table = GetTableFromCsv(filename, rb_size); + // Setup ClickHouse client and create test table + clickhouse_client = SetupClickHouseClient(); + LOG(INFO) << "ClickHouse ready with http_events table"; + } else { + // Only load CSV if not using ClickHouse + table = GetTableFromCsv(filename, rb_size); + } // Execute query. auto table_store = std::make_shared(); auto result_server = px::carnot::exec::LocalGRPCResultSinkServer(); + + // Create metadata service stub for table schemas + auto metadata_grpc_server = std::make_unique(table_store.get()); + + // Create vizier func factory context with metadata stub + px::vizier::funcs::VizierFuncFactoryContext func_context( + nullptr, // agent_manager + metadata_grpc_server->StubGenerator(), // mds_stub + nullptr, // mdtp_stub + nullptr, // cronscript_stub + table_store, + [](grpc::ClientContext*) {} // add_grpc_auth + ); + auto func_registry = std::make_unique("default_registry"); - px::carnot::funcs::RegisterFuncsOrDie(func_registry.get()); + px::vizier::funcs::RegisterFuncsOrDie(func_context, func_registry.get()); + auto clients_config = std::make_unique(px::carnot::Carnot::ClientsConfig{ [&result_server](const std::string& address, const std::string&) { @@ -257,12 +610,71 @@ int main(int argc, char* argv[]) { auto carnot = px::carnot::Carnot::Create(sole::uuid4(), std::move(func_registry), table_store, std::move(clients_config), std::move(server_config)) .ConsumeValueOrDie(); - table_store->AddTable(table_name, table); + + if (use_clickhouse) { + // Create http_events table schema in table_store using the actual stirling HTTP table definition + std::vector types; + std::vector names; + + // Convert stirling DataTableSchema to table_store Relation + for (const auto& element : px::stirling::kHTTPTable.elements()) { + std::string col_name(element.name()); + types.push_back(element.type()); + names.push_back(col_name); + } + + px::table_store::schema::Relation rel(types, names); + auto http_events_table = px::table_store::Table::Create("http_events", rel); + // Need to provide a table_id for GetTableIDs() to work + uint64_t http_events_table_id = 1; + table_store->AddTable(http_events_table, "http_events", http_events_table_id); + + // Log the schema for debugging + LOG(INFO) << "http_events table schema has " << names.size() << " columns:"; + for (size_t i = 0; i < names.size(); ++i) { + LOG(INFO) << " Column[" << i << "]: " << names[i] << " (type=" << static_cast(types[i]) << ")"; + } + + auto schema_query = "import px; px.display(px.CreateClickHouseSchemas())"; + auto schema_query_status = carnot->ExecuteQuery(schema_query, sole::uuid4(), px::CurrentTimeNS()); + if (!schema_query_status.ok()) { + LOG(FATAL) << absl::Substitute("Schema query failed to execute: $0", + schema_query_status.msg()); + } + PopulateHttpEventsTable(clickhouse_client.get()); + PopulateKubescapeStixTable(clickhouse_client.get()); + } else if (table != nullptr) { + // Add CSV table to table_store + table_store->AddTable(table_name, table); + } + auto exec_status = carnot->ExecuteQuery(query, sole::uuid4(), px::CurrentTimeNS()); if (!exec_status.ok()) { LOG(FATAL) << absl::Substitute("Query failed to execute: $0", exec_status.msg()); } + // Get and log execution stats + auto exec_stats_or = result_server.exec_stats(); + if (exec_stats_or.ok()) { + auto exec_stats = exec_stats_or.ConsumeValueOrDie(); + if (exec_stats.has_execution_stats()) { + auto stats = exec_stats.execution_stats(); + LOG(INFO) << "Query Execution Stats:"; + LOG(INFO) << " Bytes processed: " << stats.bytes_processed(); + LOG(INFO) << " Records processed: " << stats.records_processed(); + if (stats.has_timing()) { + LOG(INFO) << " Execution time: " << stats.timing().execution_time_ns() << " ns"; + } + } + + for (const auto& agent_stats : exec_stats.agent_execution_stats()) { + LOG(INFO) << "Agent Execution Stats:"; + LOG(INFO) << " Execution time: " << agent_stats.execution_time_ns() << " ns"; + LOG(INFO) << " Bytes processed: " << agent_stats.bytes_processed(); + LOG(INFO) << " Records processed: " << agent_stats.records_processed(); + } + } + auto output_names = result_server.output_tables(); if (!output_names.size()) { LOG(FATAL) << "Query produced no output tables."; diff --git a/src/carnot/exec/BUILD.bazel b/src/carnot/exec/BUILD.bazel index 228b352501c..625a6964d59 100644 --- a/src/carnot/exec/BUILD.bazel +++ b/src/carnot/exec/BUILD.bazel @@ -33,6 +33,7 @@ pl_cc_library( ], ), hdrs = [ + "clickhouse_source_node.h", "exec_node.h", "exec_state.h", ], @@ -46,6 +47,7 @@ pl_cc_library( "//src/shared/types:cc_library", "//src/table_store/table:cc_library", "@com_github_apache_arrow//:arrow", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", "@com_github_grpc_grpc//:grpc++", "@com_github_opentelemetry_proto//:logs_service_grpc_cc", "@com_github_opentelemetry_proto//:metrics_service_grpc_cc", @@ -300,3 +302,46 @@ pl_cc_test( "@com_github_grpc_grpc//:grpc++_test", ], ) + +pl_cc_test( + name = "clickhouse_source_node_test", + timeout = "long", + srcs = ["clickhouse_source_node_test.cc"], + data = [ + "//src/stirling/source_connectors/socket_tracer/testing/container_images:clickhouse.tar", + ], + tags = [ + "exclusive", + "requires_docker", + ], + deps = [ + ":cc_library", + ":exec_node_test_helpers", + ":test_utils", + "//src/carnot/planpb:plan_testutils", + "//src/common/testing/test_utils:cc_library", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", + ], +) + +pl_cc_test( + name = "clickhouse_export_sink_node_test", + timeout = "long", + srcs = ["clickhouse_export_sink_node_test.cc"], + data = [ + "//src/stirling/source_connectors/socket_tracer/testing/container_images:clickhouse.tar", + ], + tags = [ + "exclusive", + "requires_docker", + ], + deps = [ + ":cc_library", + ":exec_node_test_helpers", + ":test_utils", + "//src/carnot/plan:cc_library", + "//src/carnot/planpb:plan_pl_cc_proto", + "//src/common/testing/test_utils:cc_library", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", + ], +) diff --git a/src/carnot/exec/clickhouse_export_sink_node.cc b/src/carnot/exec/clickhouse_export_sink_node.cc new file mode 100644 index 00000000000..6a11a42d37a --- /dev/null +++ b/src/carnot/exec/clickhouse_export_sink_node.cc @@ -0,0 +1,179 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/exec/clickhouse_export_sink_node.h" + +#include +#include +#include + +#include +#include +#include +#include "glog/logging.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/common/base/macros.h" +#include "src/shared/types/typespb/types.pb.h" +#include "src/table_store/table_store.h" + +namespace px { +namespace carnot { +namespace exec { + +using table_store::schema::RowBatch; +using table_store::schema::RowDescriptor; + +std::string ClickHouseExportSinkNode::DebugStringImpl() { + return absl::Substitute("Exec::ClickHouseExportSinkNode: $0", plan_node_->DebugString()); +} + +Status ClickHouseExportSinkNode::InitImpl(const plan::Operator& plan_node) { + CHECK(plan_node.op_type() == planpb::OperatorType::CLICKHOUSE_EXPORT_SINK_OPERATOR); + if (input_descriptors_.size() != 1) { + return error::InvalidArgument( + "ClickHouse Export operator expects a single input relation, got $0", + input_descriptors_.size()); + } + + input_descriptor_ = std::make_unique(input_descriptors_[0]); + const auto* sink_plan_node = static_cast(&plan_node); + plan_node_ = std::make_unique(*sink_plan_node); + return Status::OK(); +} + +Status ClickHouseExportSinkNode::PrepareImpl(ExecState*) { return Status::OK(); } + +Status ClickHouseExportSinkNode::OpenImpl(ExecState* /*exec_state*/) { + // Connect to ClickHouse using config from plan node + const auto& config = plan_node_->clickhouse_config(); + + clickhouse::ClientOptions options; + options.SetHost(config.host()); + options.SetPort(config.port()); + options.SetUser(config.username()); + options.SetPassword(config.password()); + options.SetDefaultDatabase(config.database()); + + clickhouse_client_ = std::make_unique(options); + + return Status::OK(); +} + +Status ClickHouseExportSinkNode::CloseImpl(ExecState* exec_state) { + if (sent_eos_) { + return Status::OK(); + } + + LOG(INFO) << absl::Substitute( + "Closing ClickHouseExportSinkNode $0 in query $1 before receiving EOS", plan_node_->id(), + exec_state->query_id().str()); + + return Status::OK(); +} + +Status ClickHouseExportSinkNode::ConsumeNextImpl(ExecState* /*exec_state*/, const RowBatch& rb, + size_t /*parent_index*/) { + // Skip insertion if the batch is empty + if (rb.num_rows() == 0) { + if (rb.eos()) { + sent_eos_ = true; + } + return Status::OK(); + } + + // Build an INSERT query with the data from the row batch + clickhouse::Block block; + + // Create columns based on the column mappings + for (const auto& mapping : plan_node_->column_mappings()) { + auto arrow_col = rb.ColumnAt(mapping.input_column_index()); + int64_t num_rows = arrow_col->length(); + + // Create ClickHouse column based on data type + switch (mapping.column_type()) { + case types::INT64: { + auto col = std::make_shared(); + for (int64_t i = 0; i < num_rows; ++i) { + col->Append(types::GetValueFromArrowArray(arrow_col.get(), i)); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + case types::FLOAT64: { + auto col = std::make_shared(); + for (int64_t i = 0; i < num_rows; ++i) { + col->Append(types::GetValueFromArrowArray(arrow_col.get(), i)); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + case types::STRING: { + auto col = std::make_shared(); + for (int64_t i = 0; i < num_rows; ++i) { + col->Append(types::GetValueFromArrowArray(arrow_col.get(), i)); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + case types::TIME64NS: { + auto col = std::make_shared(9); + for (int64_t i = 0; i < num_rows; ++i) { + int64_t ns_val = types::GetValueFromArrowArray(arrow_col.get(), i); + col->Append(ns_val); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + case types::BOOLEAN: { + auto col = std::make_shared(); + for (int64_t i = 0; i < num_rows; ++i) { + col->Append(types::GetValueFromArrowArray(arrow_col.get(), i) ? 1 : 0); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + case types::UINT128: { + // UINT128 is exported as STRING (UUID format) + auto col = std::make_shared(); + for (int64_t i = 0; i < num_rows; ++i) { + auto val = types::GetValueFromArrowArray(arrow_col.get(), i); + std::string uuid_str = sole::rebuild(absl::Uint128High64(val), absl::Uint128Low64(val)).str(); + col->Append(uuid_str); + } + block.AppendColumn(mapping.clickhouse_column_name(), col); + break; + } + default: + return error::InvalidArgument("Unsupported data type for ClickHouse export: $0", + types::ToString(mapping.column_type())); + } + } + + // Insert the block into ClickHouse + clickhouse_client_->Insert(plan_node_->table_name(), block); + + if (rb.eos()) { + sent_eos_ = true; + } + + return Status::OK(); +} + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/clickhouse_export_sink_node.h b/src/carnot/exec/clickhouse_export_sink_node.h new file mode 100644 index 00000000000..26478afe037 --- /dev/null +++ b/src/carnot/exec/clickhouse_export_sink_node.h @@ -0,0 +1,55 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +#pragma once + +#include +#include +#include +#include + +#include "src/carnot/exec/exec_node.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/common/base/base.h" +#include "src/shared/types/types.h" + +namespace px { +namespace carnot { +namespace exec { + +class ClickHouseExportSinkNode : public SinkNode { + public: + virtual ~ClickHouseExportSinkNode() = default; + + protected: + std::string DebugStringImpl() override; + Status InitImpl(const plan::Operator& plan_node) override; + Status PrepareImpl(ExecState* exec_state) override; + Status OpenImpl(ExecState* exec_state) override; + Status CloseImpl(ExecState* exec_state) override; + Status ConsumeNextImpl(ExecState* exec_state, const table_store::schema::RowBatch& rb, + size_t parent_index) override; + + private: + std::unique_ptr input_descriptor_; + std::unique_ptr clickhouse_client_; + std::unique_ptr plan_node_; +}; + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/clickhouse_export_sink_node_test.cc b/src/carnot/exec/clickhouse_export_sink_node_test.cc new file mode 100644 index 00000000000..75913be408c --- /dev/null +++ b/src/carnot/exec/clickhouse_export_sink_node_test.cc @@ -0,0 +1,484 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/exec/clickhouse_export_sink_node.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "src/carnot/exec/test_utils.h" +#include "src/carnot/plan/operators.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/carnot/udf/registry.h" +#include "src/common/event/time_system.h" +#include "src/common/testing/test_utils/container_runner.h" +#include "src/common/testing/testing.h" +#include "src/shared/metadata/metadata_state.h" +#include "src/shared/types/arrow_adapter.h" +#include "src/shared/types/column_wrapper.h" +#include "src/shared/types/types.h" + +namespace px { +namespace carnot { +namespace exec { + +using table_store::schema::RowBatch; +using table_store::schema::RowDescriptor; +using ::testing::_; + +class ClickHouseExportSinkNodeTest : public ::testing::Test { + protected: + static constexpr char kClickHouseImage[] = + "src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse.tar"; + static constexpr char kClickHouseReadyMessage[] = "Ready for connections"; + static constexpr int kClickHousePort = 9000; + + void SetUp() override { + // Set up function registry and exec state + func_registry_ = std::make_unique("test_registry"); + auto table_store = std::make_shared(); + exec_state_ = std::make_unique( + func_registry_.get(), table_store, MockResultSinkStubGenerator, MockMetricsStubGenerator, + MockTraceStubGenerator, MockLogStubGenerator, sole::uuid4(), nullptr); + + // Create a minimal agent metadata state for test execution + auto metadata_state = std::make_shared( + "test_host", // hostname + 1, // asid + getpid(), // pid + 0, // start_time + sole::uuid4(), // agent_id + "", // pod_name + sole::uuid4(), // vizier_id + "test_vizier", // vizier_name + "", // vizier_namespace + time_system_.get()); // time_system + exec_state_->set_metadata_state(metadata_state); + + // Start ClickHouse container + clickhouse_server_ = + std::make_unique(px::testing::BazelRunfilePath(kClickHouseImage), + "clickhouse_export_test", kClickHouseReadyMessage); + + std::vector options = { + absl::Substitute("--publish=$0:$0", kClickHousePort), + "--env=CLICKHOUSE_PASSWORD=test_password", + "--network=host", + }; + + ASSERT_OK(clickhouse_server_->Run(std::chrono::seconds{60}, options, {}, true, + std::chrono::seconds{300})); + + // Give ClickHouse time to initialize + std::this_thread::sleep_for(std::chrono::seconds(5)); + + // Create ClickHouse client for verification + SetupClickHouseClient(); + } + + void TearDown() override { + if (client_) { + client_.reset(); + } + } + + void SetupClickHouseClient() { + clickhouse::ClientOptions client_options; + client_options.SetHost("localhost"); + client_options.SetPort(kClickHousePort); + client_options.SetUser("default"); + client_options.SetPassword("test_password"); + client_options.SetDefaultDatabase("default"); + + const int kMaxRetries = 5; + for (int i = 0; i < kMaxRetries; ++i) { + LOG(INFO) << "Attempting to connect to ClickHouse (attempt " << (i + 1) << "/" << kMaxRetries + << ")..."; + try { + client_ = std::make_unique(client_options); + client_->Execute("SELECT 1"); + break; + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to connect: " << e.what(); + if (i < kMaxRetries - 1) { + std::this_thread::sleep_for(std::chrono::seconds(2)); + } else { + throw; + } + } + } + } + + void CreateExportTable(const std::string& table_name) { + try { + client_->Execute(absl::Substitute("DROP TABLE IF EXISTS $0", table_name)); + + client_->Execute(absl::Substitute(R"( + CREATE TABLE $0 ( + time_ DateTime64(9), + hostname String, + count Int64, + latency Float64 + ) ENGINE = MergeTree() + ORDER BY time_ + )", table_name)); + + LOG(INFO) << "Export table created successfully: " << table_name; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to create export table: " << e.what(); + throw; + } + } + + std::vector> QueryTable(const std::string& query) { + std::vector> results; + + try { + client_->Select(query, [&](const clickhouse::Block& block) { + for (size_t row_idx = 0; row_idx < block.GetRowCount(); ++row_idx) { + std::vector row; + for (size_t col_idx = 0; col_idx < block.GetColumnCount(); ++col_idx) { + auto col = block[col_idx]; + std::string value; + + if (auto int_col = col->As()) { + value = std::to_string((*int_col)[row_idx]); + } else if (auto uint_col = col->As()) { + value = std::to_string((*uint_col)[row_idx]); + } else if (auto float_col = col->As()) { + value = std::to_string((*float_col)[row_idx]); + } else if (auto str_col = col->As()) { + value = (*str_col)[row_idx]; + } else if (auto dt_col = col->As()) { + value = std::to_string((*dt_col)[row_idx]); + } else { + value = ""; + } + + row.push_back(value); + } + results.push_back(row); + } + }); + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to query table: " << e.what(); + throw; + } + + return results; + } + + std::unique_ptr CreatePlanNode( + const std::string& table_name) { + planpb::Operator op; + op.set_op_type(planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR); + auto* ch_op = op.mutable_clickhouse_sink_op(); + + auto* config = ch_op->mutable_clickhouse_config(); + config->set_host("localhost"); + config->set_port(kClickHousePort); + config->set_username("default"); + config->set_password("test_password"); + config->set_database("default"); + + ch_op->set_table_name(table_name); + + // Add column mappings + auto* mapping0 = ch_op->add_column_mappings(); + mapping0->set_input_column_index(0); + mapping0->set_clickhouse_column_name("time_"); + mapping0->set_column_type(types::TIME64NS); + + auto* mapping1 = ch_op->add_column_mappings(); + mapping1->set_input_column_index(1); + mapping1->set_clickhouse_column_name("hostname"); + mapping1->set_column_type(types::STRING); + + auto* mapping2 = ch_op->add_column_mappings(); + mapping2->set_input_column_index(2); + mapping2->set_clickhouse_column_name("count"); + mapping2->set_column_type(types::INT64); + + auto* mapping3 = ch_op->add_column_mappings(); + mapping3->set_input_column_index(3); + mapping3->set_clickhouse_column_name("latency"); + mapping3->set_column_type(types::FLOAT64); + + auto plan_node = std::make_unique(1); + EXPECT_OK(plan_node->Init(op.clickhouse_sink_op())); + + return plan_node; + } + + std::unique_ptr clickhouse_server_; + std::unique_ptr client_; + std::unique_ptr exec_state_; + std::unique_ptr func_registry_; + std::unique_ptr time_system_ = std::make_unique(); +}; + +TEST_F(ClickHouseExportSinkNodeTest, BasicExport) { + const std::string table_name = "export_test_basic"; + CreateExportTable(table_name); + + auto plan_node = CreatePlanNode(table_name); + + // Define input schema + RowDescriptor input_rd({types::TIME64NS, types::STRING, types::INT64, types::FLOAT64}); + + // Create node tester + auto tester = exec::ExecNodeTester( + *plan_node, RowDescriptor({}), {input_rd}, exec_state_.get()); + + // Create test data + auto rb1 = RowBatchBuilder(input_rd, 2, /*eow*/ false, /*eos*/ false) + .AddColumn({1000000000000000000LL, 2000000000000000000LL}) + .AddColumn({"host1", "host2"}) + .AddColumn({100, 200}) + .AddColumn({1.5, 2.5}) + .get(); + + auto rb2 = RowBatchBuilder(input_rd, 1, /*eow*/ true, /*eos*/ true) + .AddColumn({3000000000000000000LL}) + .AddColumn({"host3"}) + .AddColumn({300}) + .AddColumn({3.5}) + .get(); + + // Send data to sink + tester.ConsumeNext(rb1, 0, 0); + tester.ConsumeNext(rb2, 0, 0); + tester.Close(); + + // Verify data was inserted + auto results = QueryTable(absl::Substitute("SELECT hostname, count, latency FROM $0 ORDER BY time_", table_name)); + + ASSERT_EQ(results.size(), 3); + EXPECT_EQ(results[0][0], "host1"); + EXPECT_EQ(results[0][1], "100"); + EXPECT_THAT(results[0][2], ::testing::StartsWith("1.5")); + + EXPECT_EQ(results[1][0], "host2"); + EXPECT_EQ(results[1][1], "200"); + EXPECT_THAT(results[1][2], ::testing::StartsWith("2.5")); + + EXPECT_EQ(results[2][0], "host3"); + EXPECT_EQ(results[2][1], "300"); + EXPECT_THAT(results[2][2], ::testing::StartsWith("3.5")); +} + +TEST_F(ClickHouseExportSinkNodeTest, EmptyBatch) { + const std::string table_name = "export_test_empty"; + CreateExportTable(table_name); + + auto plan_node = CreatePlanNode(table_name); + + RowDescriptor input_rd({types::TIME64NS, types::STRING, types::INT64, types::FLOAT64}); + + auto tester = exec::ExecNodeTester( + *plan_node, RowDescriptor({}), {input_rd}, exec_state_.get()); + + // Send only EOS batch + auto rb = RowBatchBuilder(input_rd, 0, /*eow*/ true, /*eos*/ true) + .AddColumn({}) + .AddColumn({}) + .AddColumn({}) + .AddColumn({}) + .get(); + + tester.ConsumeNext(rb, 0, 0); + tester.Close(); + + // Verify no data was inserted + auto results = QueryTable(absl::Substitute("SELECT COUNT(*) FROM $0", table_name)); + + ASSERT_EQ(results.size(), 1); + EXPECT_EQ(results[0][0], "0"); +} + +TEST_F(ClickHouseExportSinkNodeTest, MultipleBatches) { + const std::string table_name = "export_test_multiple"; + CreateExportTable(table_name); + + auto plan_node = CreatePlanNode(table_name); + + RowDescriptor input_rd({types::TIME64NS, types::STRING, types::INT64, types::FLOAT64}); + + auto tester = exec::ExecNodeTester( + *plan_node, RowDescriptor({}), {input_rd}, exec_state_.get()); + + // Send multiple batches + for (int i = 0; i < 5; ++i) { + bool is_last = (i == 4); + auto rb = RowBatchBuilder(input_rd, 1, /*eow*/ is_last, /*eos*/ is_last) + .AddColumn({(i + 1) * 1000000000000000000LL}) + .AddColumn({absl::Substitute("host$0", i)}) + .AddColumn({i * 100}) + .AddColumn({i * 1.5}) + .get(); + + tester.ConsumeNext(rb, 0, 0); + } + + tester.Close(); + + // Verify all batches were inserted + auto results = QueryTable(absl::Substitute("SELECT COUNT(*) FROM $0", table_name)); + + ASSERT_EQ(results.size(), 1); + EXPECT_EQ(results[0][0], "5"); + + // Verify data order + auto ordered_results = QueryTable(absl::Substitute("SELECT hostname FROM $0 ORDER BY time_", table_name)); + + ASSERT_EQ(ordered_results.size(), 5); + for (int i = 0; i < 5; ++i) { + EXPECT_EQ(ordered_results[i][0], absl::Substitute("host$0", i)); + } +} + +TEST_F(ClickHouseExportSinkNodeTest, UINT128Export) { + const std::string table_name = "export_test_uint128"; + + // Create table with String column for UUID + try { + client_->Execute(absl::Substitute("DROP TABLE IF EXISTS $0", table_name)); + + client_->Execute(absl::Substitute(R"( + CREATE TABLE $0 ( + time_ DateTime64(9), + upid String, + hostname String, + value Int64 + ) ENGINE = MergeTree() + ORDER BY time_ + )", table_name)); + + LOG(INFO) << "UINT128 export table created successfully: " << table_name; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to create UINT128 export table: " << e.what(); + throw; + } + + // Create plan node for UINT128 test + planpb::Operator op; + op.set_op_type(planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR); + auto* ch_op = op.mutable_clickhouse_sink_op(); + + auto* config = ch_op->mutable_clickhouse_config(); + config->set_host("localhost"); + config->set_port(kClickHousePort); + config->set_username("default"); + config->set_password("test_password"); + config->set_database("default"); + + ch_op->set_table_name(table_name); + + // Add column mappings + auto* mapping0 = ch_op->add_column_mappings(); + mapping0->set_input_column_index(0); + mapping0->set_clickhouse_column_name("time_"); + mapping0->set_column_type(types::TIME64NS); + + auto* mapping1 = ch_op->add_column_mappings(); + mapping1->set_input_column_index(1); + mapping1->set_clickhouse_column_name("upid"); + mapping1->set_column_type(types::UINT128); + + auto* mapping2 = ch_op->add_column_mappings(); + mapping2->set_input_column_index(2); + mapping2->set_clickhouse_column_name("hostname"); + mapping2->set_column_type(types::STRING); + + auto* mapping3 = ch_op->add_column_mappings(); + mapping3->set_input_column_index(3); + mapping3->set_clickhouse_column_name("value"); + mapping3->set_column_type(types::INT64); + + auto plan_node = std::make_unique(1); + EXPECT_OK(plan_node->Init(op.clickhouse_sink_op())); + + // Define input schema + RowDescriptor input_rd({types::TIME64NS, types::UINT128, types::STRING, types::INT64}); + + // Create node tester + auto tester = exec::ExecNodeTester( + *plan_node, RowDescriptor({}), {input_rd}, exec_state_.get()); + + // Create test UUIDs + auto uuid1 = sole::uuid4(); + auto uuid2 = sole::uuid4(); + auto uuid3 = sole::uuid4(); + + absl::uint128 upid1 = absl::MakeUint128(uuid1.ab, uuid1.cd); + absl::uint128 upid2 = absl::MakeUint128(uuid2.ab, uuid2.cd); + absl::uint128 upid3 = absl::MakeUint128(uuid3.ab, uuid3.cd); + + // Create test data with UINT128 values + auto rb1 = RowBatchBuilder(input_rd, 2, /*eow*/ false, /*eos*/ false) + .AddColumn({1000000000000000000LL, 2000000000000000000LL}) + .AddColumn({upid1, upid2}) + .AddColumn({"host1", "host2"}) + .AddColumn({100, 200}) + .get(); + + auto rb2 = RowBatchBuilder(input_rd, 1, /*eow*/ true, /*eos*/ true) + .AddColumn({3000000000000000000LL}) + .AddColumn({upid3}) + .AddColumn({"host3"}) + .AddColumn({300}) + .get(); + + // Send data to sink + tester.ConsumeNext(rb1, 0, 0); + tester.ConsumeNext(rb2, 0, 0); + tester.Close(); + + // Verify data was inserted and UINT128 values were converted to UUID strings + auto results = QueryTable(absl::Substitute("SELECT upid, hostname, value FROM $0 ORDER BY time_", table_name)); + + ASSERT_EQ(results.size(), 3); + + // Check that UINT128 values were converted to valid UUID strings + EXPECT_EQ(results[0][0], uuid1.str()); + EXPECT_EQ(results[0][1], "host1"); + EXPECT_EQ(results[0][2], "100"); + + EXPECT_EQ(results[1][0], uuid2.str()); + EXPECT_EQ(results[1][1], "host2"); + EXPECT_EQ(results[1][2], "200"); + + EXPECT_EQ(results[2][0], uuid3.str()); + EXPECT_EQ(results[2][1], "host3"); + EXPECT_EQ(results[2][2], "300"); +} + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/clickhouse_source_node.cc b/src/carnot/exec/clickhouse_source_node.cc new file mode 100644 index 00000000000..a27e4363a12 --- /dev/null +++ b/src/carnot/exec/clickhouse_source_node.cc @@ -0,0 +1,725 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/exec/clickhouse_source_node.h" + +#include +#include +#include +#include + +#include +#include + +#include "src/carnot/planpb/plan.pb.h" +#include "src/common/base/base.h" +#include "src/shared/types/arrow_adapter.h" +#include "src/shared/types/types.h" + +namespace px { +namespace carnot { +namespace exec { + +std::string ClickHouseSourceNode::DebugStringImpl() { + return absl::Substitute("Exec::ClickHouseSourceNode: ", base_query_, + output_descriptor_->DebugString()); +} + +Status ClickHouseSourceNode::InitImpl(const plan::Operator& plan_node) { + CHECK(plan_node.op_type() == planpb::OperatorType::CLICKHOUSE_SOURCE_OPERATOR); + const auto* source_plan_node = static_cast(&plan_node); + + // Copy the plan node to local object + plan_node_ = std::make_unique(*source_plan_node); + + // Extract connection parameters from plan node + host_ = plan_node_->host(); + port_ = plan_node_->port(); + username_ = plan_node_->username(); + password_ = plan_node_->password(); + database_ = plan_node_->database(); + base_query_ = plan_node_->query(); + batch_size_ = plan_node_->batch_size(); + streaming_ = plan_node_->streaming(); + + // Initialize cursor state + current_offset_ = 0; + has_more_data_ = true; + current_block_index_ = 0; + + // Extract time filtering parameters from plan node + timestamp_column_ = plan_node_->timestamp_column(); + partition_column_ = plan_node_->partition_column(); + + // Convert start/end times from nanoseconds to seconds for ClickHouse DateTime + if (plan_node_->start_time() > 0) { + start_time_ = plan_node_->start_time() / 1000000000LL; // Convert ns to seconds + } + if (plan_node_->end_time() > 0) { + end_time_ = plan_node_->end_time() / 1000000000LL; // Convert ns to seconds + } + + return Status::OK(); +} + +Status ClickHouseSourceNode::PrepareImpl(ExecState*) { return Status::OK(); } + +Status ClickHouseSourceNode::OpenImpl(ExecState*) { + // Create ClickHouse client + clickhouse::ClientOptions options; + options.SetHost(host_); + options.SetPort(port_); + options.SetUser(username_); + options.SetPassword(password_); + options.SetDefaultDatabase(database_); + + try { + client_ = std::make_unique(options); + } catch (const std::exception& e) { + return error::Internal("Failed to create ClickHouse client: $0", e.what()); + } + + return Status::OK(); +} + +Status ClickHouseSourceNode::CloseImpl(ExecState*) { + client_.reset(); + current_batch_blocks_.clear(); + + // Reset cursor state + current_offset_ = 0; + current_block_index_ = 0; + has_more_data_ = true; + + return Status::OK(); +} + +StatusOr ClickHouseSourceNode::ClickHouseTypeToPixieType( + const clickhouse::TypeRef& ch_type) { + const auto& type_name = ch_type->GetName(); + + // Integer types - Pixie only supports INT64 + if (type_name == "UInt8" || type_name == "UInt16" || type_name == "UInt32" || + type_name == "UInt64" || type_name == "Int8" || type_name == "Int16" || + type_name == "Int32" || type_name == "Int64") { + return types::DataType::INT64; + } + + // UInt128 + if (type_name == "UInt128") { + return types::DataType::UINT128; + } + + // Floating point types - Pixie only supports FLOAT64 + if (type_name == "Float32" || type_name == "Float64") { + return types::DataType::FLOAT64; + } + + // String types + if (type_name == "String" || type_name == "FixedString") { + return types::DataType::STRING; + } + + // Date/time types + if (type_name == "DateTime" || type_name.find("DateTime64") == 0) { + return types::DataType::TIME64NS; + } + + // Boolean + if (type_name == "Bool") { + return types::DataType::BOOLEAN; + } + + return error::InvalidArgument("Unsupported ClickHouse type: $0", type_name); +} + +StatusOr> ClickHouseSourceNode::ConvertClickHouseBlockToRowBatch( + const clickhouse::Block& block, bool /*is_last_block*/) { + auto num_rows = block.GetRowCount(); + auto num_cols = block.GetColumnCount(); + + // Create output row descriptor if this is the first block + if (current_block_index_ == 0) { + std::vector col_types; + for (size_t i = 0; i < num_cols; ++i) { + PX_ASSIGN_OR_RETURN(auto pixie_type, ClickHouseTypeToPixieType(block[i]->Type())); + col_types.push_back(pixie_type); + } + // Note: In a real implementation, we would get column names from the plan + // or from ClickHouse metadata + } + + auto row_batch = std::make_unique(*output_descriptor_, num_rows); + + // Convert each column + for (size_t col_idx = 0; col_idx < num_cols; ++col_idx) { + const auto& ch_column = block[col_idx]; + const auto& type_name = ch_column->Type()->GetName(); + + // Check what the expected output type is for this column + auto expected_type = output_descriptor_->type(col_idx); + + // For now, implement conversion for common types + // This is where column type inference happens + + // Special case: String in ClickHouse that should be UINT128 in Pixie + if (type_name == "String" && expected_type == types::DataType::UINT128) { + auto typed_col = ch_column->As(); + auto builder = types::MakeArrowBuilder(types::DataType::UINT128, arrow::default_memory_pool()); + PX_RETURN_IF_ERROR(builder->Reserve(num_rows)); + + for (size_t i = 0; i < num_rows; ++i) { + std::string value(typed_col->At(i)); + + // Parse "high:low" format + size_t colon_pos = value.find(':'); + if (colon_pos == std::string::npos) { + return error::InvalidArgument("Invalid UINT128 string format: $0 (expected high:low)", value); + } + + uint64_t high = std::stoull(value.substr(0, colon_pos)); + uint64_t low = std::stoull(value.substr(colon_pos + 1)); + absl::uint128 uint128_val = absl::MakeUint128(high, low); + + PX_RETURN_IF_ERROR(table_store::schema::CopyValue(builder.get(), uint128_val)); + } + + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder->Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + + continue; + } + + // Integer types - all map to INT64 in Pixie + + // TODO(ddelnano): UInt8 is a special case since it can map to Pixie's boolean type. + // Figure out how to handle that properly + if (type_name == "UInt8") { + auto typed_col = ch_column->As(); + arrow::BooleanBuilder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(typed_col->At(i) != 0); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "UInt16") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "UInt32") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "UInt64") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Int8") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Int16") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Int32") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Int64") { + auto typed_col = ch_column->As(); + arrow::Int64Builder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(typed_col->At(i)); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "String") { + auto typed_col = ch_column->As(); + arrow::StringBuilder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + + for (size_t i = 0; i < num_rows; ++i) { + // Convert string_view to string + std::string value(typed_col->At(i)); + PX_RETURN_IF_ERROR(builder.Append(value)); + } + + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + + } else if (type_name == "Float32") { + auto typed_col = ch_column->As(); + arrow::DoubleBuilder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(static_cast(typed_col->At(i))); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Float64") { + auto typed_col = ch_column->As(); + arrow::DoubleBuilder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(typed_col->At(i)); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "Bool") { + auto typed_col = ch_column->As(); + arrow::BooleanBuilder builder; + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + for (size_t i = 0; i < num_rows; ++i) { + builder.UnsafeAppend(typed_col->At(i) != 0); + } + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + } else if (type_name == "DateTime") { + auto typed_col = ch_column->As(); + arrow::Time64Builder builder(arrow::time64(arrow::TimeUnit::NANO), + arrow::default_memory_pool()); + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + + for (size_t i = 0; i < num_rows; ++i) { + // Convert DateTime (seconds since epoch) to nanoseconds + int64_t ns = static_cast(typed_col->At(i)) * 1000000000LL; + builder.UnsafeAppend(ns); + } + + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + + } else if (type_name.find("DateTime64") == 0) { + auto typed_col = ch_column->As(); + arrow::Time64Builder builder(arrow::time64(arrow::TimeUnit::NANO), + arrow::default_memory_pool()); + PX_RETURN_IF_ERROR(builder.Reserve(num_rows)); + + for (size_t i = 0; i < num_rows; ++i) { + // DateTime64 stores time with sub-second precision + // The value is already in the correct precision (e.g., nanoseconds for DateTime64(9)) + // We need to convert to nanoseconds if it's not already + int64_t value = typed_col->At(i); + + // Extract precision from type name (e.g., "DateTime64(9)" -> 9) + size_t precision = 3; // default to milliseconds + size_t start = type_name.find('('); + if (start != std::string::npos) { + size_t end = type_name.find(')', start); + if (end != std::string::npos) { + precision = std::stoi(type_name.substr(start + 1, end - start - 1)); + } + } + + // Convert to nanoseconds based on precision + int64_t ns = value; + if (precision < 9) { + // Scale up to nanoseconds + int64_t multiplier = 1; + for (size_t p = precision; p < 9; p++) { + multiplier *= 10; + } + ns = value * multiplier; + } else if (precision > 9) { + // Scale down to nanoseconds + int64_t divisor = 1; + for (size_t p = 9; p < precision; p++) { + divisor *= 10; + } + ns = value / divisor; + } + + builder.UnsafeAppend(ns); + } + + std::shared_ptr array; + PX_RETURN_IF_ERROR(builder.Finish(&array)); + PX_RETURN_IF_ERROR(row_batch->AddColumn(array)); + + } else { + return error::InvalidArgument("Unsupported ClickHouse type for conversion: $0", type_name); + } + } + + // Set end-of-window and end-of-stream flags + // Don't set them here - they should be set in GenerateNextImpl + row_batch->set_eow(false); + row_batch->set_eos(false); + + return row_batch; +} + +std::string ClickHouseSourceNode::BuildQuery() { + std::string query = base_query_; + std::vector conditions; + + // Add time filtering if start/end times are specified and timestamp column is set + if (!timestamp_column_.empty()) { + if (start_time_.has_value()) { + conditions.push_back(absl::Substitute("$0 >= $1", timestamp_column_, start_time_.value())); + } + if (end_time_.has_value()) { + conditions.push_back(absl::Substitute("$0 <= $1", timestamp_column_, end_time_.value())); + } + } + + // Add partition column filtering if specified + if (!partition_column_.empty()) { + // Get the current hostname for partition filtering + char hostname[256]; + gethostname(hostname, sizeof(hostname)); + conditions.push_back(absl::Substitute("$0 = '$1'", partition_column_, hostname)); + } + + // Parse the base query to find WHERE and ORDER BY positions + std::string lower_query = query; + std::transform(lower_query.begin(), lower_query.end(), lower_query.begin(), ::tolower); + + size_t where_pos = lower_query.find(" where "); + size_t order_by_pos = lower_query.find(" order by "); + size_t limit_pos = lower_query.find(" limit "); + + // Determine insertion point for conditions + if (!conditions.empty()) { + std::string conditions_clause = absl::StrJoin(conditions, " AND "); + + if (where_pos != std::string::npos) { + // Query already has WHERE clause + size_t insert_pos = std::string::npos; + + // Find where to insert the additional conditions + if (order_by_pos != std::string::npos && order_by_pos > where_pos) { + insert_pos = order_by_pos; + } else if (limit_pos != std::string::npos && limit_pos > where_pos) { + insert_pos = limit_pos; + } else { + insert_pos = query.length(); + } + + query.insert(insert_pos, " AND " + conditions_clause); + } else { + // No WHERE clause, need to add one + size_t insert_pos = std::string::npos; + + if (order_by_pos != std::string::npos) { + insert_pos = order_by_pos; + } else if (limit_pos != std::string::npos) { + insert_pos = limit_pos; + } else { + insert_pos = query.length(); + } + + query.insert(insert_pos, " WHERE " + conditions_clause); + } + } + + // Update lower_query after modifications + lower_query = query; + std::transform(lower_query.begin(), lower_query.end(), lower_query.begin(), ::tolower); + + // Add ORDER BY clause if needed + if (lower_query.find(" order by ") == std::string::npos) { + if (!timestamp_column_.empty()) { + query += absl::Substitute(" ORDER BY $0", timestamp_column_); + } else { + // Fall back to ordering by first column for consistent pagination + query += " ORDER BY 1"; + } + } + + // Add LIMIT and OFFSET for pagination + query += absl::Substitute(" LIMIT $0 OFFSET $1", batch_size_, current_offset_); + + return query; +} + +Status ClickHouseSourceNode::ExecuteBatchQuery() { + // Clear previous batch results + current_batch_blocks_.clear(); + current_block_index_ = 0; + + if (!has_more_data_) { + return Status::OK(); + } + + std::string query = BuildQuery(); + VLOG(1) << "Executing ClickHouse query: " << query; + + try { + size_t rows_received = 0; + client_->Select(query, [this, &rows_received](const clickhouse::Block& block) { + // Only store non-empty blocks + if (block.GetRowCount() > 0) { + VLOG(1) << "Received block with " << block.GetRowCount() << " rows"; + current_batch_blocks_.push_back(block); + rows_received += block.GetRowCount(); + } + }); + + VLOG(1) << "Total rows received: " << rows_received << ", batch size: " << batch_size_; + + // Update cursor state + current_offset_ += rows_received; + if (rows_received < batch_size_) { + // We got fewer rows than requested, so no more data available + has_more_data_ = false; + } + } catch (const std::exception& e) { + return error::Internal("Failed to execute ClickHouse batch query: $0", e.what()); + } + + return Status::OK(); +} + +Status ClickHouseSourceNode::GenerateNextImpl(ExecState* exec_state) { + // If we need to fetch more data + if (current_block_index_ >= current_batch_blocks_.size()) { + current_block_index_ = 0; + current_batch_blocks_.clear(); + + if (!has_more_data_) { + // No more data available - send empty batch with eos=true + PX_ASSIGN_OR_RETURN(auto empty_batch, + RowBatch::WithZeroRows(*output_descriptor_, true, true)); + PX_RETURN_IF_ERROR(SendRowBatchToChildren(exec_state, *empty_batch)); + return Status::OK(); + } + + // Fetch next batch from ClickHouse + PX_RETURN_IF_ERROR(ExecuteBatchQuery()); + + // If still no blocks after fetching, we're done + if (current_batch_blocks_.empty()) { + PX_ASSIGN_OR_RETURN(auto empty_batch, + RowBatch::WithZeroRows(*output_descriptor_, true, true)); + PX_RETURN_IF_ERROR(SendRowBatchToChildren(exec_state, *empty_batch)); + return Status::OK(); + } + } + + // Calculate total rows in all blocks + size_t total_rows = 0; + for (const auto& block : current_batch_blocks_) { + total_rows += block.GetRowCount(); + } + + // Create a merged RowBatch + auto merged_batch = std::make_unique(*output_descriptor_, total_rows); + + // Process each column + for (size_t col_idx = 0; col_idx < output_descriptor_->size(); ++col_idx) { + // Get the data type from output descriptor + auto data_type = output_descriptor_->type(col_idx); + + // Create appropriate builder based on data type + std::shared_ptr builder; + switch (data_type) { + case types::DataType::INT64: + builder = std::make_shared(); + break; + case types::DataType::UINT128: + builder = types::MakeArrowBuilder(types::DataType::UINT128, arrow::default_memory_pool()); + break; + case types::DataType::FLOAT64: + builder = std::make_shared(); + break; + case types::DataType::STRING: + builder = std::make_shared(); + break; + case types::DataType::BOOLEAN: + builder = std::make_shared(); + break; + case types::DataType::TIME64NS: + builder = std::make_shared(arrow::time64(arrow::TimeUnit::NANO), + arrow::default_memory_pool()); + break; + default: + return error::InvalidArgument("Unsupported data type for column $0", col_idx); + } + + // Reserve space for all rows + PX_RETURN_IF_ERROR(builder->Reserve(total_rows)); + + // Append data from all blocks + for (const auto& block : current_batch_blocks_) { + PX_ASSIGN_OR_RETURN(auto row_batch, ConvertClickHouseBlockToRowBatch(block, false)); + auto array = row_batch->ColumnAt(col_idx); + + // Append values from this block's array + switch (data_type) { + case types::DataType::INT64: { + auto typed_array = std::static_pointer_cast(array); + auto typed_builder = std::static_pointer_cast(builder); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(typed_builder->AppendNull()); + } else { + typed_builder->UnsafeAppend(typed_array->Value(i)); + } + } + break; + } + case types::DataType::UINT128: { + auto typed_array = std::static_pointer_cast(array); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(builder->AppendNull()); + } else { + auto val = types::GetValueFromArrowArray(array.get(), i); + PX_RETURN_IF_ERROR(table_store::schema::CopyValue(builder.get(), val)); + } + } + break; + } + case types::DataType::TIME64NS: { + auto typed_array = std::static_pointer_cast(array); + auto typed_builder = std::static_pointer_cast(builder); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(typed_builder->AppendNull()); + } else { + typed_builder->UnsafeAppend(typed_array->Value(i)); + } + } + break; + } + case types::DataType::FLOAT64: { + auto typed_array = std::static_pointer_cast(array); + auto typed_builder = std::static_pointer_cast(builder); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(typed_builder->AppendNull()); + } else { + typed_builder->UnsafeAppend(typed_array->Value(i)); + } + } + break; + } + case types::DataType::STRING: { + auto typed_array = std::static_pointer_cast(array); + auto typed_builder = std::static_pointer_cast(builder); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(typed_builder->AppendNull()); + } else { + PX_RETURN_IF_ERROR(typed_builder->Append(typed_array->GetString(i))); + } + } + break; + } + case types::DataType::BOOLEAN: { + auto typed_array = std::static_pointer_cast(array); + auto typed_builder = std::static_pointer_cast(builder); + for (int i = 0; i < typed_array->length(); i++) { + if (typed_array->IsNull(i)) { + PX_RETURN_IF_ERROR(typed_builder->AppendNull()); + } else { + typed_builder->UnsafeAppend(typed_array->Value(i)); + } + } + break; + } + default: + return error::InvalidArgument("Unsupported data type for column $0", col_idx); + } + } + + // Finish building and add column + std::shared_ptr merged_array; + PX_RETURN_IF_ERROR(builder->Finish(&merged_array)); + PX_RETURN_IF_ERROR(merged_batch->AddColumn(merged_array)); + } + + // Set proper end-of-window and end-of-stream flags + bool is_last_batch = !has_more_data_; + if (is_last_batch) { + merged_batch->set_eow(true); + merged_batch->set_eos(true); + } else { + merged_batch->set_eow(false); + merged_batch->set_eos(false); + } + + // Update stats + rows_processed_ += merged_batch->num_rows(); + bytes_processed_ += merged_batch->NumBytes(); + + // Send to children + PX_RETURN_IF_ERROR(SendRowBatchToChildren(exec_state, *merged_batch)); + + // Mark all blocks as processed + current_block_index_ = current_batch_blocks_.size(); + + return Status::OK(); +} + +bool ClickHouseSourceNode::NextBatchReady() { + // We're ready if we have blocks in current batch or if we can fetch more data + return (current_block_index_ < current_batch_blocks_.size()) || has_more_data_; +} + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/clickhouse_source_node.h b/src/carnot/exec/clickhouse_source_node.h new file mode 100644 index 00000000000..84a14c9063a --- /dev/null +++ b/src/carnot/exec/clickhouse_source_node.h @@ -0,0 +1,107 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +#include +#include +#include + +#include "src/carnot/exec/exec_node.h" +#include "src/carnot/exec/exec_state.h" +#include "src/carnot/plan/operators.h" +#include "src/common/base/base.h" +#include "src/common/base/status.h" +#include "src/shared/types/types.h" +#include "src/table_store/schema/row_batch.h" + +namespace px { +namespace carnot { +namespace exec { + +using table_store::schema::RowBatch; +using table_store::schema::RowDescriptor; + +class ClickHouseSourceNode : public SourceNode { + public: + ClickHouseSourceNode() = default; + virtual ~ClickHouseSourceNode() = default; + + bool NextBatchReady() override; + + protected: + std::string DebugStringImpl() override; + Status InitImpl(const plan::Operator& plan_node) override; + Status PrepareImpl(ExecState* exec_state) override; + Status OpenImpl(ExecState* exec_state) override; + Status CloseImpl(ExecState* exec_state) override; + Status GenerateNextImpl(ExecState* exec_state) override; + + private: + // Convert ClickHouse column types to Pixie data types + StatusOr ClickHouseTypeToPixieType(const clickhouse::TypeRef& ch_type); + + // Convert ClickHouse block to Pixie RowBatch + StatusOr> ConvertClickHouseBlockToRowBatch( + const clickhouse::Block& block, bool is_last_block); + + // Execute a batch query + Status ExecuteBatchQuery(); + + // Build the query with time filtering and pagination + std::string BuildQuery(); + + // Connection information + std::string host_; + int port_; + std::string username_; + std::string password_; + std::string database_; + std::string base_query_; + + // Batch size and cursor tracking + size_t batch_size_ = 1024; + size_t current_offset_ = 0; + bool has_more_data_ = true; + + // Time filtering + std::optional start_time_; + std::optional end_time_; + std::string timestamp_column_; // Column to use for timestamp-based filtering and ordering + std::string partition_column_; // Column used for partitioning + + // ClickHouse client + std::unique_ptr client_; + + // Current batch results + std::vector current_batch_blocks_; + size_t current_block_index_ = 0; + + // Streaming support + bool streaming_ = false; + + // Plan node + std::unique_ptr plan_node_; +}; + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/clickhouse_source_node_test.cc b/src/carnot/exec/clickhouse_source_node_test.cc new file mode 100644 index 00000000000..18b4897b477 --- /dev/null +++ b/src/carnot/exec/clickhouse_source_node_test.cc @@ -0,0 +1,358 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/exec/clickhouse_source_node.h" + +#include + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include "src/carnot/exec/test_utils.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/carnot/planpb/test_proto.h" +#include "src/carnot/udf/registry.h" +#include "src/common/event/time_system.h" +#include "src/common/testing/test_utils/container_runner.h" +#include "src/common/testing/testing.h" +#include "src/shared/metadata/metadata_state.h" +#include "src/shared/types/arrow_adapter.h" +#include "src/shared/types/column_wrapper.h" +#include "src/shared/types/types.h" +#include "src/shared/types/typespb/types.pb.h" + +namespace px { +namespace carnot { +namespace exec { + +using table_store::Table; +using table_store::schema::RowBatch; +using table_store::schema::RowDescriptor; +using ::testing::_; + +class ClickHouseSourceNodeTest : public ::testing::Test { + protected: + static constexpr char kClickHouseImage[] = + "src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse.tar"; + static constexpr char kClickHouseReadyMessage[] = "Ready for connections"; + static constexpr int kClickHousePort = 9000; + + void SetUp() override { + // Set up function registry and exec state + func_registry_ = std::make_unique("test_registry"); + auto table_store = std::make_shared(); + exec_state_ = std::make_unique( + func_registry_.get(), table_store, MockResultSinkStubGenerator, MockMetricsStubGenerator, + MockTraceStubGenerator, MockLogStubGenerator, sole::uuid4(), nullptr); + + // Create a minimal agent metadata state for test execution + auto metadata_state = std::make_shared( + "test_host", // hostname + 1, // asid + getpid(), // pid + 0, // start_time + sole::uuid4(), // agent_id + "", // pod_name + sole::uuid4(), // vizier_id + "test_vizier", // vizier_name + "", // vizier_namespace + time_system_.get()); // time_system + exec_state_->set_metadata_state(metadata_state); + + // Start ClickHouse container + clickhouse_server_ = + std::make_unique(px::testing::BazelRunfilePath(kClickHouseImage), + "clickhouse_test", kClickHouseReadyMessage); + + std::vector options = { + absl::Substitute("--publish=$0:$0", kClickHousePort), + "--env=CLICKHOUSE_PASSWORD=test_password", + "--network=host", + }; + + ASSERT_OK(clickhouse_server_->Run(std::chrono::seconds{60}, options, {}, true, + std::chrono::seconds{300})); + + // Give ClickHouse time to initialize + std::this_thread::sleep_for(std::chrono::seconds(5)); + + // Create ClickHouse client for test data setup + SetupClickHouseClient(); + CreateTestTable(); + } + + void TearDown() override { + if (client_) { + client_.reset(); + } + } + + void SetupClickHouseClient() { + clickhouse::ClientOptions client_options; + client_options.SetHost("localhost"); + client_options.SetPort(kClickHousePort); + client_options.SetUser("default"); + client_options.SetPassword("test_password"); + client_options.SetDefaultDatabase("default"); + + const int kMaxRetries = 5; + for (int i = 0; i < kMaxRetries; ++i) { + LOG(INFO) << "Attempting to connect to ClickHouse (attempt " << (i + 1) << "/" << kMaxRetries + << ")..."; + try { + client_ = std::make_unique(client_options); + client_->Execute("SELECT 1"); + break; + } catch (const std::exception& e) { + LOG(WARNING) << "Failed to connect: " << e.what(); + if (i < kMaxRetries - 1) { + std::this_thread::sleep_for(std::chrono::seconds(2)); + } else { + throw; + } + } + } + } + + void CreateTestTable() { + try { + client_->Execute("DROP TABLE IF EXISTS test_table"); + + client_->Execute(R"( + CREATE TABLE test_table ( + id UInt64, + name String, + value Float64, + timestamp DateTime, + partition_key String + ) ENGINE = MergeTree() + PARTITION BY (timestamp, partition_key) + ORDER BY timestamp + )"); + + auto id_col = std::make_shared(); + auto name_col = std::make_shared(); + auto value_col = std::make_shared(); + auto timestamp_col = std::make_shared(); + auto partition_key_col = std::make_shared(); + + // Add test data with increasing timestamps + std::time_t base_time = std::time(nullptr) - 3600; // Start 1 hour ago + id_col->Append(1); + name_col->Append("test1"); + value_col->Append(10.5); + timestamp_col->Append(base_time); + partition_key_col->Append("partition_a"); + + id_col->Append(2); + name_col->Append("test2"); + value_col->Append(20.5); + timestamp_col->Append(base_time + 1800); // 30 minutes later + partition_key_col->Append("partition_a"); + + id_col->Append(3); + name_col->Append("test3"); + value_col->Append(30.5); + timestamp_col->Append(base_time + 3600); // 1 hour later + partition_key_col->Append("partition_b"); + + clickhouse::Block block; + block.AppendColumn("id", id_col); + block.AppendColumn("name", name_col); + block.AppendColumn("value", value_col); + block.AppendColumn("timestamp", timestamp_col); + block.AppendColumn("partition_key", partition_key_col); + + client_->Insert("test_table", block); + + LOG(INFO) << "Test table created and populated successfully"; + } catch (const std::exception& e) { + LOG(ERROR) << "Failed to create test table: " << e.what(); + throw; + } + } + + std::unique_ptr clickhouse_server_; + std::unique_ptr client_; + std::unique_ptr exec_state_; + std::unique_ptr func_registry_; + std::unique_ptr time_system_ = std::make_unique(); +}; + +TEST_F(ClickHouseSourceNodeTest, BasicQuery) { + // Create ClickHouse source operator proto + auto op_proto = planpb::testutils::CreateClickHouseSourceOperatorPB(); + std::unique_ptr plan_node = + plan::ClickHouseSourceOperator::FromProto(op_proto, 1); + + // Define expected output schema + RowDescriptor output_rd( + {types::DataType::INT64, types::DataType::STRING, types::DataType::FLOAT64}); + + // Create node tester + auto tester = exec::ExecNodeTester( + *plan_node, output_rd, std::vector({}), exec_state_.get()); + + // Verify state machine behavior + EXPECT_TRUE(tester.node()->HasBatchesRemaining()); + + // First batch should return 2 rows (batch_size = 2) + tester.GenerateNextResult().ExpectRowBatch( + RowBatchBuilder(output_rd, 2, /*eow*/ false, /*eos*/ false) + .AddColumn({1, 2}) + .AddColumn({"test1", "test2"}) + .AddColumn({10.5, 20.5}) + .get()); + + // Second batch should return remaining 1 row with eos + EXPECT_TRUE(tester.node()->HasBatchesRemaining()); + tester.GenerateNextResult().ExpectRowBatch( + RowBatchBuilder(output_rd, 1, /*eow*/ true, /*eos*/ true) + .AddColumn({3}) + .AddColumn({"test3"}) + .AddColumn({30.5}) + .get()); + + EXPECT_FALSE(tester.node()->HasBatchesRemaining()); + tester.Close(); + + // Verify metrics + EXPECT_EQ(3, tester.node()->RowsProcessed()); + EXPECT_GT(tester.node()->BytesProcessed(), 0); +} + +TEST_F(ClickHouseSourceNodeTest, EmptyResultSet) { + // Create a table with no data + client_->Execute("DROP TABLE IF EXISTS empty_table"); + client_->Execute(R"( + CREATE TABLE empty_table ( + id UInt64, + name String, + value Float64, + timestamp DateTime, + partition_key String + ) ENGINE = MergeTree() + PARTITION BY (timestamp, partition_key) + ORDER BY timestamp + )"); + + // Create operator that queries empty table + planpb::Operator op; + op.set_op_type(planpb::OperatorType::CLICKHOUSE_SOURCE_OPERATOR); + auto* ch_op = op.mutable_clickhouse_source_op(); + ch_op->set_host("localhost"); + ch_op->set_port(kClickHousePort); + ch_op->set_username("default"); + ch_op->set_password("test_password"); + ch_op->set_database("default"); + ch_op->set_query("SELECT id, name, value FROM empty_table"); + ch_op->set_batch_size(1024); + ch_op->set_streaming(false); + ch_op->add_column_names("id"); + ch_op->add_column_names("name"); + ch_op->add_column_names("value"); + ch_op->add_column_types(types::DataType::INT64); + ch_op->add_column_types(types::DataType::STRING); + ch_op->add_column_types(types::DataType::FLOAT64); + ch_op->set_timestamp_column("timestamp"); + ch_op->set_start_time(1000000000000000000LL); // Year 2001 in nanoseconds + ch_op->set_end_time(9223372036854775807LL); // Max int64 + + std::unique_ptr plan_node = plan::ClickHouseSourceOperator::FromProto(op, 1); + RowDescriptor output_rd( + {types::DataType::INT64, types::DataType::STRING, types::DataType::FLOAT64}); + + auto tester = exec::ExecNodeTester( + *plan_node, output_rd, std::vector({}), exec_state_.get()); + + EXPECT_TRUE(tester.node()->HasBatchesRemaining()); + + // Should return empty batch with eos=true + tester.GenerateNextResult().ExpectRowBatch( + RowBatchBuilder(output_rd, 0, /*eow*/ true, /*eos*/ true) + .AddColumn({}) + .AddColumn({}) + .AddColumn({}) + .get()); + + EXPECT_FALSE(tester.node()->HasBatchesRemaining()); + tester.Close(); + + EXPECT_EQ(0, tester.node()->RowsProcessed()); + EXPECT_EQ(0, tester.node()->BytesProcessed()); +} + +TEST_F(ClickHouseSourceNodeTest, FilteredQuery) { + // Create operator with WHERE clause + planpb::Operator op; + op.set_op_type(planpb::OperatorType::CLICKHOUSE_SOURCE_OPERATOR); + auto* ch_op = op.mutable_clickhouse_source_op(); + ch_op->set_host("localhost"); + ch_op->set_port(kClickHousePort); + ch_op->set_username("default"); + ch_op->set_password("test_password"); + ch_op->set_database("default"); + ch_op->set_query("SELECT id, name, value FROM test_table WHERE value > 15.0 ORDER BY id"); + ch_op->set_batch_size(1024); + ch_op->set_streaming(false); + ch_op->add_column_names("id"); + ch_op->add_column_names("name"); + ch_op->add_column_names("value"); + ch_op->add_column_types(types::DataType::INT64); + ch_op->add_column_types(types::DataType::STRING); + ch_op->add_column_types(types::DataType::FLOAT64); + ch_op->set_timestamp_column("timestamp"); + ch_op->set_start_time(1000000000000000000LL); // Year 2001 in nanoseconds + ch_op->set_end_time(9223372036854775807LL); // Max int64 + + std::unique_ptr plan_node = plan::ClickHouseSourceOperator::FromProto(op, 1); + RowDescriptor output_rd( + {types::DataType::INT64, types::DataType::STRING, types::DataType::FLOAT64}); + + auto tester = exec::ExecNodeTester( + *plan_node, output_rd, std::vector({}), exec_state_.get()); + + EXPECT_TRUE(tester.node()->HasBatchesRemaining()); + + // Should return all filtered results in one batch (2 rows < batch_size) + tester.GenerateNextResult().ExpectRowBatch( + RowBatchBuilder(output_rd, 2, /*eow*/ true, /*eos*/ true) + .AddColumn({2, 3}) + .AddColumn({"test2", "test3"}) + .AddColumn({20.5, 30.5}) + .get()); + + EXPECT_FALSE(tester.node()->HasBatchesRemaining()); + tester.Close(); + + EXPECT_EQ(2, tester.node()->RowsProcessed()); + EXPECT_GT(tester.node()->BytesProcessed(), 0); +} + +} // namespace exec +} // namespace carnot +} // namespace px diff --git a/src/carnot/exec/exec_graph.cc b/src/carnot/exec/exec_graph.cc index 705cf381e38..de38d762d7b 100644 --- a/src/carnot/exec/exec_graph.cc +++ b/src/carnot/exec/exec_graph.cc @@ -24,6 +24,8 @@ #include #include "src/carnot/exec/agg_node.h" +#include "src/carnot/exec/clickhouse_export_sink_node.h" +#include "src/carnot/exec/clickhouse_source_node.h" #include "src/carnot/exec/empty_source_node.h" #include "src/carnot/exec/equijoin_node.h" #include "src/carnot/exec/exec_node.h" @@ -108,6 +110,14 @@ Status ExecutionGraph::Init(table_store::schema::Schema* schema, plan::PlanState .OnOTelSink([&](auto& node) { return OnOperatorImpl(node, &descriptors); }) + .OnClickHouseSource([&](auto& node) { + return OnOperatorImpl(node, + &descriptors); + }) + .OnClickHouseExportSink([&](auto& node) { + return OnOperatorImpl(node, + &descriptors); + }) .Walk(pf_); } diff --git a/src/carnot/plan/operators.cc b/src/carnot/plan/operators.cc index bfdb43427f4..d9dfebecfd6 100644 --- a/src/carnot/plan/operators.cc +++ b/src/carnot/plan/operators.cc @@ -83,6 +83,10 @@ std::unique_ptr Operator::FromProto(const planpb::Operator& pb, int64_ return CreateOperator(id, pb.udtf_source_op()); case planpb::EMPTY_SOURCE_OPERATOR: return CreateOperator(id, pb.empty_source_op()); + case planpb::CLICKHOUSE_SOURCE_OPERATOR: + return CreateOperator(id, pb.clickhouse_source_op()); + case planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR: + return CreateOperator(id, pb.clickhouse_sink_op()); case planpb::OTEL_EXPORT_SINK_OPERATOR: return CreateOperator(id, pb.otel_sink_op()); default: @@ -709,6 +713,65 @@ StatusOr EmptySourceOperator::OutputRelation( return r; } +/** + * ClickHouseSourceOperator implementation. + */ + +std::string ClickHouseSourceOperator::DebugString() const { + return absl::Substitute(R"(Op:ClickHouseSource( + host=$0 + port=$1 + username=$2 + batch_size=$3 + start_time=$4 + end_time=$5 + timestamp_column=$6 + partition_column=$7 +)", pb_.host(), pb_.port(), pb_.username(), pb_.batch_size(), pb_.start_time(), pb_.end_time(), + pb_.timestamp_column(), pb_.partition_column()); +} + +Status ClickHouseSourceOperator::Init(const planpb::ClickHouseSourceOperator& pb) { + pb_ = pb; + is_initialized_ = true; + return Status::OK(); +} + +StatusOr ClickHouseSourceOperator::OutputRelation( + const table_store::schema::Schema&, const PlanState&, + const std::vector& input_ids) const { + DCHECK(is_initialized_) << "Not initialized"; + if (!input_ids.empty()) { + return error::InvalidArgument("Source operator cannot have any inputs"); + } + table_store::schema::Relation r; + for (int i = 0; i < pb_.column_types_size(); ++i) { + r.AddColumn(static_cast(pb_.column_types(i)), pb_.column_names(i)); + } + return r; +} + +/** + * ClickHouse Export Sink Operator Implementation. + */ + +Status ClickHouseExportSinkOperator::Init(const planpb::ClickHouseExportSinkOperator& pb) { + pb_ = pb; + is_initialized_ = true; + return Status::OK(); +} + +StatusOr ClickHouseExportSinkOperator::OutputRelation( + const table_store::schema::Schema&, const PlanState&, const std::vector&) const { + DCHECK(is_initialized_) << "Not initialized"; + // There are no outputs. + return table_store::schema::Relation(); +} + +std::string ClickHouseExportSinkOperator::DebugString() const { + return absl::Substitute("Op:ClickHouseExportSink(table=$0)", pb_.table_name()); +} + /** * OTel Export Sink Operator Implementation. */ diff --git a/src/carnot/plan/operators.h b/src/carnot/plan/operators.h index 8586f6eb976..d77b5d6b18c 100644 --- a/src/carnot/plan/operators.h +++ b/src/carnot/plan/operators.h @@ -359,6 +359,69 @@ class EmptySourceOperator : public Operator { std::vector column_idxs_; }; +class ClickHouseSourceOperator : public Operator { + public: + explicit ClickHouseSourceOperator(int64_t id) + : Operator(id, planpb::CLICKHOUSE_SOURCE_OPERATOR) {} + ~ClickHouseSourceOperator() override = default; + + StatusOr OutputRelation( + const table_store::schema::Schema& schema, const PlanState& state, + const std::vector& input_ids) const override; + Status Init(const planpb::ClickHouseSourceOperator& pb); + std::string DebugString() const override; + + std::string host() const { return pb_.host(); } + int32_t port() const { return pb_.port(); } + std::string username() const { return pb_.username(); } + std::string password() const { return pb_.password(); } + std::string database() const { return pb_.database(); } + std::string query() const { return pb_.query(); } + int32_t batch_size() const { return pb_.batch_size(); } + bool streaming() const { return pb_.streaming(); } + std::vector column_names() const { + return std::vector(pb_.column_names().begin(), pb_.column_names().end()); + } + std::vector column_types() const { + std::vector types; + types.reserve(pb_.column_types_size()); + for (const auto& type : pb_.column_types()) { + types.push_back(static_cast(type)); + } + return types; + } + std::string timestamp_column() const { return pb_.timestamp_column(); } + std::string partition_column() const { return pb_.partition_column(); } + int64_t start_time() const { return pb_.start_time(); } + int64_t end_time() const { return pb_.end_time(); } + + private: + planpb::ClickHouseSourceOperator pb_; +}; + +class ClickHouseExportSinkOperator : public Operator { + public: + explicit ClickHouseExportSinkOperator(int64_t id) + : Operator(id, planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR) {} + ~ClickHouseExportSinkOperator() override = default; + + StatusOr OutputRelation( + const table_store::schema::Schema& schema, const PlanState& state, + const std::vector& input_ids) const override; + Status Init(const planpb::ClickHouseExportSinkOperator& pb); + std::string DebugString() const override; + + const planpb::ClickHouseConfig& clickhouse_config() const { return pb_.clickhouse_config(); } + const std::string& table_name() const { return pb_.table_name(); } + const ::google::protobuf::RepeatedPtrField& + column_mappings() const { + return pb_.column_mappings(); + } + + private: + planpb::ClickHouseExportSinkOperator pb_; +}; + class OTelExportSinkOperator : public Operator { public: explicit OTelExportSinkOperator(int64_t id) : Operator(id, planpb::OTEL_EXPORT_SINK_OPERATOR) {} diff --git a/src/carnot/plan/plan_fragment.cc b/src/carnot/plan/plan_fragment.cc index 91d60081347..f9cbc8aa0e7 100644 --- a/src/carnot/plan/plan_fragment.cc +++ b/src/carnot/plan/plan_fragment.cc @@ -98,6 +98,12 @@ Status PlanFragmentWalker::CallWalkFn(const Operator& op) { case planpb::OperatorType::OTEL_EXPORT_SINK_OPERATOR: PX_RETURN_IF_ERROR(CallAs(on_otel_sink_walk_fn_, op)); break; + case planpb::OperatorType::CLICKHOUSE_SOURCE_OPERATOR: + PX_RETURN_IF_ERROR(CallAs(on_clickhouse_source_walk_fn_, op)); + break; + case planpb::OperatorType::CLICKHOUSE_EXPORT_SINK_OPERATOR: + PX_RETURN_IF_ERROR(CallAs(on_clickhouse_export_sink_walk_fn_, op)); + break; default: LOG(FATAL) << absl::Substitute("Operator does not exist: $0", magic_enum::enum_name(op_type)); return error::InvalidArgument("Operator does not exist: $0", magic_enum::enum_name(op_type)); diff --git a/src/carnot/plan/plan_fragment.h b/src/carnot/plan/plan_fragment.h index 39b1cea9ceb..f80090d9c30 100644 --- a/src/carnot/plan/plan_fragment.h +++ b/src/carnot/plan/plan_fragment.h @@ -76,6 +76,8 @@ class PlanFragmentWalker { using UDTFSourceWalkFn = std::function; using EmptySourceWalkFn = std::function; using OTelSinkWalkFn = std::function; + using ClickHouseSourceWalkFn = std::function; + using ClickHouseExportSinkWalkFn = std::function; /** * Register callback for when a memory source operator is encountered. @@ -181,6 +183,17 @@ class PlanFragmentWalker { on_otel_sink_walk_fn_ = fn; return *this; } + + PlanFragmentWalker& OnClickHouseSource(const ClickHouseSourceWalkFn& fn) { + on_clickhouse_source_walk_fn_ = fn; + return *this; + } + + PlanFragmentWalker& OnClickHouseExportSink(const ClickHouseExportSinkWalkFn& fn) { + on_clickhouse_export_sink_walk_fn_ = fn; + return *this; + } + /** * Perform a walk of the plan fragment operators in a topologically-sorted order. * @param plan_fragment The plan fragment to walk. @@ -206,6 +219,8 @@ class PlanFragmentWalker { UDTFSourceWalkFn on_udtf_source_walk_fn_; EmptySourceWalkFn on_empty_source_walk_fn_; OTelSinkWalkFn on_otel_sink_walk_fn_; + ClickHouseSourceWalkFn on_clickhouse_source_walk_fn_; + ClickHouseExportSinkWalkFn on_clickhouse_export_sink_walk_fn_; }; } // namespace plan diff --git a/src/carnot/planner/cgo_export.cc b/src/carnot/planner/cgo_export.cc index cc80e3cc438..211292d251f 100644 --- a/src/carnot/planner/cgo_export.cc +++ b/src/carnot/planner/cgo_export.cc @@ -126,21 +126,21 @@ char* PlannerCompileMutations(PlannerPtr planner_ptr, const char* mutation_reque auto planner = reinterpret_cast(planner_ptr); - auto dynamic_trace_or_s = planner->CompileTrace(mutation_request_pb); - if (!dynamic_trace_or_s.ok()) { - return ExitEarly(dynamic_trace_or_s.status(), resultLen); + auto mutations_ir_or_s = planner->CompileTrace(mutation_request_pb); + if (!mutations_ir_or_s.ok()) { + return ExitEarly(mutations_ir_or_s.status(), resultLen); } - std::unique_ptr trace = - dynamic_trace_or_s.ConsumeValueOrDie(); + std::unique_ptr mutations = + mutations_ir_or_s.ConsumeValueOrDie(); // If the response is ok, then we can go ahead and set this up. CompileMutationsResponse mutations_response_pb; - WrapStatus(&mutations_response_pb, dynamic_trace_or_s.status()); + WrapStatus(&mutations_response_pb, mutations_ir_or_s.status()); PLANNER_RETURN_IF_ERROR(CompileMutationsResponse, resultLen, - trace->ToProto(&mutations_response_pb)); + mutations->ToProto(&mutations_response_pb)); - // Serialize the tracing program into bytes. + // Serialize the mutations into bytes. return PrepareResult(&mutations_response_pb, resultLen); } diff --git a/src/carnot/planner/compiler/graph_comparison.h b/src/carnot/planner/compiler/graph_comparison.h index c6f75b92037..5e0f8a5641c 100644 --- a/src/carnot/planner/compiler/graph_comparison.h +++ b/src/carnot/planner/compiler/graph_comparison.h @@ -261,7 +261,7 @@ struct PlanGraphMatcher { } virtual void DescribeTo(::std::ostream* os) const { - *os << "equals to text probobuf: " << expected_plan_.DebugString(); + *os << "equals to text protobuf: " << expected_plan_.DebugString(); } virtual void DescribeNegationTo(::std::ostream* os) const { diff --git a/src/carnot/planner/compiler_state/compiler_state.h b/src/carnot/planner/compiler_state/compiler_state.h index cd2e7902f0c..c25a14fe64d 100644 --- a/src/carnot/planner/compiler_state/compiler_state.h +++ b/src/carnot/planner/compiler_state/compiler_state.h @@ -119,7 +119,8 @@ class CompilerState : public NotCopyable { int64_t max_output_rows_per_table, std::string_view result_address, std::string_view result_ssl_targetname, const RedactionOptions& redaction_options, std::unique_ptr endpoint_config, - std::unique_ptr plugin_config, DebugInfo debug_info) + std::unique_ptr plugin_config, DebugInfo debug_info, + std::unique_ptr clickhouse_config = nullptr) : relation_map_(std::move(relation_map)), table_names_to_sensitive_columns_(table_names_to_sensitive_columns), registry_info_(registry_info), @@ -130,7 +131,8 @@ class CompilerState : public NotCopyable { redaction_options_(redaction_options), endpoint_config_(std::move(endpoint_config)), plugin_config_(std::move(plugin_config)), - debug_info_(std::move(debug_info)) {} + debug_info_(std::move(debug_info)), + clickhouse_config_(std::move(clickhouse_config)) {} CompilerState() = delete; @@ -175,6 +177,7 @@ class CompilerState : public NotCopyable { planpb::OTelEndpointConfig* endpoint_config() { return endpoint_config_.get(); } PluginConfig* plugin_config() { return plugin_config_.get(); } const DebugInfo& debug_info() { return debug_info_; } + planpb::ClickHouseConfig* clickhouse_config() { return clickhouse_config_.get(); } private: std::unique_ptr relation_map_; @@ -191,6 +194,7 @@ class CompilerState : public NotCopyable { std::unique_ptr endpoint_config_ = nullptr; std::unique_ptr plugin_config_ = nullptr; DebugInfo debug_info_; + std::unique_ptr clickhouse_config_ = nullptr; }; } // namespace planner diff --git a/src/carnot/planner/distributed/splitter/splitter.h b/src/carnot/planner/distributed/splitter/splitter.h index 5ba2a997dc3..42227c1a705 100644 --- a/src/carnot/planner/distributed/splitter/splitter.h +++ b/src/carnot/planner/distributed/splitter/splitter.h @@ -54,7 +54,7 @@ struct BlockingSplitPlan { std::unique_ptr before_blocking; // The plan that occcurs after and including blocking nodes. std::unique_ptr after_blocking; - // The that has both the before and after blocking nodes. + // The plan that has both the before and after blocking nodes. std::unique_ptr original_plan; }; diff --git a/src/carnot/planner/distributedpb/distributed_plan.pb.go b/src/carnot/planner/distributedpb/distributed_plan.pb.go index 64787d1782a..c285696167d 100755 --- a/src/carnot/planner/distributedpb/distributed_plan.pb.go +++ b/src/carnot/planner/distributedpb/distributed_plan.pb.go @@ -581,6 +581,89 @@ func (m *OTelEndpointConfig) GetTimeout() int64 { return 0 } +type ClickHouseConfig struct { + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"` + Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + Username string `protobuf:"bytes,4,opt,name=username,proto3" json:"username,omitempty"` + Password string `protobuf:"bytes,5,opt,name=password,proto3" json:"password,omitempty"` + Database string `protobuf:"bytes,6,opt,name=database,proto3" json:"database,omitempty"` +} + +func (m *ClickHouseConfig) Reset() { *m = ClickHouseConfig{} } +func (*ClickHouseConfig) ProtoMessage() {} +func (*ClickHouseConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_30dce4250507a2af, []int{8} +} +func (m *ClickHouseConfig) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClickHouseConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClickHouseConfig.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClickHouseConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClickHouseConfig.Merge(m, src) +} +func (m *ClickHouseConfig) XXX_Size() int { + return m.Size() +} +func (m *ClickHouseConfig) XXX_DiscardUnknown() { + xxx_messageInfo_ClickHouseConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_ClickHouseConfig proto.InternalMessageInfo + +func (m *ClickHouseConfig) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *ClickHouseConfig) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *ClickHouseConfig) GetPort() int32 { + if m != nil { + return m.Port + } + return 0 +} + +func (m *ClickHouseConfig) GetUsername() string { + if m != nil { + return m.Username + } + return "" +} + +func (m *ClickHouseConfig) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + +func (m *ClickHouseConfig) GetDatabase() string { + if m != nil { + return m.Database + } + return "" +} + type PluginConfig struct { StartTimeNs int64 `protobuf:"varint,1,opt,name=start_time_ns,json=startTimeNs,proto3" json:"start_time_ns,omitempty"` EndTimeNs int64 `protobuf:"varint,2,opt,name=end_time_ns,json=endTimeNs,proto3" json:"end_time_ns,omitempty"` @@ -589,7 +672,7 @@ type PluginConfig struct { func (m *PluginConfig) Reset() { *m = PluginConfig{} } func (*PluginConfig) ProtoMessage() {} func (*PluginConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_30dce4250507a2af, []int{8} + return fileDescriptor_30dce4250507a2af, []int{9} } func (m *PluginConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -639,7 +722,7 @@ type DebugInfo struct { func (m *DebugInfo) Reset() { *m = DebugInfo{} } func (*DebugInfo) ProtoMessage() {} func (*DebugInfo) Descriptor() ([]byte, []int) { - return fileDescriptor_30dce4250507a2af, []int{9} + return fileDescriptor_30dce4250507a2af, []int{10} } func (m *DebugInfo) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -683,7 +766,7 @@ type DebugInfo_OTelDebugAttribute struct { func (m *DebugInfo_OTelDebugAttribute) Reset() { *m = DebugInfo_OTelDebugAttribute{} } func (*DebugInfo_OTelDebugAttribute) ProtoMessage() {} func (*DebugInfo_OTelDebugAttribute) Descriptor() ([]byte, []int) { - return fileDescriptor_30dce4250507a2af, []int{9, 0} + return fileDescriptor_30dce4250507a2af, []int{10, 0} } func (m *DebugInfo_OTelDebugAttribute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -734,13 +817,14 @@ type LogicalPlannerState struct { RedactionOptions *RedactionOptions `protobuf:"bytes,7,opt,name=redaction_options,json=redactionOptions,proto3" json:"redaction_options,omitempty"` OTelEndpointConfig *OTelEndpointConfig `protobuf:"bytes,8,opt,name=otel_endpoint_config,json=otelEndpointConfig,proto3" json:"otel_endpoint_config,omitempty"` PluginConfig *PluginConfig `protobuf:"bytes,9,opt,name=plugin_config,json=pluginConfig,proto3" json:"plugin_config,omitempty"` + ClickhouseConfig *ClickHouseConfig `protobuf:"bytes,11,opt,name=clickhouse_config,json=clickhouseConfig,proto3" json:"clickhouse_config,omitempty"` DebugInfo *DebugInfo `protobuf:"bytes,10,opt,name=debug_info,json=debugInfo,proto3" json:"debug_info,omitempty"` } func (m *LogicalPlannerState) Reset() { *m = LogicalPlannerState{} } func (*LogicalPlannerState) ProtoMessage() {} func (*LogicalPlannerState) Descriptor() ([]byte, []int) { - return fileDescriptor_30dce4250507a2af, []int{10} + return fileDescriptor_30dce4250507a2af, []int{11} } func (m *LogicalPlannerState) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -818,6 +902,13 @@ func (m *LogicalPlannerState) GetPluginConfig() *PluginConfig { return nil } +func (m *LogicalPlannerState) GetClickhouseConfig() *ClickHouseConfig { + if m != nil { + return m.ClickhouseConfig + } + return nil +} + func (m *LogicalPlannerState) GetDebugInfo() *DebugInfo { if m != nil { return m.DebugInfo @@ -833,7 +924,7 @@ type LogicalPlannerResult struct { func (m *LogicalPlannerResult) Reset() { *m = LogicalPlannerResult{} } func (*LogicalPlannerResult) ProtoMessage() {} func (*LogicalPlannerResult) Descriptor() ([]byte, []int) { - return fileDescriptor_30dce4250507a2af, []int{11} + return fileDescriptor_30dce4250507a2af, []int{12} } func (m *LogicalPlannerResult) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -888,6 +979,7 @@ func init() { proto.RegisterType((*RedactionOptions)(nil), "px.carnot.planner.distributedpb.RedactionOptions") proto.RegisterType((*OTelEndpointConfig)(nil), "px.carnot.planner.distributedpb.OTelEndpointConfig") proto.RegisterMapType((map[string]string)(nil), "px.carnot.planner.distributedpb.OTelEndpointConfig.HeadersEntry") + proto.RegisterType((*ClickHouseConfig)(nil), "px.carnot.planner.distributedpb.ClickHouseConfig") proto.RegisterType((*PluginConfig)(nil), "px.carnot.planner.distributedpb.PluginConfig") proto.RegisterType((*DebugInfo)(nil), "px.carnot.planner.distributedpb.DebugInfo") proto.RegisterType((*DebugInfo_OTelDebugAttribute)(nil), "px.carnot.planner.distributedpb.DebugInfo.OTelDebugAttribute") @@ -900,104 +992,111 @@ func init() { } var fileDescriptor_30dce4250507a2af = []byte{ - // 1549 bytes of a gzipped FileDescriptorProto + // 1651 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x57, 0x4f, 0x6f, 0x1b, 0xc7, - 0x15, 0xd7, 0x8a, 0x94, 0x48, 0x3e, 0x92, 0x12, 0x3d, 0xa2, 0x5c, 0x96, 0x48, 0x48, 0x97, 0x48, - 0x50, 0xc1, 0x76, 0x97, 0xa9, 0x12, 0x34, 0x69, 0x80, 0xb4, 0x11, 0x45, 0xc9, 0x62, 0xac, 0x26, - 0xea, 0x50, 0x06, 0x02, 0x1f, 0xba, 0x18, 0x72, 0x87, 0xe4, 0x22, 0xcb, 0xdd, 0xd5, 0xce, 0xac, - 0x21, 0xb5, 0x28, 0xd0, 0x1e, 0x7b, 0x6a, 0x3f, 0x46, 0x4f, 0xbd, 0xf5, 0xda, 0x6b, 0x7b, 0xf4, - 0x31, 0x27, 0x21, 0xa6, 0x2f, 0x3d, 0xe6, 0x0b, 0x14, 0x28, 0xe6, 0xcd, 0x2e, 0xb5, 0xa4, 0x09, - 0x48, 0x6e, 0x2f, 0xe4, 0xcc, 0x7b, 0xbf, 0xf7, 0x67, 0xe6, 0xbd, 0xdf, 0xcc, 0x2c, 0x7c, 0x2c, - 0xc2, 0x61, 0x7b, 0xc8, 0x42, 0xcf, 0x97, 0xed, 0xc0, 0x65, 0x9e, 0xc7, 0xc3, 0xb6, 0xed, 0x08, - 0x19, 0x3a, 0x83, 0x48, 0x72, 0x3b, 0x18, 0xa4, 0x67, 0x96, 0x42, 0x98, 0x41, 0xe8, 0x4b, 0x9f, - 0x34, 0x83, 0x4b, 0x53, 0xdb, 0x99, 0xb1, 0x9d, 0xb9, 0x60, 0x57, 0xaf, 0x8e, 0xfd, 0xb1, 0x8f, - 0xd8, 0xb6, 0x1a, 0x69, 0xb3, 0x7a, 0x53, 0xc5, 0x63, 0x81, 0xd3, 0xd6, 0x9a, 0x28, 0x72, 0x54, - 0x0c, 0xf5, 0x17, 0x03, 0xde, 0x59, 0x4a, 0x28, 0x18, 0xb4, 0x6f, 0xa2, 0xd6, 0xdf, 0x47, 0xad, - 0x3f, 0x9d, 0xfa, 0x5e, 0x7b, 0xc0, 0x04, 0x6f, 0x0b, 0xc9, 0x64, 0x24, 0x82, 0x41, 0x3c, 0x88, - 0x61, 0x0f, 0x15, 0x4c, 0x4c, 0x58, 0xc8, 0xed, 0xf6, 0xc0, 0xf5, 0xfd, 0xe9, 0xc8, 0x71, 0x25, - 0x0f, 0x83, 0x41, 0x7a, 0x16, 0x63, 0xdf, 0x4b, 0x61, 0xa7, 0x5c, 0x32, 0x9b, 0x49, 0x16, 0x0c, - 0xe6, 0xc3, 0x74, 0x60, 0xc9, 0x06, 0x2e, 0xb7, 0x84, 0xf4, 0x43, 0xde, 0x16, 0xc3, 0x09, 0x9f, - 0x2a, 0xa0, 0x1e, 0x68, 0x58, 0x6b, 0x66, 0x40, 0xe9, 0x57, 0xb1, 0x65, 0xcf, 0x1b, 0xf9, 0xe4, - 0x29, 0x6c, 0x27, 0x9e, 0xac, 0x91, 0xc3, 0x5d, 0x5b, 0xd4, 0x8c, 0x07, 0x99, 0xbd, 0xad, 0xfd, - 0x96, 0x19, 0x5c, 0x9a, 0x3a, 0xac, 0x79, 0x13, 0xd6, 0x4c, 0x8c, 0xcf, 0xaf, 0x02, 0x4e, 0xb7, - 0x12, 0xc5, 0x31, 0x5a, 0x92, 0xdf, 0xc1, 0xee, 0xe5, 0xe5, 0x84, 0x89, 0xc9, 0xcf, 0x3e, 0xb2, - 0x70, 0x21, 0x96, 0x5e, 0x49, 0x6d, 0xfd, 0x81, 0xb1, 0x57, 0xdc, 0x7f, 0x9c, 0x72, 0xb9, 0xb0, - 0x6a, 0xf3, 0xeb, 0xaf, 0x4f, 0xd0, 0xaa, 0xa3, 0xa4, 0xc7, 0x28, 0xed, 0xfc, 0x60, 0x76, 0xdd, - 0xdc, 0x59, 0xa1, 0x38, 0x59, 0xa3, 0x3b, 0x49, 0x94, 0x34, 0x3e, 0x0f, 0x9b, 0xda, 0x5f, 0xeb, - 0xbb, 0x2c, 0xc0, 0x21, 0x56, 0x08, 0x97, 0xf8, 0x01, 0x54, 0x2f, 0x22, 0x1e, 0x5e, 0x59, 0x83, - 0xd0, 0xff, 0x86, 0x87, 0x16, 0xb3, 0xed, 0x90, 0x0b, 0xb5, 0x4e, 0x63, 0xaf, 0x40, 0x09, 0xea, - 0x3a, 0xa8, 0x3a, 0xd0, 0x1a, 0xf2, 0x31, 0xe4, 0xd9, 0x98, 0x7b, 0xd2, 0x72, 0xec, 0x1a, 0x60, - 0xea, 0xdb, 0x2a, 0x75, 0xdd, 0x0c, 0xe6, 0xb3, 0x67, 0xbd, 0x6e, 0xa7, 0x38, 0xbb, 0x6e, 0xe6, - 0x0e, 0x14, 0xa8, 0xd7, 0xa5, 0x39, 0x44, 0xf7, 0x6c, 0xf2, 0x73, 0xd8, 0x9e, 0x30, 0x61, 0x8d, - 0xc3, 0x60, 0x68, 0x09, 0x1e, 0xbe, 0x88, 0x97, 0x9e, 0xef, 0xdc, 0x9b, 0x5d, 0x37, 0xcb, 0x27, - 0x4c, 0x3c, 0xa1, 0x67, 0x87, 0x7d, 0x54, 0xd0, 0xf2, 0x84, 0x89, 0x27, 0x61, 0x30, 0xd4, 0x53, - 0xb2, 0x0f, 0x25, 0x34, 0x4b, 0xb2, 0xcb, 0xa8, 0xec, 0x3a, 0xdb, 0xb3, 0xeb, 0x66, 0x51, 0x19, - 0xc5, 0xa9, 0xd1, 0xa2, 0x02, 0x25, 0x79, 0xbe, 0x07, 0x5b, 0x2a, 0x1c, 0x16, 0x0f, 0xab, 0x5e, - 0xcb, 0xaa, 0x68, 0xb4, 0x34, 0x61, 0xa2, 0xcb, 0x24, 0xeb, 0x2b, 0x19, 0x79, 0x1f, 0xb6, 0x82, - 0xd0, 0x1f, 0x72, 0x21, 0xb8, 0xc6, 0xd6, 0x36, 0x10, 0x55, 0x9e, 0x4b, 0x15, 0x96, 0x7c, 0x04, - 0xf7, 0xd9, 0x70, 0xc8, 0x03, 0x29, 0xac, 0x90, 0x4f, 0x7d, 0xc9, 0x2d, 0xe1, 0x47, 0xe1, 0x90, - 0x8b, 0xda, 0x26, 0xc2, 0xab, 0xb1, 0x96, 0xa2, 0xb2, 0xaf, 0x75, 0xa4, 0x07, 0xa0, 0xbb, 0xce, - 0xf1, 0x46, 0x7e, 0x2d, 0xf7, 0x20, 0xb3, 0x57, 0xdc, 0x7f, 0x68, 0xde, 0xc2, 0x3d, 0xf3, 0x5c, - 0x99, 0xa8, 0xe2, 0xd0, 0x82, 0x4c, 0x86, 0xe4, 0x1d, 0xc8, 0x32, 0xe1, 0xd8, 0xb5, 0xfc, 0x03, - 0x63, 0xaf, 0xdc, 0xc9, 0xcf, 0xae, 0x9b, 0xd9, 0x83, 0x7e, 0xaf, 0x4b, 0x51, 0x4a, 0x28, 0x94, - 0xe7, 0x8d, 0x8a, 0xb1, 0x0a, 0x58, 0x98, 0x9f, 0xdc, 0x1a, 0x2b, 0xdd, 0xee, 0xb4, 0x34, 0x4d, - 0x37, 0xff, 0x27, 0xb0, 0x25, 0x84, 0x6b, 0x49, 0x16, 0x8e, 0xb9, 0xf4, 0xd8, 0x94, 0xd7, 0x8a, - 0xb8, 0xeb, 0x58, 0xad, 0x7e, 0xff, 0xf4, 0x1c, 0x15, 0x5f, 0xb2, 0x29, 0xa7, 0x65, 0x21, 0xdc, - 0xf3, 0x39, 0xae, 0x35, 0x81, 0xc2, 0x7c, 0x0d, 0xa4, 0x0a, 0x1b, 0xb8, 0x8a, 0xb8, 0xa3, 0xf4, - 0x84, 0x3c, 0x82, 0x7b, 0x38, 0x90, 0xce, 0x6f, 0x99, 0x74, 0x7c, 0xcf, 0xfa, 0x86, 0x5f, 0x61, - 0x37, 0x14, 0x68, 0x65, 0x41, 0xf1, 0x94, 0x5f, 0x91, 0x1a, 0xe4, 0xb4, 0x4c, 0x15, 0x3e, 0xb3, - 0x57, 0xa0, 0xc9, 0xb4, 0xf5, 0x67, 0x03, 0xa0, 0x8f, 0x14, 0xc6, 0x58, 0x04, 0xb2, 0x98, 0xa8, - 0x0e, 0x85, 0x63, 0xf2, 0x19, 0xe4, 0x43, 0xee, 0xa2, 0xaf, 0x98, 0x69, 0x3f, 0x52, 0xbb, 0x92, - 0x3a, 0x0d, 0xcc, 0xe4, 0x34, 0x30, 0x69, 0x0c, 0xa4, 0x73, 0x13, 0x62, 0x02, 0xe8, 0x6e, 0x77, - 0x1d, 0x21, 0x31, 0xfc, 0x9b, 0xfd, 0x4e, 0x0b, 0x08, 0x39, 0x75, 0x84, 0x6c, 0xfd, 0xcd, 0x80, - 0x4a, 0xf7, 0x66, 0x8b, 0xfb, 0x92, 0x49, 0x4e, 0x4e, 0xa1, 0xa8, 0xab, 0xa0, 0x8b, 0x63, 0xa0, - 0x97, 0x47, 0xb7, 0x16, 0xe7, 0x86, 0xa6, 0x14, 0x86, 0x37, 0x94, 0x3d, 0x85, 0xa2, 0xce, 0x58, - 0x7b, 0x5b, 0xbf, 0xa3, 0xb7, 0x9b, 0x7d, 0xa2, 0x20, 0xe6, 0xe3, 0xd6, 0x3f, 0x33, 0xb0, 0x9d, - 0x4a, 0xf8, 0xcc, 0x65, 0x1e, 0x09, 0x81, 0x5c, 0x0c, 0x12, 0xb2, 0x59, 0xd2, 0xc7, 0xab, 0x23, - 0x4e, 0xfb, 0xe8, 0xd6, 0x40, 0x4b, 0xde, 0xcc, 0x5f, 0x0f, 0x62, 0x4a, 0x9e, 0xfb, 0x6a, 0x7e, - 0xe4, 0xc9, 0xf0, 0x8a, 0x6e, 0x5f, 0x2c, 0x4a, 0xc9, 0x0b, 0xa8, 0x2e, 0xc6, 0xb4, 0xd9, 0x58, - 0x1d, 0x31, 0x7a, 0x79, 0xc7, 0xff, 0x4f, 0xd4, 0x2e, 0x1b, 0xf7, 0x6c, 0x1d, 0xb6, 0x72, 0xb1, - 0x24, 0x26, 0x3f, 0x86, 0x8c, 0xcd, 0xc6, 0x78, 0xa2, 0x14, 0xf7, 0x77, 0x97, 0xc2, 0x28, 0xbf, - 0x07, 0x4f, 0xa8, 0x42, 0xd4, 0x9f, 0x43, 0x75, 0xd5, 0x4a, 0x48, 0x05, 0x32, 0xaa, 0x79, 0x75, - 0xcf, 0xa9, 0x21, 0x79, 0x0c, 0x1b, 0x2f, 0x98, 0x1b, 0xf1, 0xb8, 0xdf, 0xee, 0xbf, 0xe9, 0x54, - 0x59, 0x53, 0x0d, 0xfa, 0x74, 0xfd, 0x13, 0xa3, 0x7e, 0x08, 0xbb, 0x2b, 0xf3, 0x5d, 0xe1, 0xbc, - 0x9a, 0x76, 0x9e, 0x4d, 0x39, 0x69, 0xfd, 0xd1, 0x80, 0x0a, 0xe5, 0x36, 0x1b, 0xaa, 0xc6, 0xfd, - 0x2a, 0x50, 0xbf, 0x82, 0x3c, 0x06, 0x12, 0x09, 0x6e, 0x8d, 0x22, 0xd7, 0xb5, 0xc2, 0x44, 0x89, - 0xfe, 0xf2, 0xb4, 0x12, 0x09, 0x7e, 0x1c, 0xb9, 0xee, 0xdc, 0x88, 0xfc, 0x12, 0xde, 0x55, 0xe8, - 0xe0, 0x32, 0xc6, 0x5a, 0x81, 0xe3, 0x58, 0x03, 0x2e, 0xa4, 0xc5, 0x47, 0x23, 0x3f, 0x94, 0xfa, - 0xc0, 0xa6, 0xb5, 0x48, 0xf0, 0xb3, 0x4b, 0x6d, 0x76, 0xe6, 0x38, 0x1d, 0x2e, 0xe4, 0x11, 0xea, - 0x5b, 0xff, 0x31, 0x80, 0x7c, 0x75, 0xce, 0xdd, 0x23, 0xcf, 0x0e, 0x7c, 0xc7, 0x93, 0x87, 0xbe, - 0x37, 0x72, 0xc6, 0xe4, 0x87, 0x90, 0x89, 0x42, 0x57, 0x2f, 0xa3, 0x93, 0x9b, 0x5d, 0x37, 0x33, - 0xcf, 0xe8, 0x29, 0x55, 0x32, 0xf2, 0x1c, 0x72, 0x13, 0xce, 0x6c, 0x1e, 0x8a, 0xb8, 0xd4, 0x9f, - 0xdf, 0x5a, 0xea, 0x37, 0x03, 0x98, 0x27, 0xda, 0x85, 0x2e, 0x72, 0xe2, 0x90, 0xd4, 0x21, 0xef, - 0x78, 0x82, 0x0f, 0xa3, 0x90, 0x63, 0x81, 0xf3, 0x74, 0x3e, 0xc7, 0x43, 0xc5, 0x99, 0x72, 0x3f, - 0x92, 0x78, 0x2f, 0x64, 0x68, 0x32, 0xad, 0x7f, 0x0a, 0xa5, 0xb4, 0xbb, 0xdb, 0x6a, 0x50, 0x48, - 0xd7, 0x80, 0x42, 0xe9, 0xcc, 0x8d, 0xc6, 0x8e, 0x17, 0x2f, 0xbc, 0x05, 0x65, 0x21, 0x59, 0x28, - 0x2d, 0xe5, 0xdc, 0xf2, 0xf4, 0xbd, 0x9a, 0xa1, 0x45, 0x14, 0x9e, 0x3b, 0x53, 0xfe, 0xa5, 0x20, - 0x0d, 0x28, 0x72, 0xcf, 0x9e, 0x23, 0xd6, 0x11, 0x51, 0xe0, 0x9e, 0xad, 0xf5, 0xad, 0x7f, 0x18, - 0x50, 0xe8, 0xf2, 0x41, 0x34, 0x46, 0xf6, 0x5f, 0xc0, 0xae, 0x2f, 0xb9, 0x6b, 0xd9, 0x4a, 0x62, - 0x31, 0x19, 0xef, 0x8b, 0x88, 0xe9, 0xf9, 0xd9, 0xed, 0x44, 0x49, 0x5c, 0xe1, 0x3e, 0xe2, 0xec, - 0x20, 0xf1, 0x42, 0x77, 0x94, 0xef, 0x45, 0x99, 0xa8, 0xff, 0x42, 0xd7, 0x74, 0x51, 0xbc, 0xf2, - 0xb0, 0x5d, 0xb9, 0x31, 0xad, 0xbf, 0x6f, 0xc0, 0xce, 0xa9, 0x3f, 0x76, 0x86, 0xcc, 0x3d, 0xd3, - 0x29, 0xe9, 0x63, 0xf1, 0x37, 0x70, 0x2f, 0xfd, 0x3e, 0x55, 0x8f, 0xc0, 0x84, 0x33, 0x3f, 0x7d, - 0x1b, 0xbe, 0xa3, 0x37, 0x5a, 0xb1, 0x97, 0x8f, 0xdd, 0xcf, 0xa1, 0xa4, 0x6c, 0x2d, 0x5f, 0x73, - 0x21, 0xe6, 0xf8, 0xbb, 0xab, 0xe9, 0x18, 0x13, 0x86, 0x16, 0x83, 0x9b, 0x89, 0x7a, 0x1d, 0x84, - 0x5c, 0x44, 0xae, 0x9c, 0xbf, 0x3c, 0xb2, 0xb8, 0xb0, 0xb2, 0x96, 0x26, 0x4f, 0x8d, 0xa7, 0xb0, - 0x1b, 0xc3, 0x96, 0x6e, 0xcc, 0x0d, 0x6c, 0x78, 0x7c, 0xac, 0x51, 0x04, 0x2c, 0xde, 0x9b, 0x3b, - 0xda, 0xaa, 0x9f, 0xbe, 0x3d, 0xd5, 0xae, 0xcc, 0x89, 0x3a, 0x4f, 0x3d, 0x77, 0xc7, 0x5d, 0x59, - 0xe6, 0x3f, 0xad, 0x84, 0xcb, 0x27, 0xc2, 0xef, 0xa1, 0x8a, 0x0d, 0xc4, 0x63, 0x06, 0x59, 0x43, - 0x6c, 0x55, 0x7c, 0x59, 0x14, 0xf7, 0x3f, 0xfc, 0x1f, 0xd8, 0xd7, 0xb9, 0x3f, 0xbb, 0x6e, 0xae, - 0xa0, 0x3d, 0x25, 0x2a, 0xd0, 0xd2, 0x51, 0x40, 0xa1, 0x1c, 0x20, 0x43, 0x92, 0xb8, 0x77, 0x7d, - 0xaa, 0xa4, 0x79, 0x45, 0x4b, 0x41, 0x9a, 0x65, 0x3d, 0x00, 0x4d, 0x07, 0xbc, 0x10, 0xf5, 0xa3, - 0xf4, 0xe1, 0xdd, 0x89, 0x40, 0x0b, 0x76, 0x32, 0xfc, 0x22, 0x9b, 0x37, 0x2a, 0xeb, 0x5f, 0x64, - 0xf3, 0x9b, 0x95, 0x5c, 0xeb, 0x4f, 0x06, 0x54, 0x17, 0xfb, 0x56, 0x17, 0x91, 0x3c, 0x82, 0x4d, - 0xfd, 0xc5, 0x82, 0xcd, 0x5f, 0xdc, 0xdf, 0xc1, 0xb7, 0x7b, 0xfc, 0x31, 0x63, 0xf6, 0x71, 0x40, - 0x63, 0x08, 0xe9, 0x42, 0x16, 0xaf, 0x4f, 0xdd, 0xd8, 0x1f, 0xbc, 0xed, 0x45, 0x46, 0xd1, 0xba, - 0x73, 0xf8, 0xf2, 0x55, 0x63, 0xed, 0xdb, 0x57, 0x8d, 0xb5, 0xef, 0x5f, 0x35, 0x8c, 0x3f, 0xcc, - 0x1a, 0xc6, 0x5f, 0x67, 0x0d, 0xe3, 0x5f, 0xb3, 0x86, 0xf1, 0x72, 0xd6, 0x30, 0xbe, 0x9b, 0x35, - 0x8c, 0x7f, 0xcf, 0x1a, 0x6b, 0xdf, 0xcf, 0x1a, 0xc6, 0x5f, 0x5e, 0x37, 0xd6, 0x5e, 0xbe, 0x6e, - 0xac, 0x7d, 0xfb, 0xba, 0xb1, 0xf6, 0xbc, 0xbc, 0xe0, 0x7a, 0xb0, 0x89, 0xdf, 0x39, 0x1f, 0xfe, - 0x37, 0x00, 0x00, 0xff, 0xff, 0x01, 0xc0, 0xd4, 0xed, 0x38, 0x0e, 0x00, 0x00, + 0x15, 0xd7, 0x8a, 0xb4, 0x44, 0x3e, 0x8a, 0x12, 0x3d, 0xa2, 0x5c, 0x96, 0x48, 0x48, 0x97, 0x48, + 0x50, 0xc1, 0x76, 0x97, 0xa9, 0x12, 0x34, 0x69, 0x80, 0xb4, 0x11, 0x25, 0xdb, 0x52, 0xac, 0x26, + 0xea, 0x50, 0x06, 0x02, 0x1f, 0xb2, 0x18, 0x72, 0x47, 0xe4, 0xc2, 0xcb, 0xdd, 0xd5, 0xcc, 0xac, + 0x2b, 0xb5, 0x28, 0xd0, 0x1e, 0x7b, 0x6a, 0x2f, 0xfd, 0x0e, 0x45, 0x0f, 0xfd, 0x08, 0xbd, 0xb6, + 0x47, 0x1f, 0x73, 0x12, 0x62, 0xfa, 0xd2, 0x63, 0xbe, 0x40, 0x81, 0x62, 0xde, 0xec, 0xae, 0x96, + 0x34, 0x01, 0x29, 0xcd, 0x45, 0x9a, 0x79, 0xef, 0xf7, 0x7e, 0xef, 0xcd, 0xbe, 0x3f, 0x33, 0x84, + 0x0f, 0xa5, 0x18, 0x76, 0x87, 0x4c, 0x04, 0xa1, 0xea, 0x46, 0x3e, 0x0b, 0x02, 0x2e, 0xba, 0xae, + 0x27, 0x95, 0xf0, 0x06, 0xb1, 0xe2, 0x6e, 0x34, 0xc8, 0xef, 0x1c, 0x8d, 0xb0, 0x23, 0x11, 0xaa, + 0x90, 0xb4, 0xa3, 0x73, 0xdb, 0xd8, 0xd9, 0x89, 0x9d, 0x3d, 0x63, 0xd7, 0xac, 0x8f, 0xc2, 0x51, + 0x88, 0xd8, 0xae, 0x5e, 0x19, 0xb3, 0x66, 0x5b, 0xfb, 0x63, 0x91, 0xd7, 0x35, 0x9a, 0x38, 0xf6, + 0xb4, 0x0f, 0xfd, 0x2f, 0x01, 0xbc, 0x35, 0x17, 0x50, 0x34, 0xe8, 0x5e, 0x79, 0x6d, 0xbe, 0x8b, + 0xda, 0x70, 0x32, 0x09, 0x83, 0xee, 0x80, 0x49, 0xde, 0x95, 0x8a, 0xa9, 0x58, 0x46, 0x83, 0x64, + 0x91, 0xc0, 0xee, 0x69, 0x98, 0x1c, 0x33, 0xc1, 0xdd, 0xee, 0xc0, 0x0f, 0xc3, 0xc9, 0xa9, 0xe7, + 0x2b, 0x2e, 0xa2, 0x41, 0x7e, 0x97, 0x60, 0xdf, 0xc9, 0x61, 0x27, 0x5c, 0x31, 0x97, 0x29, 0x16, + 0x0d, 0xb2, 0x65, 0xde, 0xb1, 0x62, 0x03, 0x9f, 0x3b, 0x52, 0x85, 0x82, 0x77, 0xe5, 0x70, 0xcc, + 0x27, 0x1a, 0x68, 0x16, 0x06, 0xd6, 0x99, 0x5a, 0xb0, 0xf6, 0xab, 0xc4, 0xf2, 0x30, 0x38, 0x0d, + 0xc9, 0x13, 0xd8, 0x48, 0x99, 0x9c, 0x53, 0x8f, 0xfb, 0xae, 0x6c, 0x58, 0x77, 0x0b, 0xdb, 0xeb, + 0x3b, 0x1d, 0x3b, 0x3a, 0xb7, 0x8d, 0x5b, 0xfb, 0xca, 0xad, 0x9d, 0x1a, 0x9f, 0x5c, 0x44, 0x9c, + 0xae, 0xa7, 0x8a, 0x47, 0x68, 0x49, 0x7e, 0x07, 0x5b, 0xe7, 0xe7, 0x63, 0x26, 0xc7, 0x3f, 0xfb, + 0xc0, 0xc1, 0x83, 0x38, 0xe6, 0x24, 0x8d, 0xe5, 0xbb, 0xd6, 0x76, 0x65, 0xe7, 0x41, 0x8e, 0x72, + 0xe6, 0xd4, 0xf6, 0x97, 0x5f, 0x1e, 0xa0, 0x55, 0x4f, 0x4b, 0x1f, 0xa1, 0xb4, 0xf7, 0x83, 0xe9, + 0x65, 0x7b, 0x73, 0x81, 0xe2, 0x60, 0x89, 0x6e, 0xa6, 0x5e, 0xf2, 0xf8, 0x12, 0xac, 0x18, 0xbe, + 0xce, 0x37, 0x45, 0x80, 0x3d, 0xcc, 0x10, 0x1e, 0xf1, 0x3d, 0xa8, 0x9f, 0xc5, 0x5c, 0x5c, 0x38, + 0x03, 0x11, 0x3e, 0xe7, 0xc2, 0x61, 0xae, 0x2b, 0xb8, 0xd4, 0xe7, 0xb4, 0xb6, 0xcb, 0x94, 0xa0, + 0xae, 0x87, 0xaa, 0x5d, 0xa3, 0x21, 0x1f, 0x42, 0x89, 0x8d, 0x78, 0xa0, 0x1c, 0xcf, 0x6d, 0x00, + 0x86, 0xbe, 0xa1, 0x43, 0x37, 0xc5, 0x60, 0x3f, 0x7d, 0x7a, 0xb8, 0xdf, 0xab, 0x4c, 0x2f, 0xdb, + 0xab, 0xbb, 0x1a, 0x74, 0xb8, 0x4f, 0x57, 0x11, 0x7d, 0xe8, 0x92, 0x9f, 0xc3, 0xc6, 0x98, 0x49, + 0x67, 0x24, 0xa2, 0xa1, 0x23, 0xb9, 0x78, 0x91, 0x1c, 0xbd, 0xd4, 0xbb, 0x3d, 0xbd, 0x6c, 0x57, + 0x0f, 0x98, 0x7c, 0x4c, 0x8f, 0xf7, 0xfa, 0xa8, 0xa0, 0xd5, 0x31, 0x93, 0x8f, 0x45, 0x34, 0x34, + 0x5b, 0xb2, 0x03, 0x6b, 0x68, 0x96, 0x46, 0x57, 0xd0, 0xd1, 0xf5, 0x36, 0xa6, 0x97, 0xed, 0x8a, + 0x36, 0x4a, 0x42, 0xa3, 0x15, 0x0d, 0x4a, 0xe3, 0x7c, 0x07, 0xd6, 0xb5, 0x3b, 0x4c, 0x1e, 0x66, + 0xbd, 0x51, 0xd4, 0xde, 0xe8, 0xda, 0x98, 0xc9, 0x7d, 0xa6, 0x58, 0x5f, 0xcb, 0xc8, 0xbb, 0xb0, + 0x1e, 0x89, 0x70, 0xc8, 0xa5, 0xe4, 0x06, 0xdb, 0xb8, 0x85, 0xa8, 0x6a, 0x26, 0xd5, 0x58, 0xf2, + 0x01, 0xdc, 0x61, 0xc3, 0x21, 0x8f, 0x94, 0x74, 0x04, 0x9f, 0x84, 0x8a, 0x3b, 0x32, 0x8c, 0xc5, + 0x90, 0xcb, 0xc6, 0x0a, 0xc2, 0xeb, 0x89, 0x96, 0xa2, 0xb2, 0x6f, 0x74, 0xe4, 0x10, 0xc0, 0x54, + 0x9d, 0x17, 0x9c, 0x86, 0x8d, 0xd5, 0xbb, 0x85, 0xed, 0xca, 0xce, 0x3d, 0xfb, 0x9a, 0xde, 0xb3, + 0x4f, 0xb4, 0x89, 0x4e, 0x0e, 0x2d, 0xab, 0x74, 0x49, 0xde, 0x82, 0x22, 0x93, 0x9e, 0xdb, 0x28, + 0xdd, 0xb5, 0xb6, 0xab, 0xbd, 0xd2, 0xf4, 0xb2, 0x5d, 0xdc, 0xed, 0x1f, 0xee, 0x53, 0x94, 0x12, + 0x0a, 0xd5, 0xac, 0x50, 0xd1, 0x57, 0x19, 0x13, 0xf3, 0x93, 0x6b, 0x7d, 0xe5, 0xcb, 0x9d, 0xae, + 0x4d, 0xf2, 0xc5, 0xff, 0x11, 0xac, 0x4b, 0xe9, 0x3b, 0x8a, 0x89, 0x11, 0x57, 0x01, 0x9b, 0xf0, + 0x46, 0x05, 0xbf, 0x3a, 0x66, 0xab, 0xdf, 0x3f, 0x3a, 0x41, 0xc5, 0xe7, 0x6c, 0xc2, 0x69, 0x55, + 0x4a, 0xff, 0x24, 0xc3, 0x75, 0xc6, 0x50, 0xce, 0xce, 0x40, 0xea, 0x70, 0x0b, 0x4f, 0x91, 0x54, + 0x94, 0xd9, 0x90, 0xfb, 0x70, 0x1b, 0x17, 0xca, 0xfb, 0x2d, 0x53, 0x5e, 0x18, 0x38, 0xcf, 0xf9, + 0x05, 0x56, 0x43, 0x99, 0xd6, 0x66, 0x14, 0x4f, 0xf8, 0x05, 0x69, 0xc0, 0xaa, 0x91, 0xe9, 0xc4, + 0x17, 0xb6, 0xcb, 0x34, 0xdd, 0x76, 0xfe, 0x6c, 0x01, 0xf4, 0xb1, 0x85, 0xd1, 0x17, 0x81, 0x22, + 0x06, 0x6a, 0x5c, 0xe1, 0x9a, 0x7c, 0x02, 0x25, 0xc1, 0x7d, 0xe4, 0x4a, 0x3a, 0xed, 0x47, 0xfa, + 0xab, 0xe4, 0xa6, 0x81, 0x9d, 0x4e, 0x03, 0x9b, 0x26, 0x40, 0x9a, 0x99, 0x10, 0x1b, 0xc0, 0x54, + 0xbb, 0xef, 0x49, 0x85, 0xee, 0xdf, 0xac, 0x77, 0x5a, 0x46, 0xc8, 0x91, 0x27, 0x55, 0xe7, 0x1f, + 0x16, 0xd4, 0xf6, 0xaf, 0x3e, 0x71, 0x5f, 0x31, 0xc5, 0xc9, 0x11, 0x54, 0x4c, 0x16, 0x4c, 0x72, + 0x2c, 0x64, 0xb9, 0x7f, 0x6d, 0x72, 0xae, 0xda, 0x94, 0xc2, 0xf0, 0xaa, 0x65, 0x8f, 0xa0, 0x62, + 0x22, 0x36, 0x6c, 0xcb, 0x37, 0x64, 0xbb, 0xfa, 0x4e, 0x14, 0x64, 0xb6, 0xee, 0xfc, 0xab, 0x00, + 0x1b, 0xb9, 0x80, 0x8f, 0x7d, 0x16, 0x10, 0x01, 0xe4, 0x6c, 0x90, 0x36, 0x9b, 0xa3, 0x42, 0xbc, + 0x3a, 0x92, 0xb0, 0x1f, 0x5e, 0xeb, 0x68, 0x8e, 0xcd, 0xfe, 0xf5, 0x20, 0x69, 0xc9, 0x93, 0x50, + 0xef, 0x1f, 0x06, 0x4a, 0x5c, 0xd0, 0x8d, 0xb3, 0x59, 0x29, 0x79, 0x01, 0xf5, 0x59, 0x9f, 0x2e, + 0x1b, 0xe9, 0x11, 0x63, 0x8e, 0xf7, 0xe8, 0xfb, 0x78, 0xdd, 0x67, 0xa3, 0x43, 0xd7, 0xb8, 0xad, + 0x9d, 0xcd, 0x89, 0xc9, 0x8f, 0xa1, 0xe0, 0xb2, 0x11, 0x4e, 0x94, 0xca, 0xce, 0xd6, 0x9c, 0x1b, + 0xcd, 0xbb, 0xfb, 0x98, 0x6a, 0x44, 0xf3, 0x19, 0xd4, 0x17, 0x9d, 0x84, 0xd4, 0xa0, 0xa0, 0x8b, + 0xd7, 0xd4, 0x9c, 0x5e, 0x92, 0x07, 0x70, 0xeb, 0x05, 0xf3, 0x63, 0x9e, 0xd4, 0xdb, 0x9d, 0x37, + 0x49, 0xb5, 0x35, 0x35, 0xa0, 0x8f, 0x97, 0x3f, 0xb2, 0x9a, 0x7b, 0xb0, 0xb5, 0x30, 0xde, 0x05, + 0xe4, 0xf5, 0x3c, 0x79, 0x31, 0x47, 0xd2, 0xf9, 0xa3, 0x05, 0x35, 0xca, 0x5d, 0x36, 0xd4, 0x85, + 0xfb, 0x45, 0xa4, 0xff, 0x4a, 0xf2, 0x00, 0x48, 0x2c, 0xb9, 0x73, 0x1a, 0xfb, 0xbe, 0x23, 0x52, + 0x25, 0xf2, 0x95, 0x68, 0x2d, 0x96, 0xfc, 0x51, 0xec, 0xfb, 0x99, 0x11, 0xf9, 0x25, 0xbc, 0xad, + 0xd1, 0xd1, 0x79, 0x82, 0x75, 0x22, 0xcf, 0x73, 0x06, 0x5c, 0x2a, 0x87, 0x9f, 0x9e, 0x86, 0x42, + 0x99, 0x81, 0x4d, 0x1b, 0xb1, 0xe4, 0xc7, 0xe7, 0xc6, 0xec, 0xd8, 0xf3, 0x7a, 0x5c, 0xaa, 0x87, + 0xa8, 0xef, 0xfc, 0xd7, 0x02, 0xf2, 0xc5, 0x09, 0xf7, 0x1f, 0x06, 0x6e, 0x14, 0x7a, 0x81, 0xda, + 0x0b, 0x83, 0x53, 0x6f, 0x44, 0x7e, 0x08, 0x85, 0x58, 0xf8, 0xe6, 0x18, 0xbd, 0xd5, 0xe9, 0x65, + 0xbb, 0xf0, 0x94, 0x1e, 0x51, 0x2d, 0x23, 0xcf, 0x60, 0x75, 0xcc, 0x99, 0xcb, 0x85, 0x4c, 0x52, + 0xfd, 0xe9, 0xb5, 0xa9, 0x7e, 0xd3, 0x81, 0x7d, 0x60, 0x28, 0x4c, 0x92, 0x53, 0x42, 0xd2, 0x84, + 0x92, 0x17, 0x48, 0x3e, 0x8c, 0x05, 0xc7, 0x04, 0x97, 0x68, 0xb6, 0xc7, 0xa1, 0xe2, 0x4d, 0x78, + 0x18, 0x2b, 0xbc, 0x17, 0x0a, 0x34, 0xdd, 0x36, 0x3f, 0x86, 0xb5, 0x3c, 0xdd, 0x75, 0x39, 0x28, + 0xe7, 0x73, 0xf0, 0x77, 0x0b, 0x6a, 0x7b, 0xbe, 0x37, 0x7c, 0x7e, 0x10, 0xc6, 0x92, 0x27, 0xa7, + 0x6f, 0x42, 0x69, 0x1c, 0x4a, 0x95, 0x1b, 0x4d, 0xd9, 0x5e, 0x8f, 0x2c, 0xbd, 0x4e, 0x98, 0x70, + 0xad, 0x65, 0x91, 0xfe, 0xd8, 0x3a, 0xe4, 0x5b, 0x14, 0xd7, 0x9a, 0x23, 0x96, 0x5c, 0x20, 0x47, + 0xd1, 0x70, 0xa4, 0x7b, 0xad, 0x8b, 0x98, 0x94, 0xbf, 0x09, 0x85, 0x8b, 0xb7, 0x57, 0x99, 0x66, + 0x7b, 0xad, 0xd3, 0x13, 0x5d, 0x3f, 0xb7, 0xf0, 0xaa, 0x2a, 0xd3, 0x6c, 0xdf, 0xa1, 0xb0, 0x76, + 0xec, 0xc7, 0x23, 0x2f, 0x48, 0xe2, 0xec, 0x40, 0x55, 0x2a, 0x26, 0x94, 0xa3, 0xbf, 0x84, 0x13, + 0x98, 0x47, 0x40, 0x81, 0x56, 0x50, 0x78, 0xe2, 0x4d, 0xf8, 0xe7, 0x92, 0xb4, 0xa0, 0xc2, 0x03, + 0x37, 0x43, 0x2c, 0x23, 0xa2, 0xcc, 0x03, 0xd7, 0xe8, 0x3b, 0xff, 0xb4, 0xa0, 0xbc, 0xcf, 0x07, + 0xf1, 0x08, 0x47, 0xd5, 0x19, 0x6c, 0x85, 0x8a, 0xfb, 0x8e, 0xab, 0x25, 0x0e, 0x53, 0x49, 0x12, + 0x65, 0x32, 0x4b, 0x3e, 0xb9, 0xbe, 0xab, 0x53, 0x2a, 0x4c, 0x3a, 0xee, 0x76, 0x53, 0x16, 0xba, + 0xa9, 0xb9, 0x67, 0x65, 0xb2, 0xf9, 0x0b, 0x53, 0x80, 0xb3, 0xe2, 0x85, 0x37, 0xc3, 0xc2, 0x2c, + 0x76, 0xfe, 0xba, 0x02, 0x9b, 0x47, 0xe1, 0xc8, 0x1b, 0x32, 0xff, 0xd8, 0x84, 0x64, 0x66, 0xf8, + 0x57, 0x70, 0x3b, 0xff, 0x98, 0xd6, 0x2f, 0xd6, 0xb4, 0xc1, 0x7f, 0xfa, 0x5d, 0x86, 0x13, 0xb2, + 0xd1, 0x9a, 0x3b, 0x7f, 0x47, 0x7c, 0x0a, 0x6b, 0xda, 0xd6, 0x09, 0x4d, 0xe3, 0x26, 0x03, 0xe9, + 0xed, 0xc5, 0xb3, 0x23, 0xe9, 0x6e, 0x5a, 0x89, 0xae, 0x36, 0xfa, 0x29, 0x23, 0xb8, 0x8c, 0x7d, + 0x95, 0x3d, 0x93, 0x4c, 0xa1, 0x54, 0x8d, 0x34, 0x7d, 0x17, 0x3d, 0x81, 0xad, 0x04, 0x36, 0x77, + 0xbd, 0x63, 0xe9, 0x98, 0x97, 0x25, 0x45, 0xc0, 0xec, 0x25, 0xbf, 0x69, 0xac, 0xfa, 0xf9, 0xab, + 0x5e, 0x7f, 0x95, 0x6c, 0xaa, 0x64, 0xa1, 0xaf, 0xde, 0xf0, 0xab, 0xcc, 0x0f, 0x2b, 0x5a, 0x13, + 0xf3, 0xe3, 0xeb, 0xf7, 0x50, 0xc7, 0x02, 0xe2, 0x49, 0xbb, 0x3b, 0x43, 0x2c, 0x55, 0x7c, 0x06, + 0x55, 0x76, 0xde, 0xff, 0x3f, 0x46, 0x45, 0xef, 0xce, 0xf4, 0xb2, 0xbd, 0x60, 0x46, 0x51, 0xa2, + 0x1d, 0xcd, 0xcd, 0x2d, 0x0a, 0xd5, 0x08, 0x3b, 0x24, 0xf5, 0x7b, 0xd3, 0x77, 0x55, 0xbe, 0xaf, + 0xe8, 0x5a, 0x94, 0xef, 0xb2, 0xaf, 0xe0, 0xf6, 0x50, 0x4f, 0x88, 0xb1, 0x9e, 0x10, 0x29, 0x6f, + 0xe5, 0x86, 0x9f, 0x6c, 0x7e, 0xb6, 0xd0, 0xda, 0x15, 0x57, 0xc2, 0x7f, 0x08, 0x60, 0xda, 0x0d, + 0x5f, 0x07, 0xe6, 0x85, 0x7e, 0xef, 0xe6, 0x8d, 0x46, 0xcb, 0x6e, 0xba, 0xfc, 0xac, 0x58, 0xb2, + 0x6a, 0xcb, 0x9f, 0x15, 0x4b, 0x2b, 0xb5, 0xd5, 0xce, 0x9f, 0x2c, 0xa8, 0xcf, 0xf6, 0x85, 0x29, + 0x12, 0x72, 0x1f, 0x56, 0xcc, 0xcf, 0x37, 0x6c, 0xae, 0xca, 0xce, 0x26, 0xfe, 0x90, 0x49, 0x7e, + 0xd9, 0xd9, 0x7d, 0x5c, 0xd0, 0x04, 0x42, 0xf6, 0xa1, 0x88, 0x6f, 0x09, 0xd3, 0x38, 0xef, 0x7d, + 0xd7, 0x5b, 0x9d, 0xa2, 0x75, 0x6f, 0xef, 0xe5, 0xab, 0xd6, 0xd2, 0xd7, 0xaf, 0x5a, 0x4b, 0xdf, + 0xbe, 0x6a, 0x59, 0x7f, 0x98, 0xb6, 0xac, 0xbf, 0x4d, 0x5b, 0xd6, 0xbf, 0xa7, 0x2d, 0xeb, 0xe5, + 0xb4, 0x65, 0x7d, 0x33, 0x6d, 0x59, 0xff, 0x99, 0xb6, 0x96, 0xbe, 0x9d, 0xb6, 0xac, 0xbf, 0xbc, + 0x6e, 0x2d, 0xbd, 0x7c, 0xdd, 0x5a, 0xfa, 0xfa, 0x75, 0x6b, 0xe9, 0x59, 0x75, 0x86, 0x7a, 0xb0, + 0x82, 0x3f, 0xfa, 0xde, 0xff, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xd4, 0x09, 0xb7, 0x07, 0x45, + 0x0f, 0x00, 0x00, } func (this *MetadataInfo) Equal(that interface{}) bool { @@ -1333,6 +1432,45 @@ func (this *OTelEndpointConfig) Equal(that interface{}) bool { } return true } +func (this *ClickHouseConfig) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ClickHouseConfig) + if !ok { + that2, ok := that.(ClickHouseConfig) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Hostname != that1.Hostname { + return false + } + if this.Host != that1.Host { + return false + } + if this.Port != that1.Port { + return false + } + if this.Username != that1.Username { + return false + } + if this.Password != that1.Password { + return false + } + if this.Database != that1.Database { + return false + } + return true +} func (this *PluginConfig) Equal(that interface{}) bool { if that == nil { return this == nil @@ -1456,6 +1594,9 @@ func (this *LogicalPlannerState) Equal(that interface{}) bool { if !this.PluginConfig.Equal(that1.PluginConfig) { return false } + if !this.ClickhouseConfig.Equal(that1.ClickhouseConfig) { + return false + } if !this.DebugInfo.Equal(that1.DebugInfo) { return false } @@ -1652,6 +1793,21 @@ func (this *OTelEndpointConfig) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *ClickHouseConfig) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&distributedpb.ClickHouseConfig{") + s = append(s, "Hostname: "+fmt.Sprintf("%#v", this.Hostname)+",\n") + s = append(s, "Host: "+fmt.Sprintf("%#v", this.Host)+",\n") + s = append(s, "Port: "+fmt.Sprintf("%#v", this.Port)+",\n") + s = append(s, "Username: "+fmt.Sprintf("%#v", this.Username)+",\n") + s = append(s, "Password: "+fmt.Sprintf("%#v", this.Password)+",\n") + s = append(s, "Database: "+fmt.Sprintf("%#v", this.Database)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *PluginConfig) GoString() string { if this == nil { return "nil" @@ -1690,7 +1846,7 @@ func (this *LogicalPlannerState) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 12) + s := make([]string, 0, 13) s = append(s, "&distributedpb.LogicalPlannerState{") if this.DistributedState != nil { s = append(s, "DistributedState: "+fmt.Sprintf("%#v", this.DistributedState)+",\n") @@ -1709,6 +1865,9 @@ func (this *LogicalPlannerState) GoString() string { if this.PluginConfig != nil { s = append(s, "PluginConfig: "+fmt.Sprintf("%#v", this.PluginConfig)+",\n") } + if this.ClickhouseConfig != nil { + s = append(s, "ClickhouseConfig: "+fmt.Sprintf("%#v", this.ClickhouseConfig)+",\n") + } if this.DebugInfo != nil { s = append(s, "DebugInfo: "+fmt.Sprintf("%#v", this.DebugInfo)+",\n") } @@ -2274,6 +2433,69 @@ func (m *OTelEndpointConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ClickHouseConfig) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClickHouseConfig) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ClickHouseConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Database) > 0 { + i -= len(m.Database) + copy(dAtA[i:], m.Database) + i = encodeVarintDistributedPlan(dAtA, i, uint64(len(m.Database))) + i-- + dAtA[i] = 0x32 + } + if len(m.Password) > 0 { + i -= len(m.Password) + copy(dAtA[i:], m.Password) + i = encodeVarintDistributedPlan(dAtA, i, uint64(len(m.Password))) + i-- + dAtA[i] = 0x2a + } + if len(m.Username) > 0 { + i -= len(m.Username) + copy(dAtA[i:], m.Username) + i = encodeVarintDistributedPlan(dAtA, i, uint64(len(m.Username))) + i-- + dAtA[i] = 0x22 + } + if m.Port != 0 { + i = encodeVarintDistributedPlan(dAtA, i, uint64(m.Port)) + i-- + dAtA[i] = 0x18 + } + if len(m.Host) > 0 { + i -= len(m.Host) + copy(dAtA[i:], m.Host) + i = encodeVarintDistributedPlan(dAtA, i, uint64(len(m.Host))) + i-- + dAtA[i] = 0x12 + } + if len(m.Hostname) > 0 { + i -= len(m.Hostname) + copy(dAtA[i:], m.Hostname) + i = encodeVarintDistributedPlan(dAtA, i, uint64(len(m.Hostname))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *PluginConfig) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2401,6 +2623,18 @@ func (m *LogicalPlannerState) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ClickhouseConfig != nil { + { + size, err := m.ClickhouseConfig.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintDistributedPlan(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x5a + } if m.DebugInfo != nil { { size, err := m.DebugInfo.MarshalToSizedBuffer(dAtA[:i]) @@ -2772,6 +3006,38 @@ func (m *OTelEndpointConfig) Size() (n int) { return n } +func (m *ClickHouseConfig) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Hostname) + if l > 0 { + n += 1 + l + sovDistributedPlan(uint64(l)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovDistributedPlan(uint64(l)) + } + if m.Port != 0 { + n += 1 + sovDistributedPlan(uint64(m.Port)) + } + l = len(m.Username) + if l > 0 { + n += 1 + l + sovDistributedPlan(uint64(l)) + } + l = len(m.Password) + if l > 0 { + n += 1 + l + sovDistributedPlan(uint64(l)) + } + l = len(m.Database) + if l > 0 { + n += 1 + l + sovDistributedPlan(uint64(l)) + } + return n +} + func (m *PluginConfig) Size() (n int) { if m == nil { return 0 @@ -2857,6 +3123,10 @@ func (m *LogicalPlannerState) Size() (n int) { l = m.DebugInfo.Size() n += 1 + l + sovDistributedPlan(uint64(l)) } + if m.ClickhouseConfig != nil { + l = m.ClickhouseConfig.Size() + n += 1 + l + sovDistributedPlan(uint64(l)) + } return n } @@ -3045,6 +3315,21 @@ func (this *OTelEndpointConfig) String() string { }, "") return s } +func (this *ClickHouseConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ClickHouseConfig{`, + `Hostname:` + fmt.Sprintf("%v", this.Hostname) + `,`, + `Host:` + fmt.Sprintf("%v", this.Host) + `,`, + `Port:` + fmt.Sprintf("%v", this.Port) + `,`, + `Username:` + fmt.Sprintf("%v", this.Username) + `,`, + `Password:` + fmt.Sprintf("%v", this.Password) + `,`, + `Database:` + fmt.Sprintf("%v", this.Database) + `,`, + `}`, + }, "") + return s +} func (this *PluginConfig) String() string { if this == nil { return "nil" @@ -3095,6 +3380,7 @@ func (this *LogicalPlannerState) String() string { `OTelEndpointConfig:` + strings.Replace(this.OTelEndpointConfig.String(), "OTelEndpointConfig", "OTelEndpointConfig", 1) + `,`, `PluginConfig:` + strings.Replace(this.PluginConfig.String(), "PluginConfig", "PluginConfig", 1) + `,`, `DebugInfo:` + strings.Replace(this.DebugInfo.String(), "DebugInfo", "DebugInfo", 1) + `,`, + `ClickhouseConfig:` + strings.Replace(this.ClickhouseConfig.String(), "ClickHouseConfig", "ClickHouseConfig", 1) + `,`, `}`, }, "") return s @@ -4705,6 +4991,235 @@ func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { } return nil } +func (m *ClickHouseConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ClickHouseConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ClickHouseConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hostname", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hostname = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + } + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Username", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Username = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Password = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Database", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Database = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipDistributedPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthDistributedPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *PluginConfig) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -5300,6 +5815,42 @@ func (m *LogicalPlannerState) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseConfig", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowDistributedPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthDistributedPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthDistributedPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClickhouseConfig == nil { + m.ClickhouseConfig = &ClickHouseConfig{} + } + if err := m.ClickhouseConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipDistributedPlan(dAtA[iNdEx:]) diff --git a/src/carnot/planner/distributedpb/distributed_plan.proto b/src/carnot/planner/distributedpb/distributed_plan.proto index b5a4e8d08a1..581b8748d37 100644 --- a/src/carnot/planner/distributedpb/distributed_plan.proto +++ b/src/carnot/planner/distributedpb/distributed_plan.proto @@ -142,6 +142,23 @@ message OTelEndpointConfig { int64 timeout = 4; } +// ClickHouseConfig contains the connection parameters for ClickHouse. +message ClickHouseConfig { + // The hostname of the node executing the query. + string hostname = 1; + // The ClickHouse server host. + string host = 2; + // The ClickHouse server port. + int32 port = 3; + // The ClickHouse username. + string username = 4; + // The ClickHouse password. + string password = 5; + // The ClickHouse database name. + string database = 6; +} + + message PluginConfig { // The start_time of the script in nanoseconds. int64 start_time_ns = 1; @@ -183,6 +200,8 @@ message LogicalPlannerState { // PluginConfig contains plugin related configuration. PluginConfig plugin_config = 9; + ClickHouseConfig clickhouse_config = 11; + // Debug options for the compiler. DebugInfo debug_info = 10; } diff --git a/src/carnot/planner/ir/BUILD.bazel b/src/carnot/planner/ir/BUILD.bazel index 55b3ac401d4..6a064c629f0 100644 --- a/src/carnot/planner/ir/BUILD.bazel +++ b/src/carnot/planner/ir/BUILD.bazel @@ -47,6 +47,7 @@ pl_cc_library( "//src/carnot/planpb:plan_pl_cc_proto", "//src/shared/metadata:cc_library", "//src/shared/metadatapb:metadata_pl_cc_proto", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", "@com_github_vinzenz_libpypa//:libpypa", ], ) @@ -67,6 +68,14 @@ pl_cc_test( ], ) +pl_cc_test( + name = "clickhouse_export_sink_ir_test", + srcs = ["clickhouse_export_sink_ir_test.cc"], + deps = [ + "//src/carnot/planner/compiler:test_utils", + ], +) + pl_cc_test( name = "pattern_match_test", srcs = ["pattern_match_test.cc"], diff --git a/src/carnot/planner/ir/all_ir_nodes.h b/src/carnot/planner/ir/all_ir_nodes.h index 5c0b49744cd..b5689d1389f 100644 --- a/src/carnot/planner/ir/all_ir_nodes.h +++ b/src/carnot/planner/ir/all_ir_nodes.h @@ -20,6 +20,8 @@ #include "src/carnot/planner/ir/blocking_agg_ir.h" #include "src/carnot/planner/ir/bool_ir.h" +#include "src/carnot/planner/ir/clickhouse_source_ir.h" +#include "src/carnot/planner/ir/clickhouse_export_sink_ir.h" #include "src/carnot/planner/ir/column_ir.h" #include "src/carnot/planner/ir/data_ir.h" #include "src/carnot/planner/ir/drop_ir.h" diff --git a/src/carnot/planner/ir/clickhouse_export_sink_ir.cc b/src/carnot/planner/ir/clickhouse_export_sink_ir.cc new file mode 100644 index 00000000000..b4492ff8ede --- /dev/null +++ b/src/carnot/planner/ir/clickhouse_export_sink_ir.cc @@ -0,0 +1,131 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/planner/ir/clickhouse_export_sink_ir.h" +#include "src/carnot/planner/ir/ir.h" +#include "src/carnot/planpb/plan.pb.h" +#include + +namespace px { +namespace carnot { +namespace planner { + +StatusOr>> +ClickHouseExportSinkIR::RequiredInputColumns() const { + return std::vector>{required_column_names_}; +} + +Status ClickHouseExportSinkIR::Init(OperatorIR* parent, const std::string& table_name, + const std::string& clickhouse_dsn) { + table_name_ = table_name; + + // Parse the ClickHouse DSN and initialize the config + PX_ASSIGN_OR_RETURN(auto config, ParseClickHouseDSN(clickhouse_dsn)); + clickhouse_config_ = std::make_unique(config); + + return AddParent(parent); +} + +StatusOr ClickHouseExportSinkIR::ParseClickHouseDSN(const std::string& dsn) { + // Expected format: [clickhouse://]username:password@host:port/database + // The clickhouse:// prefix is optional + std::regex dsn_regex(R"((?:clickhouse://)?([^:]+):([^@]+)@([^:]+):(\d+)/(.+))"); + std::smatch matches; + + if (!std::regex_match(dsn, matches, dsn_regex)) { + return error::InvalidArgument("Invalid ClickHouse DSN format. Expected: [clickhouse://]username:password@host:port/database"); + } + + planpb::ClickHouseConfig config; + + // Extract the components + config.set_username(matches[1].str()); + config.set_password(matches[2].str()); + config.set_host(matches[3].str()); + config.set_port(std::stoi(matches[4].str())); + config.set_database(matches[5].str()); + + // hostname will be set by the runtime + config.set_hostname(""); + + return config; +} + +Status ClickHouseExportSinkIR::ToProto(planpb::Operator* op) const { + op->set_op_type(planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR); + auto clickhouse_op = op->mutable_clickhouse_sink_op(); + + // ClickHouse config must be set before calling ToProto + if (clickhouse_config_ == nullptr) { + return error::InvalidArgument("ClickHouse config not set"); + } + + // Set the ClickHouse configuration + *clickhouse_op->mutable_clickhouse_config() = *clickhouse_config_; + clickhouse_op->set_table_name(table_name_); + + // Map all input columns to ClickHouse columns + DCHECK(is_type_resolved()); + int64_t idx = 0; + for (const auto& [col_name, col_type] : *resolved_table_type()) { + DCHECK(col_type->IsValueType()); + auto value_type = std::static_pointer_cast(col_type); + + auto column_mapping = clickhouse_op->add_column_mappings(); + column_mapping->set_input_column_index(idx); + column_mapping->set_clickhouse_column_name(col_name); + column_mapping->set_column_type(value_type->data_type()); + idx++; + } + + return Status::OK(); +} + +Status ClickHouseExportSinkIR::CopyFromNodeImpl( + const IRNode* node, absl::flat_hash_map*) { + const ClickHouseExportSinkIR* source = static_cast(node); + table_name_ = source->table_name_; + required_column_names_ = source->required_column_names_; + if (source->clickhouse_config_ != nullptr) { + clickhouse_config_ = std::make_unique(*source->clickhouse_config_); + } + return Status::OK(); +} + +Status ClickHouseExportSinkIR::ResolveType(CompilerState* compiler_state) { + DCHECK_EQ(1U, parent_types().size()); + + auto parent_table_type = std::static_pointer_cast(parent_types()[0]); + + // Store ClickHouse config from compiler state only if not already set by Init() + if (clickhouse_config_ == nullptr && compiler_state->clickhouse_config() != nullptr) { + clickhouse_config_ = std::make_unique(*compiler_state->clickhouse_config()); + } + + // Populate required column names + for (const auto& col_name : parent_table_type->ColumnNames()) { + required_column_names_.insert(col_name); + } + + // Export sink passes through the input schema + return SetResolvedType(parent_table_type); +} + +} // namespace planner +} // namespace carnot +} // namespace px diff --git a/src/carnot/planner/ir/clickhouse_export_sink_ir.h b/src/carnot/planner/ir/clickhouse_export_sink_ir.h new file mode 100644 index 00000000000..f4bc98246d6 --- /dev/null +++ b/src/carnot/planner/ir/clickhouse_export_sink_ir.h @@ -0,0 +1,74 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include + +#include +#include "src/carnot/planner/compiler_state/compiler_state.h" +#include "src/carnot/planner/ir/column_ir.h" +#include "src/carnot/planner/ir/operator_ir.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/common/base/base.h" + +namespace px { +namespace carnot { +namespace planner { + +/** + * @brief The IR representation for the ClickHouseExportSink operator. + * Represents a configuration to export a DataFrame to a ClickHouse database. + */ +class ClickHouseExportSinkIR : public OperatorIR { + public: + explicit ClickHouseExportSinkIR(int64_t id) : OperatorIR(id, IRNodeType::kClickHouseExportSink) {} + + Status Init(OperatorIR* parent, const std::string& table_name, const std::string& clickhouse_dsn); + + StatusOr ParseClickHouseDSN(const std::string& dsn); + + Status ToProto(planpb::Operator* op) const override; + + Status CopyFromNodeImpl(const IRNode* node, + absl::flat_hash_map*) override; + + Status ResolveType(CompilerState* compiler_state); + inline bool IsBlocking() const override { return true; } + + StatusOr>> RequiredInputColumns() const override; + + const std::string& table_name() const { return table_name_; } + + protected: + StatusOr> PruneOutputColumnsToImpl( + const absl::flat_hash_set& /*kept_columns*/) override { + return error::Unimplemented("Unexpected call to ClickHouseExportSinkIR::PruneOutputColumnsTo."); + } + + private: + std::string table_name_; + absl::flat_hash_set required_column_names_; + std::unique_ptr clickhouse_config_; +}; + +} // namespace planner +} // namespace carnot +} // namespace px diff --git a/src/carnot/planner/ir/clickhouse_export_sink_ir_test.cc b/src/carnot/planner/ir/clickhouse_export_sink_ir_test.cc new file mode 100644 index 00000000000..f3f13ad329d --- /dev/null +++ b/src/carnot/planner/ir/clickhouse_export_sink_ir_test.cc @@ -0,0 +1,159 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include +#include +#include + +#include +#include +#include + +#include "src/carnot/planner/compiler/test_utils.h" +#include "src/carnot/planner/ir/clickhouse_export_sink_ir.h" +#include "src/carnot/planner/ir/memory_source_ir.h" +#include "src/carnot/planpb/plan.pb.h" +#include "src/common/testing/protobuf.h" +#include "src/table_store/table_store.h" + +namespace px { +namespace carnot { +namespace planner { + +using ClickHouseExportSinkTest = ASTVisitorTest; + +TEST_F(ClickHouseExportSinkTest, basic_export) { + // Create a simple relation with some columns + table_store::schema::Relation relation{ + {types::TIME64NS, types::STRING, types::INT64, types::FLOAT64}, + {"time_", "hostname", "count", "latency"}, + {types::ST_NONE, types::ST_NONE, types::ST_NONE, types::ST_DURATION_NS}}; + + (*compiler_state_->relation_map())["table"] = relation; + + auto src = MakeMemSource("table"); + EXPECT_OK(src->ResolveType(compiler_state_.get())); + + std::string clickhouse_dsn = "default:test_password@localhost:9000/default"; + ASSERT_OK_AND_ASSIGN(auto clickhouse_sink, + graph->CreateNode(src->ast(), src, "http_events", clickhouse_dsn)); + + clickhouse_sink->PullParentTypes(); + EXPECT_OK(clickhouse_sink->UpdateOpAfterParentTypesResolved()); + + // ResolveType will try to get config from compiler state, but we'll set it directly + // by creating a new CompilerState with ClickHouse config + auto new_relation_map = std::make_unique(); + (*new_relation_map)["table"] = relation; + + auto clickhouse_config = std::make_unique(); + clickhouse_config->set_host("localhost"); + clickhouse_config->set_port(9000); + clickhouse_config->set_username("default"); + clickhouse_config->set_password("test_password"); + clickhouse_config->set_database("default"); + + auto new_compiler_state = std::make_unique( + std::move(new_relation_map), + SensitiveColumnMap{}, + compiler_state_->registry_info(), + compiler_state_->time_now(), + 0, // max_output_rows_per_table + "", // result_address + "", // result_ssl_targetname + RedactionOptions{}, + nullptr, // endpoint_config + nullptr, // plugin_config + DebugInfo{}, + std::move(clickhouse_config)); + + // ResolveType will copy the config from compiler state + EXPECT_OK(clickhouse_sink->ResolveType(new_compiler_state.get())); + + planpb::Operator pb; + EXPECT_OK(clickhouse_sink->ToProto(&pb)); + + EXPECT_EQ(pb.op_type(), planpb::CLICKHOUSE_EXPORT_SINK_OPERATOR); + EXPECT_EQ(pb.clickhouse_sink_op().table_name(), "http_events"); + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings_size(), 4); + + // Verify column mappings + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(0).input_column_index(), 0); + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(0).clickhouse_column_name(), "time_"); + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(0).column_type(), types::TIME64NS); + + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(1).input_column_index(), 1); + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(1).clickhouse_column_name(), "hostname"); + EXPECT_EQ(pb.clickhouse_sink_op().column_mappings(1).column_type(), types::STRING); +} + +TEST_F(ClickHouseExportSinkTest, required_input_columns) { + table_store::schema::Relation relation{ + {types::TIME64NS, types::STRING, types::INT64}, + {"time_", "hostname", "count"}, + {types::ST_NONE, types::ST_NONE, types::ST_NONE}}; + + (*compiler_state_->relation_map())["table"] = relation; + + auto src = MakeMemSource("table"); + EXPECT_OK(src->ResolveType(compiler_state_.get())); + + std::string clickhouse_dsn = "default:test_password@localhost:9000/default"; + ASSERT_OK_AND_ASSIGN(auto clickhouse_sink, + graph->CreateNode(src->ast(), src, "http_events", clickhouse_dsn)); + + clickhouse_sink->PullParentTypes(); + EXPECT_OK(clickhouse_sink->UpdateOpAfterParentTypesResolved()); + + // Need to call ResolveType to populate required_column_names_ + auto clickhouse_config = std::make_unique(); + clickhouse_config->set_host("localhost"); + clickhouse_config->set_port(9000); + clickhouse_config->set_username("default"); + clickhouse_config->set_password("test_password"); + clickhouse_config->set_database("default"); + + auto new_relation_map = std::make_unique(); + (*new_relation_map)["table"] = relation; + + auto new_compiler_state = std::make_unique( + std::move(new_relation_map), + SensitiveColumnMap{}, + compiler_state_->registry_info(), + compiler_state_->time_now(), + 0, + "", + "", + RedactionOptions{}, + nullptr, + nullptr, + DebugInfo{}, + std::move(clickhouse_config)); + + EXPECT_OK(clickhouse_sink->ResolveType(new_compiler_state.get())); + + ASSERT_OK_AND_ASSIGN(auto required_input_columns, clickhouse_sink->RequiredInputColumns()); + ASSERT_EQ(required_input_columns.size(), 1); + EXPECT_THAT(required_input_columns[0], + ::testing::UnorderedElementsAre("time_", "hostname", "count")); +} + + +} // namespace planner +} // namespace carnot +} // namespace px diff --git a/src/carnot/planner/ir/clickhouse_source_ir.cc b/src/carnot/planner/ir/clickhouse_source_ir.cc new file mode 100644 index 00000000000..9d6aba8dfc1 --- /dev/null +++ b/src/carnot/planner/ir/clickhouse_source_ir.cc @@ -0,0 +1,328 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "src/carnot/planner/ir/clickhouse_source_ir.h" + +#include + +#include "src/carnot/planner/ir/ir.h" + +namespace px { +namespace carnot { +namespace planner { + +std::string ClickHouseSourceIR::DebugString() const { + return absl::Substitute("$0(id=$1, table=$2)", type_string(), id(), table_name_); +} + +Status ClickHouseSourceIR::ToProto(planpb::Operator* op) const { + auto pb = op->mutable_clickhouse_source_op(); + op->set_op_type(planpb::CLICKHOUSE_SOURCE_OPERATOR); + + // Set ClickHouse connection parameters from stored values + pb->set_host(host_); + pb->set_port(port_); + pb->set_username(username_); + pb->set_password(password_); + pb->set_database(database_); + + if (!column_index_map_set()) { + return error::InvalidArgument("ClickHouseSource columns are not set."); + } + + DCHECK(is_type_resolved()); + DCHECK_EQ(column_index_map_.size(), resolved_table_type()->ColumnNames().size()); + + // Build the query with explicit column list to match output_descriptor_ order + std::vector column_list; + for (const auto& [idx, col_name] : Enumerate(resolved_table_type()->ColumnNames())) { + column_list.push_back(col_name); + pb->add_column_names(col_name); + auto val_type = std::static_pointer_cast( + resolved_table_type()->GetColumnType(col_name).ConsumeValueOrDie()); + pb->add_column_types(val_type->data_type()); + } + + // Generate SELECT with explicit columns instead of SELECT * to ensure correct column ordering + pb->set_query(absl::Substitute("SELECT $0 FROM $1", absl::StrJoin(column_list, ", "), table_name_)); + + if (IsTimeStartSet()) { + pb->set_start_time(time_start_ns()); + } + + if (IsTimeStopSet()) { + pb->set_end_time(time_stop_ns()); + } + + // Set batch size + pb->set_batch_size(1024); + + // Set timestamp and partition columns from stored values + pb->set_timestamp_column(timestamp_column_); + pb->set_partition_column("hostname"); + + return Status::OK(); +} + +Status ClickHouseSourceIR::Init(const std::string& table_name, + const std::vector& select_columns, + const std::string& host, int port, + const std::string& username, const std::string& password, + const std::string& database, + const std::string& timestamp_column) { + table_name_ = table_name; + column_names_ = select_columns; + host_ = host; + port_ = port; + username_ = username; + password_ = password; + database_ = database; + timestamp_column_ = timestamp_column; + return Status::OK(); +} + +StatusOr> ClickHouseSourceIR::PruneOutputColumnsToImpl( + const absl::flat_hash_set& output_colnames) { + DCHECK(column_index_map_set()); + DCHECK(is_type_resolved()); + std::vector new_col_names; + std::vector new_col_index_map; + + auto col_names = resolved_table_type()->ColumnNames(); + for (const auto& [idx, name] : Enumerate(col_names)) { + if (output_colnames.contains(name)) { + new_col_names.push_back(name); + new_col_index_map.push_back(column_index_map_[idx]); + } + } + if (new_col_names != resolved_table_type()->ColumnNames()) { + column_names_ = new_col_names; + } + column_index_map_ = new_col_index_map; + return output_colnames; +} + +Status ClickHouseSourceIR::CopyFromNodeImpl(const IRNode* node, + absl::flat_hash_map*) { + const ClickHouseSourceIR* source_ir = static_cast(node); + + table_name_ = source_ir->table_name_; + time_start_ns_ = source_ir->time_start_ns_; + time_stop_ns_ = source_ir->time_stop_ns_; + column_names_ = source_ir->column_names_; + column_index_map_set_ = source_ir->column_index_map_set_; + column_index_map_ = source_ir->column_index_map_; + + username_ = source_ir->username_; + password_ = source_ir->password_; + database_ = source_ir->database_; + port_ = source_ir->port_; + host_ = source_ir->host_; + + return Status::OK(); +} + +StatusOr ClickHouseSourceIR::ClickHouseTypeToPixieType( + const std::string& ch_type_name) { + // Integer types - Pixie only supports INT64 + if (ch_type_name == "UInt8" || ch_type_name == "UInt16" || ch_type_name == "UInt32" || + ch_type_name == "UInt64" || ch_type_name == "Int8" || ch_type_name == "Int16" || + ch_type_name == "Int32" || ch_type_name == "Int64") { + return types::DataType::INT64; + } + // UInt128 + if (ch_type_name == "UInt128") { + return types::DataType::UINT128; + } + // Floating point types - Pixie only supports FLOAT64 + if (ch_type_name == "Float32" || ch_type_name == "Float64") { + return types::DataType::FLOAT64; + } + // String types + if (ch_type_name == "String" || ch_type_name == "FixedString" || + absl::StartsWith(ch_type_name, "FixedString(")) { + return types::DataType::STRING; + } + // Date/time types + if (ch_type_name == "DateTime" || absl::StartsWith(ch_type_name, "DateTime64")) { + return types::DataType::TIME64NS; + } + // Boolean type (stored as UInt8 in ClickHouse) + if (ch_type_name == "Bool") { + return types::DataType::BOOLEAN; + } + return types::DataType::STRING; +} + +StatusOr ClickHouseSourceIR::InferRelationFromClickHouse( + CompilerState* compiler_state, const std::string& table_name) { + // Check if ClickHouse config is available + // TODO(ddelnano): Add this check in when the configuration plumbing is done. + auto* ch_config = compiler_state->clickhouse_config(); + PX_UNUSED(ch_config); + + // Use stored connection parameters from Init() + + clickhouse::ClientOptions options; + options.SetHost(host_); + options.SetPort(port_); + options.SetUser(username_); + options.SetPassword(password_); + options.SetDefaultDatabase(database_); + + // Create ClickHouse client + std::unique_ptr client; + try { + client = std::make_unique(options); + } catch (const std::exception& e) { + return error::Internal("Failed to connect to ClickHouse at $0:$1 - $2", + host_, port_, e.what()); + } + + // Query ClickHouse for table schema using DESCRIBE TABLE + std::string describe_query = absl::Substitute("DESCRIBE TABLE $0", table_name); + + table_store::schema::Relation relation; + bool query_executed = false; + + try { + client->Select(describe_query, [&](const clickhouse::Block& block) { + query_executed = true; + // DESCRIBE TABLE returns columns: name, type, default_type, default_expression, comment, + // codec_expression, ttl_expression + size_t num_rows = block.GetRowCount(); + + if (num_rows == 0) { + return; + } + + // Get the column name and type columns + auto name_column = block[0]->As(); + auto type_column = block[1]->As(); + + for (size_t i = 0; i < num_rows; ++i) { + std::string col_name = std::string(name_column->At(i)); + std::string col_type = std::string(type_column->At(i)); + + // Convert ClickHouse type to Pixie type + auto pixie_type_or = ClickHouseTypeToPixieType(col_type); + if (!pixie_type_or.ok()) { + LOG(WARNING) << "Failed to convert ClickHouse type '" << col_type + << "' for column '" << col_name << "'. Using STRING as fallback."; + relation.AddColumn(types::DataType::STRING, col_name, types::SemanticType::ST_NONE); + } else { + types::DataType pixie_type = pixie_type_or.ConsumeValueOrDie(); + // Determine semantic type based on column name or type + types::SemanticType semantic_type = types::SemanticType::ST_NONE; + if (pixie_type == types::DataType::TIME64NS) { + semantic_type = types::SemanticType::ST_TIME_NS; + } + relation.AddColumn(pixie_type, col_name, semantic_type); + } + } + }); + } catch (const std::exception& e) { + return error::Internal("Failed to query ClickHouse table schema for '$0': $1", + table_name, e.what()); + } + + if (!query_executed || relation.NumColumns() == 0) { + return error::Internal("Table '$0' not found in ClickHouse or has no columns.", table_name); + } + + return relation; +} + +Status ClickHouseSourceIR::ResolveType(CompilerState* compiler_state) { + table_store::schema::Relation table_relation; + + auto existing_relation = false; + auto relation_it = compiler_state->relation_map()->find(table_name()); + if (relation_it == compiler_state->relation_map()->end()) { + // Table not found in relation_map, try to infer from ClickHouse + VLOG(1) << absl::Substitute("Table '$0' not found in relation_map. Attempting to infer schema from ClickHouse...", table_name()); + + auto relation_or = InferRelationFromClickHouse(compiler_state, table_name()); + if (!relation_or.ok()) { + return CreateIRNodeError("Table '$0' not found in relation_map and failed to infer from ClickHouse: $1", + table_name_, relation_or.status().msg()); + } + + table_relation = relation_or.ConsumeValueOrDie(); + } else { + table_relation = relation_it->second; + existing_relation = true; + } + auto full_table_type = TableType::Create(table_relation); + if (select_all()) { + // For select_all, add all table columns plus ClickHouse-added columns (hostname, event_time) + std::vector column_indices; + int64_t table_column_count = static_cast(table_relation.NumColumns()); + + // Add all table columns + for (int64_t i = 0; i < table_column_count; ++i) { + column_indices.push_back(i); + } + + // Add ClickHouse-added columns + if (existing_relation) { + full_table_type->AddColumn("hostname", ValueType::Create(types::DataType::STRING, types::SemanticType::ST_NONE)); + column_indices.push_back(table_column_count); // hostname is after all table columns + + full_table_type->AddColumn("event_time", ValueType::Create(types::DataType::TIME64NS, types::SemanticType::ST_TIME_NS)); + column_indices.push_back(table_column_count + 1); // event_time is after hostname + } + + SetColumnIndexMap(column_indices); + return SetResolvedType(full_table_type); + } + + std::vector column_indices; + auto new_table = TableType::Create(); + + // Calculate the index offset for ClickHouse-added columns (after all table columns) + int64_t table_column_count = static_cast(table_relation.NumColumns()); + auto next_count = 0; + + for (const auto& col_name : column_names_) { + // Handle special ClickHouse-added columns that don't exist in the source table + if (col_name == "hostname") { + new_table->AddColumn(col_name, ValueType::Create(types::DataType::STRING, types::SemanticType::ST_NONE)); + // hostname is added by ClickHouse after all table columns + column_indices.push_back(table_column_count + (next_count++)); + continue; + } + if (col_name == "event_time") { + new_table->AddColumn(col_name, ValueType::Create(types::DataType::TIME64NS, types::SemanticType::ST_TIME_NS)); + // event_time is added by ClickHouse after hostname + column_indices.push_back(table_column_count + (next_count++)); + continue; + } + + PX_ASSIGN_OR_RETURN(auto col_type, full_table_type->GetColumnType(col_name)); + new_table->AddColumn(col_name, col_type); + column_indices.push_back(table_relation.GetColumnIndex(col_name)); + } + + SetColumnIndexMap(column_indices); + return SetResolvedType(new_table); +} + +} // namespace planner +} // namespace carnot +} // namespace px diff --git a/src/carnot/planner/ir/clickhouse_source_ir.h b/src/carnot/planner/ir/clickhouse_source_ir.h new file mode 100644 index 00000000000..1f578e7bcef --- /dev/null +++ b/src/carnot/planner/ir/clickhouse_source_ir.h @@ -0,0 +1,144 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include +#include +#include + +#include "src/carnot/planner/compiler_state/compiler_state.h" +#include "src/carnot/planner/ir/expression_ir.h" +#include "src/carnot/planner/ir/operator_ir.h" +#include "src/carnot/planner/types/types.h" +#include "src/common/base/base.h" +#include "src/shared/types/types.h" +#include "src/table_store/schema/relation.h" + +namespace px { +namespace carnot { +namespace planner { + +/** + * @brief The ClickHouseSourceIR represents a source that reads data from a ClickHouse database. + */ +class ClickHouseSourceIR : public OperatorIR { + public: + ClickHouseSourceIR() = delete; + explicit ClickHouseSourceIR(int64_t id) : OperatorIR(id, IRNodeType::kClickHouseSource) {} + + /** + * @brief Initialize the ClickHouse source. + * + * @param table_name the table to load. + * @param select_columns the columns to select. If vector is empty, then select all columns. + * @param host the ClickHouse server host. + * @param port the ClickHouse server port. + * @param username the ClickHouse username. + * @param password the ClickHouse password. + * @param database the ClickHouse database. + * @return Status + */ + Status Init(const std::string& table_name, const std::vector& select_columns, + const std::string& host = "localhost", int port = 9000, + const std::string& username = "default", const std::string& password = "", + const std::string& database = "default", + const std::string& timestamp_column = "event_time"); + + std::string table_name() const { return table_name_; } + std::string host() const { return host_; } + int port() const { return port_; } + std::string username() const { return username_; } + std::string password() const { return password_; } + std::string database() const { return database_; } + std::string timestamp_column() const { return timestamp_column_; } + + void SetTimeStartNS(int64_t time_start_ns) { time_start_ns_ = time_start_ns; } + void SetTimeStopNS(int64_t time_stop_ns) { time_stop_ns_ = time_stop_ns; } + bool IsTimeStartSet() const { return time_start_ns_.has_value(); } + bool IsTimeStopSet() const { return time_stop_ns_.has_value(); } + + std::string DebugString() const override; + + int64_t time_start_ns() const { return time_start_ns_.value(); } + int64_t time_stop_ns() const { return time_stop_ns_.value(); } + + const std::vector& column_index_map() const { return column_index_map_; } + bool column_index_map_set() const { return column_index_map_set_; } + void SetColumnIndexMap(const std::vector& column_index_map) { + column_index_map_set_ = true; + column_index_map_ = column_index_map; + } + + Status ToProto(planpb::Operator*) const override; + + bool select_all() const { return column_names_.size() == 0; } + + Status CopyFromNodeImpl(const IRNode* node, + absl::flat_hash_map* copied_nodes_map) override; + const std::vector& column_names() const { return column_names_; } + + StatusOr>> RequiredInputColumns() const override { + return std::vector>{}; + } + + void SetColumnNames(const std::vector& col_names) { column_names_ = col_names; } + + bool IsSource() const override { return true; } + + Status ResolveType(CompilerState* compiler_state); + + protected: + // Helper method to query ClickHouse for table schema and create a Relation + StatusOr InferRelationFromClickHouse( + CompilerState* compiler_state, const std::string& table_name); + + // Helper method to convert ClickHouse type string to Pixie DataType + static StatusOr ClickHouseTypeToPixieType(const std::string& ch_type_name); + + StatusOr> PruneOutputColumnsToImpl( + const absl::flat_hash_set& output_colnames) override; + + private: + std::string table_name_; + + // ClickHouse connection parameters + std::string host_ = "localhost"; + int port_ = 9000; + std::string username_ = "default"; + std::string password_ = ""; + std::string database_ = "default"; + + // ClickHouse column configuration + std::string timestamp_column_ = "event_time"; + + std::optional time_start_ns_; + std::optional time_stop_ns_; + + // Hold of columns in the order that they are selected. + std::vector column_names_; + + // The mapping of the source's column indices to the current columns, as given by column_names_. + std::vector column_index_map_; + bool column_index_map_set_ = false; +}; + +} // namespace planner +} // namespace carnot +} // namespace px diff --git a/src/carnot/planner/ir/operators.inl b/src/carnot/planner/ir/operators.inl index 817295e3a6e..bb712c71c11 100644 --- a/src/carnot/planner/ir/operators.inl +++ b/src/carnot/planner/ir/operators.inl @@ -37,5 +37,7 @@ PX_CARNOT_IR_NODE(Rolling) PX_CARNOT_IR_NODE(Stream) PX_CARNOT_IR_NODE(EmptySource) PX_CARNOT_IR_NODE(OTelExportSink) +PX_CARNOT_IR_NODE(ClickHouseSource) +PX_CARNOT_IR_NODE(ClickHouseExportSink) #endif diff --git a/src/carnot/planner/ir/pattern_match.h b/src/carnot/planner/ir/pattern_match.h index f8c484f47b9..0eb386ddbc5 100644 --- a/src/carnot/planner/ir/pattern_match.h +++ b/src/carnot/planner/ir/pattern_match.h @@ -160,6 +160,10 @@ inline ClassMatch OTelExportSink() { return ClassMatch(); } +inline ClassMatch ClickHouseExportSink() { + return ClassMatch(); +} + inline ClassMatch EmptySource() { return ClassMatch(); } @@ -266,7 +270,7 @@ struct ResultSink : public ParentMatch { bool Match(const IRNode* node) const override { return ExternalGRPCSink().Match(node) || MemorySink().Match(node) || - OTelExportSink().Match(node); + OTelExportSink().Match(node) || ClickHouseExportSink().Match(node); } }; diff --git a/src/carnot/planner/logical_planner.cc b/src/carnot/planner/logical_planner.cc index 19ed07104cf..c2ab8d53a9e 100644 --- a/src/carnot/planner/logical_planner.cc +++ b/src/carnot/planner/logical_planner.cc @@ -97,6 +97,18 @@ StatusOr> CreateCompilerState( for (const auto& debug_info_pb : logical_state.debug_info().otel_debug_attributes()) { debug_info.otel_debug_attrs.push_back({debug_info_pb.name(), debug_info_pb.value()}); } + + std::unique_ptr clickhouse_config = nullptr; + if (logical_state.has_clickhouse_config()) { + clickhouse_config = std::make_unique(); + clickhouse_config->set_hostname(logical_state.clickhouse_config().hostname()); + clickhouse_config->set_host(logical_state.clickhouse_config().host()); + clickhouse_config->set_port(logical_state.clickhouse_config().port()); + clickhouse_config->set_username(logical_state.clickhouse_config().username()); + clickhouse_config->set_password(logical_state.clickhouse_config().password()); + clickhouse_config->set_database(logical_state.clickhouse_config().database()); + } + // Create a CompilerState obj using the relation map and grabbing the current time. return std::make_unique( std::move(rel_map), sensitive_columns, registry_info, px::CurrentTimeNS(), @@ -105,7 +117,8 @@ StatusOr> CreateCompilerState( // TODO(philkuz) add an endpoint config to logical_state and pass that in here. RedactionOptionsFromPb(logical_state.redaction_options()), std::move(otel_endpoint_config), // TODO(philkuz) propagate the otel debug attributes here. - std::move(plugin_config), debug_info); + std::move(plugin_config), debug_info, + std::move(clickhouse_config)); } StatusOr> LogicalPlanner::Create(const udfspb::UDFInfo& udf_info) { diff --git a/src/carnot/planner/logical_planner_test.cc b/src/carnot/planner/logical_planner_test.cc index 4c3e8659c88..3ee106f50f9 100644 --- a/src/carnot/planner/logical_planner_test.cc +++ b/src/carnot/planner/logical_planner_test.cc @@ -946,7 +946,7 @@ px.export(df, px.otel.Data( px.otel.metric.Gauge( name='resp_latency', value=df.resp_latency_ns, - ) + ), ] )) )pxl"; @@ -1039,6 +1039,180 @@ px.export(otel_df, px.otel.Data( )))otel"); } +constexpr char kClickHouseSourceQuery[] = R"pxl( +import px + +# Test ClickHouse source node functionality +df = px.DataFrame('http_events', start_time='-10m', end_time='-5m', clickhouse_dsn='user:test@clickhouse-server:9000/pixie') +df = df['time_', 'req_headers'] +px.display(df, 'clickhouse_data') +)pxl"; + +TEST_F(LogicalPlannerTest, ClickHouseSourceNode) { + auto planner = LogicalPlanner::Create(info_).ConsumeValueOrDie(); + + // Create a test schema that includes a ClickHouse table + auto state = testutils::CreateTwoPEMsOneKelvinPlannerState(testutils::kHttpEventsSchema); + + auto plan_or_s = planner->Plan(MakeQueryRequest(state, kClickHouseSourceQuery)); + EXPECT_OK(plan_or_s); + auto plan = plan_or_s.ConsumeValueOrDie(); + EXPECT_OK(plan->ToProto()); + + // Verify the plan contains ClickHouse source operators + auto plan_pb = plan->ToProto().ConsumeValueOrDie(); + bool has_clickhouse_source = false; + + for (const auto& [address, agent_plan] : plan_pb.qb_address_to_plan()) { + for (const auto& planFragment : agent_plan.nodes()) { + for (const auto& planNode : planFragment.nodes()) { + if (planNode.op().op_type() == planpb::OperatorType::CLICKHOUSE_SOURCE_OPERATOR) { + EXPECT_THAT(planNode.op().clickhouse_source_op().host(), "clickhouse-server"); + EXPECT_THAT(planNode.op().clickhouse_source_op().port(), 9000); + EXPECT_THAT(planNode.op().clickhouse_source_op().database(), "pixie"); + EXPECT_THAT(planNode.op().clickhouse_source_op().username(), "user"); + EXPECT_THAT(planNode.op().clickhouse_source_op().password(), "test"); + has_clickhouse_source = true; + break; + } + } + if (has_clickhouse_source) break; + } + if (has_clickhouse_source) break; + } + + // Note: This test validates that the planner can process ClickHouse queries + // The actual presence of ClickHouse operators depends on the table configuration + EXPECT_OK(plan->ToProto()); + EXPECT_TRUE(has_clickhouse_source); +} + +constexpr char kClickHouseExportQuery[] = R"pxl( +import px + +# Test ClickHouse export using endpoint config +df = px.DataFrame('http_events', start_time='-10m') +df = df[['time_', 'req_path', 'resp_status', 'resp_latency_ns']] +px.export(df, px.otel.ClickHouseRows(table='http_events')) +)pxl"; + +TEST_F(LogicalPlannerTest, ClickHouseExportWithEndpointConfig) { + auto planner = LogicalPlanner::Create(info_).ConsumeValueOrDie(); + + // Create a planner state with an OTel endpoint config containing ClickHouse DSN + auto state = testutils::CreateTwoPEMsOneKelvinPlannerState(testutils::kHttpEventsSchema); + + // Set up the endpoint config with ClickHouse DSN in the URL field + auto* endpoint_config = state.mutable_otel_endpoint_config(); + endpoint_config->set_url("clickhouse_user:clickhouse_pass@clickhouse.example.com:9000/pixie_db"); + endpoint_config->set_insecure(true); + endpoint_config->set_timeout(10); + + auto plan_or_s = planner->Plan(MakeQueryRequest(state, kClickHouseExportQuery)); + EXPECT_OK(plan_or_s); + auto plan = plan_or_s.ConsumeValueOrDie(); + EXPECT_OK(plan->ToProto()); + + // Verify the plan contains ClickHouse export sink operators with correct config + auto plan_pb = plan->ToProto().ConsumeValueOrDie(); + bool has_clickhouse_export = false; + + for (const auto& [address, agent_plan] : plan_pb.qb_address_to_plan()) { + for (const auto& planFragment : agent_plan.nodes()) { + for (const auto& planNode : planFragment.nodes()) { + if (planNode.op().op_type() == planpb::OperatorType::CLICKHOUSE_EXPORT_SINK_OPERATOR) { + const auto& clickhouse_sink_op = planNode.op().clickhouse_sink_op(); + + // Verify table name + EXPECT_EQ(clickhouse_sink_op.table_name(), "http_events"); + + // Verify the DSN was parsed correctly into ClickHouseConfig + const auto& config = clickhouse_sink_op.clickhouse_config(); + EXPECT_EQ(config.username(), "clickhouse_user"); + EXPECT_EQ(config.password(), "clickhouse_pass"); + EXPECT_EQ(config.host(), "clickhouse.example.com"); + EXPECT_EQ(config.port(), 9000); + EXPECT_EQ(config.database(), "pixie_db"); + + // Verify column mappings were created + EXPECT_GT(clickhouse_sink_op.column_mappings_size(), 0); + + has_clickhouse_export = true; + break; + } + } + if (has_clickhouse_export) break; + } + if (has_clickhouse_export) break; + } + + EXPECT_TRUE(has_clickhouse_export); +} + +constexpr char kClickHouseExportWithExplicitEndpointQuery[] = R"pxl( +import px + +# Test ClickHouse export with explicit endpoint config +df = px.DataFrame('http_events', start_time='-10m') +df = df[['time_', 'req_path', 'resp_status']] + +endpoint = px.otel.Endpoint( + url="explicit_user:explicit_pass@explicit-host:9001/explicit_db", + insecure=False, + timeout=20 +) + +px.export(df, px.otel.ClickHouseRows(table='custom_table', endpoint=endpoint)) +)pxl"; + +TEST_F(LogicalPlannerTest, ClickHouseExportWithExplicitEndpoint) { + auto planner = LogicalPlanner::Create(info_).ConsumeValueOrDie(); + + // Create a planner state with a default endpoint config + auto state = testutils::CreateTwoPEMsOneKelvinPlannerState(testutils::kHttpEventsSchema); + + // Set up a default endpoint config (should be overridden by explicit endpoint) + auto* endpoint_config = state.mutable_otel_endpoint_config(); + endpoint_config->set_url("default_user:default_pass@default-host:9000/default_db"); + + auto plan_or_s = planner->Plan(MakeQueryRequest(state, kClickHouseExportWithExplicitEndpointQuery)); + EXPECT_OK(plan_or_s); + auto plan = plan_or_s.ConsumeValueOrDie(); + EXPECT_OK(plan->ToProto()); + + // Verify the plan uses the explicit endpoint config, not the default + auto plan_pb = plan->ToProto().ConsumeValueOrDie(); + bool has_clickhouse_export = false; + + for (const auto& [address, agent_plan] : plan_pb.qb_address_to_plan()) { + for (const auto& planFragment : agent_plan.nodes()) { + for (const auto& planNode : planFragment.nodes()) { + if (planNode.op().op_type() == planpb::OperatorType::CLICKHOUSE_EXPORT_SINK_OPERATOR) { + const auto& clickhouse_sink_op = planNode.op().clickhouse_sink_op(); + + // Verify table name + EXPECT_EQ(clickhouse_sink_op.table_name(), "custom_table"); + + // Verify the explicit endpoint was used, not the default + const auto& config = clickhouse_sink_op.clickhouse_config(); + EXPECT_EQ(config.username(), "explicit_user"); + EXPECT_EQ(config.password(), "explicit_pass"); + EXPECT_EQ(config.host(), "explicit-host"); + EXPECT_EQ(config.port(), 9001); + EXPECT_EQ(config.database(), "explicit_db"); + + has_clickhouse_export = true; + break; + } + } + if (has_clickhouse_export) break; + } + if (has_clickhouse_export) break; + } + + EXPECT_TRUE(has_clickhouse_export); +} + } // namespace planner } // namespace carnot } // namespace px diff --git a/src/carnot/planner/objects/dataframe.cc b/src/carnot/planner/objects/dataframe.cc index 13140b40e17..8bcb2c09710 100644 --- a/src/carnot/planner/objects/dataframe.cc +++ b/src/carnot/planner/objects/dataframe.cc @@ -17,8 +17,12 @@ */ #include "src/carnot/planner/objects/dataframe.h" + +#include + #include "src/carnot/planner/ast/ast_visitor.h" #include "src/carnot/planner/ir/ast_utils.h" +#include "src/carnot/planner/ir/clickhouse_source_ir.h" #include "src/carnot/planner/objects/collection_object.h" #include "src/carnot/planner/objects/expr_object.h" #include "src/carnot/planner/objects/funcobject.h" @@ -28,11 +32,80 @@ #include "src/carnot/planner/objects/time.h" #include "src/common/base/statusor.h" +#include +#include + namespace px { namespace carnot { namespace planner { namespace compiler { +struct ClickHouseDSN { + std::string host = "localhost"; + int port = 9000; + std::string username = "default"; + std::string password = ""; + std::string database = "default"; +}; + +/** + * @brief Parse a ClickHouse DSN string + * + * Supports formats: + * clickhouse://user:password@host:port/database + * user:password@host:port/database + * host:port + * host + */ +StatusOr ParseClickHouseDSN(const std::string& dsn_str) { + ClickHouseDSN dsn; + std::string remaining = dsn_str; + + // Strip clickhouse:// prefix if present + if (absl::StartsWith(remaining, "clickhouse://")) { + remaining = remaining.substr(13); + } + + // Parse user:password@ if present + size_t at_pos = remaining.find('@'); + if (at_pos != std::string::npos) { + std::string auth_part = remaining.substr(0, at_pos); + remaining = remaining.substr(at_pos + 1); + + size_t colon_pos = auth_part.find(':'); + if (colon_pos != std::string::npos) { + dsn.username = auth_part.substr(0, colon_pos); + dsn.password = auth_part.substr(colon_pos + 1); + } else { + dsn.username = auth_part; + } + } + + // Parse host:port/database + size_t slash_pos = remaining.find('/'); + std::string host_port; + if (slash_pos != std::string::npos) { + host_port = remaining.substr(0, slash_pos); + dsn.database = remaining.substr(slash_pos + 1); + } else { + host_port = remaining; + } + + // Parse host:port + size_t colon_pos = host_port.find(':'); + if (colon_pos != std::string::npos) { + dsn.host = host_port.substr(0, colon_pos); + std::string port_str = host_port.substr(colon_pos + 1); + if (!absl::SimpleAtoi(port_str, &dsn.port)) { + return error::InvalidArgument("Invalid port in ClickHouse DSN: $0", port_str); + } + } else if (!host_port.empty()) { + dsn.host = host_port; + } + + return dsn; +} + StatusOr> GetAsDataFrame(QLObjectPtr obj) { if (!Dataframe::IsDataframe(obj)) { return obj->CreateError("Expected DataFrame, received $0", obj->name()); @@ -109,22 +182,81 @@ StatusOr DataFrameConstructor(CompilerState* compiler_state, IR* gr PX_ASSIGN_OR_RETURN(std::vector columns, ParseAsListOfStrings(args.GetArg("select"), "select")); std::string table_name = table->str(); - PX_ASSIGN_OR_RETURN(MemorySourceIR * mem_source_op, - graph->CreateNode(ast, table_name, columns)); - - if (!NoneObject::IsNoneObject(args.GetArg("start_time"))) { - PX_ASSIGN_OR_RETURN(ExpressionIR * start_time, GetArgAs(ast, args, "start_time")); - PX_ASSIGN_OR_RETURN(auto start_time_ns, - ParseAllTimeFormats(compiler_state->time_now().val, start_time)); - mem_source_op->SetTimeStartNS(start_time_ns); + + // Check if we should use ClickHouse or memory source + bool is_clickhouse = false; + ClickHouseDSN dsn; + std::string timestamp_column = "event_time"; + if (!NoneObject::IsNoneObject(args.GetArg("clickhouse_dsn"))) { + is_clickhouse = true; + PX_ASSIGN_OR_RETURN(StringIR * dsn_ir, GetArgAs(ast, args, "clickhouse_dsn")); + PX_ASSIGN_OR_RETURN(dsn, ParseClickHouseDSN(dsn_ir->str())); + + // Get timestamp column if specified + if (!NoneObject::IsNoneObject(args.GetArg("clickhouse_ts_col"))) { + PX_ASSIGN_OR_RETURN(StringIR * ts_col_ir, GetArgAs(ast, args, "clickhouse_ts_col")); + timestamp_column = ts_col_ir->str(); + } } - if (!NoneObject::IsNoneObject(args.GetArg("end_time"))) { - PX_ASSIGN_OR_RETURN(ExpressionIR * end_time, GetArgAs(ast, args, "end_time")); - PX_ASSIGN_OR_RETURN(auto end_time_ns, - ParseAllTimeFormats(compiler_state->time_now().val, end_time)); - mem_source_op->SetTimeStopNS(end_time_ns); + + if (is_clickhouse) { + // Create ClickHouseSourceIR + // Note: hostname and event_time columns are handled in ClickHouseSourceIR::ResolveType + // Only add them if the user explicitly selected some columns + std::vector clickhouse_columns = columns; + + if (!columns.empty()) { + // User selected specific columns - add hostname and event_time if not already present + if (std::find(clickhouse_columns.begin(), clickhouse_columns.end(), "hostname") == clickhouse_columns.end()) { + clickhouse_columns.push_back("hostname"); + } + + if (std::find(clickhouse_columns.begin(), clickhouse_columns.end(), "event_time") == clickhouse_columns.end()) { + clickhouse_columns.push_back("event_time"); + } + } + // If columns is empty, select_all() will be true and ResolveType will handle adding all columns + + PX_ASSIGN_OR_RETURN(ClickHouseSourceIR * clickhouse_source_op, + graph->CreateNode(ast, table_name, clickhouse_columns, + dsn.host, dsn.port, dsn.username, + dsn.password, dsn.database, + timestamp_column)); + + if (!NoneObject::IsNoneObject(args.GetArg("start_time"))) { + PX_ASSIGN_OR_RETURN(ExpressionIR * start_time, + GetArgAs(ast, args, "start_time")); + PX_ASSIGN_OR_RETURN(auto start_time_ns, + ParseAllTimeFormats(compiler_state->time_now().val, start_time)); + clickhouse_source_op->SetTimeStartNS(start_time_ns); + } + if (!NoneObject::IsNoneObject(args.GetArg("end_time"))) { + PX_ASSIGN_OR_RETURN(ExpressionIR * end_time, GetArgAs(ast, args, "end_time")); + PX_ASSIGN_OR_RETURN(auto end_time_ns, + ParseAllTimeFormats(compiler_state->time_now().val, end_time)); + clickhouse_source_op->SetTimeStopNS(end_time_ns); + } + return Dataframe::Create(compiler_state, clickhouse_source_op, visitor); + } else { + // Create MemorySourceIR (existing behavior) + PX_ASSIGN_OR_RETURN(MemorySourceIR * mem_source_op, + graph->CreateNode(ast, table_name, columns)); + + if (!NoneObject::IsNoneObject(args.GetArg("start_time"))) { + PX_ASSIGN_OR_RETURN(ExpressionIR * start_time, + GetArgAs(ast, args, "start_time")); + PX_ASSIGN_OR_RETURN(auto start_time_ns, + ParseAllTimeFormats(compiler_state->time_now().val, start_time)); + mem_source_op->SetTimeStartNS(start_time_ns); + } + if (!NoneObject::IsNoneObject(args.GetArg("end_time"))) { + PX_ASSIGN_OR_RETURN(ExpressionIR * end_time, GetArgAs(ast, args, "end_time")); + PX_ASSIGN_OR_RETURN(auto end_time_ns, + ParseAllTimeFormats(compiler_state->time_now().val, end_time)); + mem_source_op->SetTimeStopNS(end_time_ns); + } + return Dataframe::Create(compiler_state, mem_source_op, visitor); } - return Dataframe::Create(compiler_state, mem_source_op, visitor); } StatusOr> ProcessCols(IR* graph, const pypa::AstPtr& ast, QLObjectPtr obj, @@ -423,8 +555,8 @@ Status Dataframe::Init() { PX_ASSIGN_OR_RETURN( std::shared_ptr constructor_fn, FuncObject::Create( - name(), {"table", "select", "start_time", "end_time"}, - {{"select", "[]"}, {"start_time", "None"}, {"end_time", "None"}}, + name(), {"table", "select", "start_time", "end_time", "clickhouse_dsn", "clickhouse_ts_col"}, + {{"select", "[]"}, {"start_time", "None"}, {"end_time", "None"}, {"clickhouse_dsn", "None"}, {"clickhouse_ts_col", "None"}}, /* has_variable_len_args */ false, /* has_variable_len_kwargs */ false, std::bind(&DataFrameConstructor, compiler_state_, graph(), std::placeholders::_1, diff --git a/src/carnot/planner/objects/otel.cc b/src/carnot/planner/objects/otel.cc index 7f79d6196bb..6f4b0d3410f 100644 --- a/src/carnot/planner/objects/otel.cc +++ b/src/carnot/planner/objects/otel.cc @@ -18,12 +18,14 @@ #include "src/carnot/planner/objects/otel.h" #include +#include #include #include #include #include +#include "src/carnot/planner/ir/clickhouse_export_sink_ir.h" #include "src/carnot/planner/ir/otel_export_sink_ir.h" #include "src/carnot/planner/objects/dataframe.h" #include "src/carnot/planner/objects/dict_object.h" @@ -70,6 +72,12 @@ Status ExportToOTel(const OTelData& data, const pypa::AstPtr& ast, Dataframe* df return op->graph()->CreateNode(ast, op, data).status(); } +Status ExportToClickHouse(const std::string& table_name, const std::string& clickhouse_dsn, + const pypa::AstPtr& ast, Dataframe* df) { + auto op = df->op(); + return op->graph()->CreateNode(ast, op, table_name, clickhouse_dsn).status(); +} + StatusOr GetArgAsString(const pypa::AstPtr& ast, const ParsedArgs& args, std::string_view arg_name) { PX_ASSIGN_OR_RETURN(StringIR * arg_ir, GetArgAs(ast, args, arg_name)); @@ -100,6 +108,40 @@ StatusOr> OTelDataContainer::Create( return std::shared_ptr(new OTelDataContainer(ast_visitor, std::move(data))); } +StatusOr> ClickHouseRows::Create( + ASTVisitor* ast_visitor, const std::string& table_name) { + return std::shared_ptr(new ClickHouseRows(ast_visitor, table_name)); +} + +StatusOr ClickHouseRowsDefinition(CompilerState* compiler_state, + const pypa::AstPtr& ast, const ParsedArgs& args, + ASTVisitor* visitor) { + PX_ASSIGN_OR_RETURN(StringIR* table_name_ir, GetArgAs(ast, args, "table")); + std::string table_name = table_name_ir->str(); + + // Parse endpoint config to get the ClickHouse DSN from the URL field + std::string clickhouse_dsn; + QLObjectPtr endpoint = args.GetArg("endpoint"); + if (NoneObject::IsNoneObject(endpoint)) { + if (!compiler_state->endpoint_config()) { + return endpoint->CreateError("no default config found for endpoint, please specify one"); + } + clickhouse_dsn = compiler_state->endpoint_config()->url(); + } else { + if (endpoint->type() != EndpointConfig::EndpointType.type()) { + return endpoint->CreateError("expected Endpoint type for 'endpoint' arg, received $0", + endpoint->name()); + } + auto endpoint_config = static_cast(endpoint.get()); + clickhouse_dsn = endpoint_config->url(); + } + + return Exporter::Create(visitor, [table_name, clickhouse_dsn](auto&& ast_arg, auto&& df) -> Status { + return ExportToClickHouse(table_name, clickhouse_dsn, std::forward(ast_arg), + std::forward(df)); + }); +} + StatusOr> ParseAttributes(DictObject* attributes) { auto values = attributes->values(); auto keys = attributes->keys(); @@ -339,6 +381,17 @@ Status OTelModule::Init(CompilerState* compiler_state, IR* ir) { AddMethod(kEndpointOpID, endpoint_fn); PX_RETURN_IF_ERROR(endpoint_fn->SetDocString(kEndpointOpDocstring)); + PX_ASSIGN_OR_RETURN( + std::shared_ptr clickhouse_rows_fn, + FuncObject::Create(kClickHouseRowsOpID, {"table", "endpoint"}, {{"endpoint", "None"}}, + /* has_variable_len_args */ false, + /* has_variable_len_kwargs */ false, + std::bind(&ClickHouseRowsDefinition, compiler_state, std::placeholders::_1, + std::placeholders::_2, std::placeholders::_3), + ast_visitor())); + AddMethod(kClickHouseRowsOpID, clickhouse_rows_fn); + PX_RETURN_IF_ERROR(clickhouse_rows_fn->SetDocString(kClickHouseRowsOpDocstring)); + return Status::OK(); } diff --git a/src/carnot/planner/objects/otel.h b/src/carnot/planner/objects/otel.h index 5f4c1d19eb7..9cb96ea325c 100644 --- a/src/carnot/planner/objects/otel.h +++ b/src/carnot/planner/objects/otel.h @@ -87,6 +87,24 @@ class OTelModule : public QLObject { timeout (int, optional): The number of seconds before the request should timeout when exporting to the OTel collector. )doc"; + inline static constexpr char kClickHouseRowsOpID[] = "ClickHouseRows"; + inline static constexpr char kClickHouseRowsOpDocstring[] = R"doc( + Specifies a ClickHouse table to export DataFrame rows to. + + Describes the table name in ClickHouse where columnar DataFrame data will be + inserted. All columns from the DataFrame will be mapped to corresponding + columns in the ClickHouse table. Passed as the data argument to `px.export`. + + :topic: otel + + Args: + table (string): The name of the ClickHouse table to insert data into. + + Returns: + ClickHouseRows: Configuration for exporting DataFrame data to ClickHouse. + Can be passed to `px.export`. + )doc"; + protected: explicit OTelModule(ASTVisitor* ast_visitor) : QLObject(OTelModuleType, ast_visitor) {} Status Init(CompilerState* compiler_state, IR* ir); @@ -228,6 +246,8 @@ class EndpointConfig : public QLObject { Status ToProto(planpb::OTelEndpointConfig* endpoint_config); + const std::string& url() const { return url_; } + protected: EndpointConfig(ASTVisitor* ast_visitor, std::string url, std::vector attributes, bool insecure, @@ -269,6 +289,30 @@ class OTelDataContainer : public QLObject { std::variant data_; }; +class ClickHouseRows : public QLObject { + public: + static constexpr TypeDescriptor ClickHouseRowsType = { + /* name */ "ClickHouseRows", + /* type */ QLObjectType::kClickHouseRows, + }; + + static StatusOr> Create( + ASTVisitor* ast_visitor, const std::string& table_name); + + static bool IsClickHouseRows(const QLObjectPtr& obj) { + return obj->type() == ClickHouseRowsType.type(); + } + + const std::string& table_name() const { return table_name_; } + + protected: + ClickHouseRows(ASTVisitor* ast_visitor, std::string table_name) + : QLObject(ClickHouseRowsType, ast_visitor), table_name_(std::move(table_name)) {} + + private: + std::string table_name_; +}; + } // namespace compiler } // namespace planner } // namespace carnot diff --git a/src/carnot/planner/objects/qlobject.h b/src/carnot/planner/objects/qlobject.h index 4231fb78b0e..0ebf03da257 100644 --- a/src/carnot/planner/objects/qlobject.h +++ b/src/carnot/planner/objects/qlobject.h @@ -66,6 +66,7 @@ enum class QLObjectType { kExporter, kOTelEndpoint, kOTelDataContainer, + kClickHouseRows, }; std::string QLObjectTypeString(QLObjectType type); diff --git a/src/carnot/planner/plannerpb/service.pb.go b/src/carnot/planner/plannerpb/service.pb.go index 172eeb1cd81..c7a23641ce0 100755 --- a/src/carnot/planner/plannerpb/service.pb.go +++ b/src/carnot/planner/plannerpb/service.pb.go @@ -146,6 +146,7 @@ func (m *FuncToExecute_ArgValue) GetValue() string { type Configs struct { OTelEndpointConfig *Configs_OTelEndpointConfig `protobuf:"bytes,1,opt,name=otel_endpoint_config,json=otelEndpointConfig,proto3" json:"otel_endpoint_config,omitempty"` PluginConfig *Configs_PluginConfig `protobuf:"bytes,2,opt,name=plugin_config,json=pluginConfig,proto3" json:"plugin_config,omitempty"` + ClickhouseConfig *Configs_ClickHouseConfig `protobuf:"bytes,3,opt,name=clickhouse_config,json=clickhouseConfig,proto3" json:"clickhouse_config,omitempty"` } func (m *Configs) Reset() { *m = Configs{} } @@ -194,6 +195,13 @@ func (m *Configs) GetPluginConfig() *Configs_PluginConfig { return nil } +func (m *Configs) GetClickhouseConfig() *Configs_ClickHouseConfig { + if m != nil { + return m.ClickhouseConfig + } + return nil +} + type Configs_OTelEndpointConfig struct { URL string `protobuf:"bytes,1,opt,name=url,proto3" json:"url,omitempty"` Headers map[string]string `protobuf:"bytes,2,rep,name=headers,proto3" json:"headers,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` @@ -312,6 +320,89 @@ func (m *Configs_PluginConfig) GetEndTimeNs() int64 { return 0 } +type Configs_ClickHouseConfig struct { + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"` + Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + Username string `protobuf:"bytes,4,opt,name=username,proto3" json:"username,omitempty"` + Password string `protobuf:"bytes,5,opt,name=password,proto3" json:"password,omitempty"` + Database string `protobuf:"bytes,6,opt,name=database,proto3" json:"database,omitempty"` +} + +func (m *Configs_ClickHouseConfig) Reset() { *m = Configs_ClickHouseConfig{} } +func (*Configs_ClickHouseConfig) ProtoMessage() {} +func (*Configs_ClickHouseConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_710b3465b5cdfdeb, []int{1, 2} +} +func (m *Configs_ClickHouseConfig) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *Configs_ClickHouseConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_Configs_ClickHouseConfig.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *Configs_ClickHouseConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_Configs_ClickHouseConfig.Merge(m, src) +} +func (m *Configs_ClickHouseConfig) XXX_Size() int { + return m.Size() +} +func (m *Configs_ClickHouseConfig) XXX_DiscardUnknown() { + xxx_messageInfo_Configs_ClickHouseConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_Configs_ClickHouseConfig proto.InternalMessageInfo + +func (m *Configs_ClickHouseConfig) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *Configs_ClickHouseConfig) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *Configs_ClickHouseConfig) GetPort() int32 { + if m != nil { + return m.Port + } + return 0 +} + +func (m *Configs_ClickHouseConfig) GetUsername() string { + if m != nil { + return m.Username + } + return "" +} + +func (m *Configs_ClickHouseConfig) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + +func (m *Configs_ClickHouseConfig) GetDatabase() string { + if m != nil { + return m.Database + } + return "" +} + type QueryRequest struct { LogicalPlannerState *distributedpb.LogicalPlannerState `protobuf:"bytes,5,opt,name=logical_planner_state,json=logicalPlannerState,proto3" json:"logical_planner_state,omitempty"` QueryStr string `protobuf:"bytes,1,opt,name=query_str,json=queryStr,proto3" json:"query_str,omitempty"` @@ -857,6 +948,7 @@ func init() { proto.RegisterType((*Configs_OTelEndpointConfig)(nil), "px.carnot.planner.plannerpb.Configs.OTelEndpointConfig") proto.RegisterMapType((map[string]string)(nil), "px.carnot.planner.plannerpb.Configs.OTelEndpointConfig.HeadersEntry") proto.RegisterType((*Configs_PluginConfig)(nil), "px.carnot.planner.plannerpb.Configs.PluginConfig") + proto.RegisterType((*Configs_ClickHouseConfig)(nil), "px.carnot.planner.plannerpb.Configs.ClickHouseConfig") proto.RegisterType((*QueryRequest)(nil), "px.carnot.planner.plannerpb.QueryRequest") proto.RegisterType((*QueryResponse)(nil), "px.carnot.planner.plannerpb.QueryResponse") proto.RegisterType((*CompileMutationsRequest)(nil), "px.carnot.planner.plannerpb.CompileMutationsRequest") @@ -873,77 +965,83 @@ func init() { } var fileDescriptor_710b3465b5cdfdeb = []byte{ - // 1108 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x56, 0x51, 0x6f, 0x1b, 0xc5, - 0x13, 0xf7, 0xd9, 0x69, 0x63, 0x8f, 0x9d, 0xfe, 0xd3, 0x4d, 0xfe, 0xe0, 0xba, 0xe2, 0x12, 0x9d, - 0x0a, 0x0a, 0x01, 0xce, 0x90, 0x06, 0x82, 0x2a, 0x01, 0xc2, 0x4d, 0x20, 0x54, 0xa5, 0x84, 0x4b, - 0xda, 0x87, 0xaa, 0xe2, 0x74, 0xbe, 0x9b, 0xb8, 0x27, 0xce, 0x7b, 0xd7, 0xdd, 0xbd, 0xca, 0xe1, - 0x85, 0x16, 0x89, 0x77, 0x24, 0xbe, 0x02, 0x42, 0x20, 0x3e, 0x03, 0xef, 0x3c, 0xe6, 0xb1, 0x4f, - 0x11, 0x71, 0x24, 0xc4, 0x63, 0x3f, 0x02, 0xda, 0xdd, 0xbb, 0xc4, 0x49, 0xdc, 0xc4, 0x89, 0x78, - 0xe4, 0xe9, 0x66, 0x67, 0x67, 0x7e, 0x33, 0xfb, 0x9b, 0x99, 0xdd, 0x83, 0x79, 0xce, 0xfc, 0xa6, - 0xef, 0x31, 0x1a, 0x8b, 0x66, 0x12, 0x79, 0x94, 0x22, 0xcb, 0xbf, 0x49, 0xbb, 0xc9, 0x91, 0x3d, - 0x0e, 0x7d, 0xb4, 0x13, 0x16, 0x8b, 0x98, 0x5c, 0x4d, 0x7a, 0xb6, 0x36, 0xb5, 0x33, 0x13, 0x7b, - 0xdf, 0xb4, 0xf1, 0xc1, 0x10, 0xa0, 0x60, 0x8b, 0x7a, 0xdd, 0xd0, 0x77, 0x05, 0xf3, 0xfc, 0x90, - 0x76, 0x9a, 0x21, 0x6b, 0x46, 0x71, 0x27, 0xf4, 0xbd, 0x28, 0x69, 0xe7, 0x92, 0xc6, 0x6e, 0xbc, - 0xaa, 0xdc, 0xe3, 0x6e, 0x37, 0xa6, 0xcd, 0xb6, 0xc7, 0xb1, 0xc9, 0x85, 0x27, 0x52, 0x2e, 0x73, - 0x50, 0x42, 0x66, 0x36, 0xdd, 0x89, 0x3b, 0xb1, 0x12, 0x9b, 0x52, 0xca, 0xb4, 0x4b, 0xc3, 0x62, - 0x87, 0x5c, 0xb0, 0xb0, 0x9d, 0x0a, 0x0c, 0x92, 0xf6, 0xe0, 0xca, 0x95, 0x16, 0xda, 0xd1, 0xfa, - 0xcb, 0x80, 0x89, 0x4f, 0x52, 0xea, 0x6f, 0xc4, 0x2b, 0x3d, 0xf4, 0x53, 0x81, 0xe4, 0x2a, 0x54, - 0x36, 0x53, 0xea, 0xbb, 0xd4, 0xeb, 0x62, 0xdd, 0x98, 0x35, 0xe6, 0x2a, 0x4e, 0x59, 0x2a, 0xee, - 0x78, 0x5d, 0x24, 0x0e, 0x80, 0xc7, 0x3a, 0xee, 0x63, 0x2f, 0x4a, 0x91, 0xd7, 0x8b, 0xb3, 0xa5, - 0xb9, 0xea, 0xc2, 0x75, 0xfb, 0x04, 0x56, 0xec, 0x43, 0xe0, 0xf6, 0xc7, 0xac, 0x73, 0x4f, 0xfa, - 0x3a, 0x15, 0x2f, 0x93, 0x38, 0xb1, 0x61, 0x2a, 0x4e, 0x45, 0x92, 0x0a, 0x57, 0x78, 0xed, 0x08, - 0xdd, 0x84, 0xe1, 0x66, 0xd8, 0xab, 0x97, 0x54, 0xe8, 0xcb, 0x7a, 0x6b, 0x43, 0xee, 0xac, 0xa9, - 0x8d, 0xc6, 0x22, 0x94, 0x73, 0x18, 0x42, 0x60, 0x6c, 0x20, 0x4f, 0x25, 0x93, 0x69, 0xb8, 0xa0, - 0xf2, 0xab, 0x17, 0x95, 0x52, 0x2f, 0xac, 0xdf, 0xc7, 0x60, 0xfc, 0x66, 0x4c, 0x37, 0xc3, 0x0e, - 0x27, 0x4f, 0x0d, 0x98, 0x8e, 0x05, 0x46, 0x2e, 0xd2, 0x20, 0x89, 0x43, 0x2a, 0x5c, 0x5f, 0xed, - 0x28, 0x98, 0xea, 0xc2, 0xd2, 0x89, 0x07, 0xca, 0x40, 0xec, 0x2f, 0x36, 0x30, 0x5a, 0xc9, 0xfc, - 0xb5, 0xae, 0xf5, 0x52, 0x7f, 0x67, 0x86, 0x1c, 0xd7, 0x3b, 0x44, 0x06, 0x3b, 0xac, 0x23, 0xf7, - 0x60, 0x22, 0x89, 0xd2, 0x4e, 0x48, 0xf3, 0xd8, 0x45, 0x15, 0xfb, 0x9d, 0x91, 0x62, 0xaf, 0x29, - 0xcf, 0x0c, 0xbd, 0x96, 0x0c, 0xac, 0x1a, 0x4f, 0x8b, 0x30, 0x24, 0x05, 0x72, 0x05, 0x4a, 0x29, - 0x8b, 0x34, 0x4f, 0xad, 0xf1, 0xfe, 0xce, 0x4c, 0xe9, 0xae, 0x73, 0xdb, 0x91, 0x3a, 0xf2, 0x15, - 0x8c, 0x3f, 0x44, 0x2f, 0x40, 0x96, 0x17, 0x74, 0xf9, 0x9c, 0xe7, 0xb7, 0x57, 0x35, 0xcc, 0x0a, - 0x15, 0x6c, 0xcb, 0xc9, 0x41, 0x49, 0x03, 0xca, 0x21, 0xe5, 0xe8, 0xa7, 0x0c, 0x55, 0x51, 0xcb, - 0xce, 0xfe, 0x9a, 0xd4, 0x61, 0x5c, 0x84, 0x5d, 0x8c, 0x53, 0x51, 0x1f, 0x9b, 0x35, 0xe6, 0x4a, - 0x4e, 0xbe, 0x6c, 0xdc, 0x80, 0xda, 0x20, 0x1c, 0x99, 0x84, 0xd2, 0xd7, 0xb8, 0x95, 0x15, 0x5a, - 0x8a, 0xc3, 0xeb, 0x7c, 0xa3, 0xf8, 0xbe, 0xd1, 0x70, 0xa0, 0x36, 0xc8, 0x10, 0xb1, 0x60, 0x82, - 0x0b, 0x8f, 0x09, 0x57, 0x82, 0xbb, 0x94, 0x2b, 0x94, 0x92, 0x53, 0x55, 0xca, 0x8d, 0xb0, 0x8b, - 0x77, 0x38, 0x31, 0xa1, 0x8a, 0x34, 0xd8, 0xb7, 0x28, 0x2a, 0x8b, 0x0a, 0xd2, 0x40, 0xef, 0x5b, - 0x3f, 0x17, 0xa1, 0xf6, 0x65, 0x8a, 0x6c, 0xcb, 0xc1, 0x47, 0x29, 0x72, 0x41, 0x1e, 0xc2, 0xff, - 0xb3, 0x01, 0x76, 0x33, 0x72, 0x5c, 0x39, 0xa8, 0x58, 0xbf, 0xa0, 0x0a, 0xb9, 0x38, 0x84, 0xc4, - 0x43, 0x13, 0x69, 0xdf, 0xd6, 0xde, 0x6b, 0x7a, 0x73, 0x5d, 0xfa, 0x3a, 0x53, 0xd1, 0x71, 0xa5, - 0x9c, 0xc8, 0x47, 0x32, 0xb2, 0xcb, 0x05, 0xcb, 0x27, 0x52, 0x29, 0xd6, 0x05, 0x23, 0x9f, 0x01, - 0x60, 0x0f, 0x7d, 0x57, 0x8e, 0x28, 0xaf, 0x97, 0x54, 0x01, 0xe7, 0x47, 0x9f, 0x48, 0xa7, 0x22, - 0xbd, 0xa5, 0x8a, 0x93, 0x0f, 0x61, 0x5c, 0xf7, 0x22, 0x57, 0xc5, 0xa8, 0x2e, 0x5c, 0x1b, 0xa5, - 0x11, 0x9c, 0xdc, 0xe9, 0xd6, 0x58, 0xb9, 0x38, 0x59, 0xb2, 0xbe, 0x33, 0x60, 0x22, 0x23, 0x8a, - 0x27, 0x31, 0xe5, 0x48, 0xde, 0x80, 0x8b, 0xfa, 0x0a, 0xcb, 0xe6, 0x6b, 0x4a, 0xc2, 0xe6, 0xb7, - 0x9b, 0xbd, 0xae, 0x04, 0x27, 0x33, 0x21, 0xcb, 0x30, 0x26, 0x43, 0x64, 0xe3, 0xf0, 0xf6, 0xa9, - 0x2c, 0x2e, 0x1f, 0xac, 0x24, 0x69, 0x8e, 0xf2, 0xb6, 0x7e, 0x2b, 0xc2, 0xcb, 0x37, 0xe3, 0x6e, - 0x12, 0x46, 0xf8, 0x79, 0x2a, 0x3c, 0x11, 0xc6, 0x94, 0xff, 0x57, 0xb8, 0x17, 0x14, 0xce, 0x7a, - 0x0d, 0x26, 0x97, 0x31, 0x42, 0x81, 0x1b, 0xcc, 0xf3, 0x51, 0x4d, 0xf4, 0xb0, 0x9b, 0xd5, 0x7a, - 0x00, 0x35, 0xed, 0x7b, 0x37, 0x09, 0xe4, 0xf9, 0x46, 0x9c, 0x49, 0x72, 0x0d, 0x2e, 0x79, 0x1d, - 0xa4, 0xc2, 0x4d, 0xe2, 0x40, 0xbf, 0x2b, 0xfa, 0x72, 0xaf, 0x29, 0xed, 0x5a, 0x1c, 0xc8, 0xb7, - 0xc5, 0xfa, 0xb5, 0x08, 0xff, 0x3b, 0x52, 0x33, 0x72, 0x1f, 0x2e, 0xc8, 0xa7, 0x13, 0xb3, 0x76, - 0x68, 0x0d, 0xab, 0xcd, 0xe1, 0x27, 0xd6, 0x0e, 0x99, 0x9d, 0x3f, 0xac, 0x07, 0xc7, 0x59, 0xc6, - 0x24, 0x8a, 0xb7, 0xba, 0x48, 0xc5, 0x6a, 0xc1, 0xd1, 0x90, 0xe4, 0x01, 0x5c, 0x0e, 0xd4, 0xa9, - 0x95, 0xab, 0xb6, 0x53, 0x89, 0x55, 0x17, 0xde, 0x3a, 0x91, 0xbf, 0xa3, 0x5c, 0xad, 0x16, 0x9c, - 0xc9, 0xe0, 0x28, 0x7f, 0x6b, 0x30, 0xa1, 0xe9, 0x75, 0x53, 0x45, 0x56, 0x56, 0x99, 0xd7, 0x47, - 0xa8, 0x8c, 0x66, 0x77, 0xb5, 0xe0, 0xd4, 0xfc, 0x81, 0x75, 0x0b, 0xa0, 0xdc, 0xcd, 0x78, 0xb1, - 0x7e, 0x34, 0xa0, 0x7e, 0xbc, 0xbf, 0xcf, 0x33, 0x6f, 0xb7, 0xa0, 0x92, 0xa3, 0xe6, 0xf7, 0xff, - 0x9b, 0xa7, 0xe4, 0x78, 0x28, 0xac, 0x73, 0xe0, 0x6e, 0xfd, 0x64, 0xc0, 0x95, 0x4f, 0x91, 0x22, - 0xf3, 0x04, 0xca, 0xe7, 0x61, 0xdd, 0x67, 0x61, 0x22, 0x4e, 0x9d, 0x3b, 0xe3, 0xdf, 0x9e, 0xbb, - 0x57, 0x00, 0x92, 0x5e, 0xe4, 0x72, 0x15, 0x3e, 0x6b, 0xc5, 0x4a, 0xd2, 0xcb, 0xf2, 0xb1, 0xbe, - 0x81, 0xc6, 0xb0, 0x2c, 0xcf, 0xc3, 0x5e, 0x13, 0xaa, 0xea, 0x47, 0x62, 0x30, 0x54, 0xeb, 0x52, - 0x7f, 0x67, 0x06, 0x06, 0x90, 0x41, 0x9a, 0x68, 0x79, 0xe1, 0x49, 0x09, 0x2e, 0xe5, 0xb9, 0xea, - 0x5f, 0x4b, 0x82, 0x72, 0xaa, 0x14, 0xa7, 0xea, 0xda, 0x24, 0x27, 0xb7, 0xc8, 0xe0, 0x1b, 0xd4, - 0x98, 0x1f, 0xc5, 0x34, 0x3b, 0xd7, 0xb7, 0x30, 0x79, 0xb4, 0x63, 0xc8, 0xe2, 0x59, 0x2a, 0x9d, - 0x5f, 0xa0, 0x8d, 0x77, 0xcf, 0xe8, 0x95, 0x25, 0xf0, 0xbd, 0x01, 0xe4, 0x38, 0xef, 0xe4, 0xbd, - 0x13, 0xd1, 0x5e, 0xd8, 0x4e, 0x8d, 0xa5, 0x33, 0xfb, 0xe9, 0x3c, 0x5a, 0x1f, 0x6d, 0xef, 0x9a, - 0x85, 0x67, 0xbb, 0x66, 0xe1, 0xf9, 0xae, 0x69, 0x3c, 0xe9, 0x9b, 0xc6, 0x2f, 0x7d, 0xd3, 0xf8, - 0xa3, 0x6f, 0x1a, 0xdb, 0x7d, 0xd3, 0xf8, 0xb3, 0x6f, 0x1a, 0x7f, 0xf7, 0xcd, 0xc2, 0xf3, 0xbe, - 0x69, 0xfc, 0xb0, 0x67, 0x16, 0xb6, 0xf7, 0xcc, 0xc2, 0xb3, 0x3d, 0xb3, 0x70, 0xbf, 0xb2, 0x8f, - 0xdd, 0xbe, 0xa8, 0x7e, 0x9d, 0xaf, 0xff, 0x13, 0x00, 0x00, 0xff, 0xff, 0xa7, 0x05, 0x48, 0x4a, - 0x3a, 0x0c, 0x00, 0x00, + // 1205 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xec, 0x57, 0x5b, 0x6f, 0x1b, 0xc5, + 0x17, 0xf7, 0xda, 0xb9, 0xd8, 0xc7, 0x4e, 0xff, 0xee, 0xb4, 0x7f, 0x70, 0xb7, 0x62, 0x5b, 0xad, + 0x0a, 0x0a, 0x01, 0xd6, 0x90, 0xa6, 0x04, 0x55, 0x02, 0x84, 0x93, 0x40, 0xa8, 0x4a, 0x09, 0x9b, + 0xb4, 0x0f, 0x55, 0xc5, 0x6a, 0xbd, 0x3b, 0x71, 0x56, 0x5d, 0xef, 0x6c, 0x67, 0x66, 0x8b, 0xc3, + 0x0b, 0x2d, 0x12, 0xef, 0x48, 0x7c, 0x05, 0x84, 0xb8, 0x7c, 0x11, 0x9e, 0x50, 0x1e, 0xfb, 0x14, + 0x11, 0x47, 0x42, 0x3c, 0xf6, 0x23, 0xa0, 0xb9, 0x6c, 0xe2, 0x24, 0x6e, 0xe2, 0x44, 0x3c, 0xf2, + 0xe4, 0x73, 0xfd, 0x9d, 0x33, 0xe7, 0x9c, 0x39, 0xb3, 0x86, 0x19, 0x46, 0x83, 0x66, 0xe0, 0xd3, + 0x84, 0xf0, 0x66, 0x1a, 0xfb, 0x49, 0x82, 0x69, 0xfe, 0x9b, 0xb6, 0x9b, 0x0c, 0xd3, 0xc7, 0x51, + 0x80, 0x9d, 0x94, 0x12, 0x4e, 0xd0, 0xe5, 0xb4, 0xe7, 0x28, 0x53, 0x47, 0x9b, 0x38, 0x7b, 0xa6, + 0xe6, 0xfb, 0x43, 0x80, 0xc2, 0xcd, 0xc4, 0xef, 0x46, 0x81, 0xc7, 0xa9, 0x1f, 0x44, 0x49, 0xa7, + 0x19, 0xd1, 0x66, 0x4c, 0x3a, 0x51, 0xe0, 0xc7, 0x69, 0x3b, 0xa7, 0x14, 0xb6, 0xf9, 0xaa, 0x74, + 0x27, 0xdd, 0x2e, 0x49, 0x9a, 0x6d, 0x9f, 0xe1, 0x26, 0xe3, 0x3e, 0xcf, 0x98, 0xc8, 0x41, 0x12, + 0xda, 0xec, 0x62, 0x87, 0x74, 0x88, 0x24, 0x9b, 0x82, 0xd2, 0xd2, 0xf9, 0x61, 0xb1, 0x23, 0xc6, + 0x69, 0xd4, 0xce, 0x38, 0x0e, 0xd3, 0xf6, 0x20, 0xe7, 0x09, 0x0b, 0xe5, 0x68, 0xff, 0x65, 0xc0, + 0xd4, 0xc7, 0x59, 0x12, 0xac, 0x91, 0xa5, 0x1e, 0x0e, 0x32, 0x8e, 0xd1, 0x65, 0xa8, 0xac, 0x67, + 0x49, 0xe0, 0x25, 0x7e, 0x17, 0x37, 0x8c, 0xab, 0xc6, 0x74, 0xc5, 0x2d, 0x0b, 0xc1, 0x1d, 0xbf, + 0x8b, 0x91, 0x0b, 0xe0, 0xd3, 0x8e, 0xf7, 0xd8, 0x8f, 0x33, 0xcc, 0x1a, 0xc5, 0xab, 0xa5, 0xe9, + 0xea, 0xec, 0x75, 0xe7, 0x98, 0xaa, 0x38, 0x07, 0xc0, 0x9d, 0x8f, 0x68, 0xe7, 0x9e, 0xf0, 0x75, + 0x2b, 0xbe, 0xa6, 0x18, 0x72, 0xe0, 0x02, 0xc9, 0x78, 0x9a, 0x71, 0x8f, 0xfb, 0xed, 0x18, 0x7b, + 0x29, 0xc5, 0xeb, 0x51, 0xaf, 0x51, 0x92, 0xa1, 0xcf, 0x2b, 0xd5, 0x9a, 0xd0, 0xac, 0x48, 0x85, + 0x39, 0x07, 0xe5, 0x1c, 0x06, 0x21, 0x18, 0x1b, 0xc8, 0x53, 0xd2, 0xe8, 0x22, 0x8c, 0xcb, 0xfc, + 0x1a, 0x45, 0x29, 0x54, 0x8c, 0xfd, 0xc7, 0x04, 0x4c, 0x2e, 0x90, 0x64, 0x3d, 0xea, 0x30, 0xf4, + 0xd4, 0x80, 0x8b, 0x84, 0xe3, 0xd8, 0xc3, 0x49, 0x98, 0x92, 0x28, 0xe1, 0x5e, 0x20, 0x35, 0x12, + 0xa6, 0x3a, 0x3b, 0x7f, 0xec, 0x81, 0x34, 0x88, 0xf3, 0xf9, 0x1a, 0x8e, 0x97, 0xb4, 0xbf, 0x92, + 0xb5, 0x5e, 0xea, 0x6f, 0x5f, 0x41, 0x47, 0xe5, 0x2e, 0x12, 0xc1, 0x0e, 0xca, 0xd0, 0x3d, 0x98, + 0x4a, 0xe3, 0xac, 0x13, 0x25, 0x79, 0xec, 0xa2, 0x8c, 0xfd, 0xce, 0x48, 0xb1, 0x57, 0xa4, 0xa7, + 0x46, 0xaf, 0xa5, 0x03, 0x1c, 0x6a, 0xc3, 0xf9, 0x20, 0x8e, 0x82, 0x87, 0x1b, 0x24, 0x63, 0x38, + 0xc7, 0x2e, 0x49, 0xec, 0x1b, 0x23, 0x61, 0x2f, 0x08, 0xef, 0x65, 0xe1, 0xad, 0xf1, 0xeb, 0xfb, + 0x78, 0x4a, 0x62, 0x3e, 0x2d, 0xc2, 0x90, 0x63, 0xa2, 0x4b, 0x50, 0xca, 0x68, 0xac, 0x7a, 0xd1, + 0x9a, 0xec, 0x6f, 0x5f, 0x29, 0xdd, 0x75, 0x6f, 0xbb, 0x42, 0x86, 0xbe, 0x84, 0xc9, 0x0d, 0xec, + 0x87, 0x98, 0xe6, 0x43, 0xb3, 0x78, 0xc6, 0x1a, 0x3b, 0xcb, 0x0a, 0x66, 0x29, 0xe1, 0x74, 0xd3, + 0xcd, 0x41, 0x91, 0x09, 0xe5, 0x28, 0x61, 0x38, 0xc8, 0x28, 0x96, 0x87, 0x2d, 0xbb, 0x7b, 0x3c, + 0x6a, 0xc0, 0x24, 0x8f, 0xba, 0x98, 0x64, 0xbc, 0x31, 0x76, 0xd5, 0x98, 0x2e, 0xb9, 0x39, 0x6b, + 0xde, 0x84, 0xda, 0x20, 0x1c, 0xaa, 0x43, 0xe9, 0x21, 0xde, 0xd4, 0xc3, 0x24, 0xc8, 0xe1, 0xb3, + 0x74, 0xb3, 0xf8, 0x9e, 0x61, 0xba, 0x50, 0x1b, 0xec, 0x02, 0xb2, 0x61, 0x8a, 0x71, 0x9f, 0x72, + 0x4f, 0x80, 0x7b, 0x09, 0x93, 0x28, 0x25, 0xb7, 0x2a, 0x85, 0x6b, 0x51, 0x17, 0xdf, 0x61, 0xc8, + 0x82, 0x2a, 0x4e, 0xc2, 0x3d, 0x8b, 0xa2, 0xb4, 0xa8, 0xe0, 0x24, 0x54, 0x7a, 0xf3, 0x57, 0x03, + 0xea, 0x87, 0xcb, 0x2f, 0x8e, 0xb6, 0x41, 0x18, 0x1f, 0xbc, 0x8e, 0x39, 0x2f, 0xc6, 0x5f, 0xd0, + 0x3a, 0x3b, 0x49, 0x0b, 0x59, 0x4a, 0x28, 0x97, 0x65, 0x18, 0x77, 0x25, 0x2d, 0x30, 0x32, 0x86, + 0xa9, 0xc4, 0x18, 0x53, 0x18, 0x39, 0x2f, 0x74, 0xa9, 0xcf, 0xd8, 0x57, 0x84, 0x86, 0x8d, 0x71, + 0xa5, 0xcb, 0x79, 0xa1, 0x0b, 0x7d, 0xee, 0x8b, 0x75, 0xd4, 0x98, 0x50, 0xba, 0x9c, 0xb7, 0x7f, + 0x2a, 0x42, 0xed, 0x8b, 0x0c, 0xd3, 0x4d, 0x17, 0x3f, 0xca, 0x30, 0xe3, 0x68, 0x03, 0xfe, 0xaf, + 0x37, 0x9a, 0xa7, 0x3b, 0xe9, 0x89, 0xcd, 0x85, 0x25, 0x6a, 0x75, 0x76, 0x6e, 0x48, 0xc7, 0x0f, + 0xac, 0x28, 0xe7, 0xb6, 0xf2, 0x5e, 0x51, 0xca, 0x55, 0xe1, 0xeb, 0x5e, 0x88, 0x8f, 0x0a, 0xc5, + 0x8a, 0x7a, 0x24, 0x22, 0x7b, 0x8c, 0xd3, 0xbc, 0x26, 0x52, 0xb0, 0xca, 0x29, 0xfa, 0x14, 0x00, + 0xf7, 0x70, 0xe0, 0x89, 0x9d, 0xc5, 0x1a, 0x25, 0x39, 0x6d, 0x33, 0xa3, 0xaf, 0x28, 0xb7, 0x22, + 0xbc, 0x85, 0x88, 0xa1, 0x0f, 0x60, 0x52, 0x5d, 0x20, 0x26, 0xab, 0x56, 0x9d, 0xbd, 0x36, 0xca, + 0xd4, 0xba, 0xb9, 0xd3, 0xad, 0xb1, 0x72, 0xb1, 0x5e, 0xb2, 0xbf, 0x35, 0x60, 0x4a, 0x17, 0x8a, + 0xa5, 0x24, 0x61, 0x18, 0xbd, 0x01, 0x13, 0x6a, 0xa7, 0xeb, 0x85, 0x73, 0x41, 0xc0, 0xe6, 0xeb, + 0xde, 0x59, 0x95, 0x84, 0xab, 0x4d, 0xd0, 0x22, 0x8c, 0x89, 0x10, 0x7a, 0x3f, 0xbc, 0x7d, 0x62, + 0x15, 0x17, 0xf7, 0x39, 0x51, 0x34, 0x57, 0x7a, 0xdb, 0xbf, 0x15, 0xe1, 0xe5, 0x05, 0xd2, 0x4d, + 0xa3, 0x18, 0x7f, 0x96, 0x71, 0x9f, 0x47, 0x24, 0x61, 0xff, 0x35, 0xee, 0x05, 0x8d, 0xb3, 0x5f, + 0x83, 0xfa, 0x22, 0x8e, 0x31, 0xc7, 0x6b, 0xd4, 0x0f, 0xb0, 0x5c, 0x3f, 0xc3, 0x9e, 0x1a, 0xfb, + 0x01, 0xd4, 0x94, 0xef, 0xdd, 0x34, 0x14, 0xe7, 0x1b, 0x71, 0x81, 0xa0, 0x6b, 0x70, 0xce, 0xef, + 0xe0, 0x84, 0x7b, 0x29, 0x09, 0xd5, 0x43, 0xab, 0x5e, 0xbb, 0x9a, 0x94, 0xae, 0x90, 0x50, 0x3c, + 0xb6, 0xf6, 0x2f, 0x45, 0xf8, 0xdf, 0xa1, 0x9e, 0xa1, 0xfb, 0x30, 0x2e, 0xbe, 0x25, 0xb0, 0x1e, + 0x87, 0xd6, 0xb0, 0xde, 0x1c, 0xfc, 0xe6, 0x70, 0x22, 0xea, 0xe4, 0x5f, 0x1a, 0xfb, 0xc7, 0x59, + 0xc4, 0x69, 0x4c, 0x36, 0xbb, 0x38, 0xe1, 0xcb, 0x05, 0x57, 0x41, 0xa2, 0x07, 0x70, 0x3e, 0x94, + 0xa7, 0x96, 0xae, 0xca, 0x4e, 0x3f, 0x1d, 0x6f, 0x1d, 0x5b, 0xbf, 0xc3, 0xb5, 0x5a, 0x2e, 0xb8, + 0xf5, 0xf0, 0x70, 0xfd, 0x56, 0x60, 0x4a, 0x95, 0xd7, 0xcb, 0x64, 0xb1, 0x74, 0x67, 0x5e, 0x1f, + 0xa1, 0x33, 0xaa, 0xba, 0xcb, 0x05, 0xb7, 0x16, 0x0c, 0xf0, 0x2d, 0x80, 0x72, 0x57, 0xd7, 0xc5, + 0xfe, 0xc1, 0x80, 0xc6, 0xd1, 0xf9, 0x3e, 0xcb, 0x7d, 0xbb, 0x05, 0x95, 0x1c, 0x35, 0x7f, 0xac, + 0xde, 0x3c, 0x21, 0xc7, 0x03, 0x61, 0xdd, 0x7d, 0x77, 0xfb, 0x47, 0x03, 0x2e, 0x7d, 0x82, 0x13, + 0x4c, 0x7d, 0x8e, 0xc5, 0x5b, 0xb6, 0x1a, 0xd0, 0x28, 0xe5, 0x27, 0xde, 0x3b, 0xe3, 0xdf, 0xbe, + 0x77, 0xaf, 0x00, 0xa4, 0xbd, 0xd8, 0x63, 0x32, 0xbc, 0x1e, 0xc5, 0x4a, 0xda, 0xd3, 0xf9, 0xd8, + 0x5f, 0x83, 0x39, 0x2c, 0xcb, 0xb3, 0x54, 0xaf, 0x09, 0x55, 0xf9, 0x65, 0x35, 0x18, 0xaa, 0x75, + 0xae, 0xbf, 0x7d, 0x05, 0x06, 0x90, 0x41, 0x98, 0x28, 0x7a, 0xf6, 0x49, 0x09, 0xce, 0xe5, 0xb9, + 0xaa, 0x6f, 0x6d, 0x84, 0xc5, 0xad, 0x92, 0x35, 0x95, 0x6b, 0x13, 0x1d, 0x3f, 0x22, 0x83, 0x6f, + 0x90, 0x39, 0x33, 0x8a, 0xa9, 0x3e, 0xd7, 0x37, 0x50, 0x3f, 0x3c, 0x31, 0x68, 0xee, 0x34, 0x9d, + 0xce, 0x17, 0xa8, 0x79, 0xe3, 0x94, 0x5e, 0x3a, 0x81, 0xef, 0x0c, 0x40, 0x47, 0xeb, 0x8e, 0xde, + 0x3d, 0x16, 0xed, 0x85, 0xe3, 0x64, 0xce, 0x9f, 0xda, 0x4f, 0xe5, 0xd1, 0xfa, 0x70, 0x6b, 0xc7, + 0x2a, 0x3c, 0xdb, 0xb1, 0x0a, 0xcf, 0x77, 0x2c, 0xe3, 0x49, 0xdf, 0x32, 0x7e, 0xee, 0x5b, 0xc6, + 0xef, 0x7d, 0xcb, 0xd8, 0xea, 0x5b, 0xc6, 0x9f, 0x7d, 0xcb, 0xf8, 0xbb, 0x6f, 0x15, 0x9e, 0xf7, + 0x2d, 0xe3, 0xfb, 0x5d, 0xab, 0xb0, 0xb5, 0x6b, 0x15, 0x9e, 0xed, 0x5a, 0x85, 0xfb, 0x95, 0x3d, + 0xec, 0xf6, 0x84, 0xfc, 0x2f, 0x71, 0xfd, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8b, 0xee, 0x54, + 0x71, 0x4b, 0x0d, 0x00, 0x00, } func (this *FuncToExecute) Equal(that interface{}) bool { @@ -1033,6 +1131,9 @@ func (this *Configs) Equal(that interface{}) bool { if !this.PluginConfig.Equal(that1.PluginConfig) { return false } + if !this.ClickhouseConfig.Equal(that1.ClickhouseConfig) { + return false + } return true } func (this *Configs_OTelEndpointConfig) Equal(that interface{}) bool { @@ -1100,6 +1201,45 @@ func (this *Configs_PluginConfig) Equal(that interface{}) bool { } return true } +func (this *Configs_ClickHouseConfig) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Configs_ClickHouseConfig) + if !ok { + that2, ok := that.(Configs_ClickHouseConfig) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Hostname != that1.Hostname { + return false + } + if this.Host != that1.Host { + return false + } + if this.Port != that1.Port { + return false + } + if this.Username != that1.Username { + return false + } + if this.Password != that1.Password { + return false + } + if this.Database != that1.Database { + return false + } + return true +} func (this *QueryRequest) Equal(that interface{}) bool { if that == nil { return this == nil @@ -1474,7 +1614,7 @@ func (this *Configs) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 6) + s := make([]string, 0, 7) s = append(s, "&plannerpb.Configs{") if this.OTelEndpointConfig != nil { s = append(s, "OTelEndpointConfig: "+fmt.Sprintf("%#v", this.OTelEndpointConfig)+",\n") @@ -1482,6 +1622,9 @@ func (this *Configs) GoString() string { if this.PluginConfig != nil { s = append(s, "PluginConfig: "+fmt.Sprintf("%#v", this.PluginConfig)+",\n") } + if this.ClickhouseConfig != nil { + s = append(s, "ClickhouseConfig: "+fmt.Sprintf("%#v", this.ClickhouseConfig)+",\n") + } s = append(s, "}") return strings.Join(s, "") } @@ -1521,6 +1664,21 @@ func (this *Configs_PluginConfig) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *Configs_ClickHouseConfig) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&plannerpb.Configs_ClickHouseConfig{") + s = append(s, "Hostname: "+fmt.Sprintf("%#v", this.Hostname)+",\n") + s = append(s, "Host: "+fmt.Sprintf("%#v", this.Host)+",\n") + s = append(s, "Port: "+fmt.Sprintf("%#v", this.Port)+",\n") + s = append(s, "Username: "+fmt.Sprintf("%#v", this.Username)+",\n") + s = append(s, "Password: "+fmt.Sprintf("%#v", this.Password)+",\n") + s = append(s, "Database: "+fmt.Sprintf("%#v", this.Database)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *QueryRequest) GoString() string { if this == nil { return "nil" @@ -1942,6 +2100,18 @@ func (m *Configs) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if m.ClickhouseConfig != nil { + { + size, err := m.ClickhouseConfig.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintService(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } if m.PluginConfig != nil { { size, err := m.PluginConfig.MarshalToSizedBuffer(dAtA[:i]) @@ -2066,6 +2236,69 @@ func (m *Configs_PluginConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *Configs_ClickHouseConfig) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *Configs_ClickHouseConfig) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Configs_ClickHouseConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Database) > 0 { + i -= len(m.Database) + copy(dAtA[i:], m.Database) + i = encodeVarintService(dAtA, i, uint64(len(m.Database))) + i-- + dAtA[i] = 0x32 + } + if len(m.Password) > 0 { + i -= len(m.Password) + copy(dAtA[i:], m.Password) + i = encodeVarintService(dAtA, i, uint64(len(m.Password))) + i-- + dAtA[i] = 0x2a + } + if len(m.Username) > 0 { + i -= len(m.Username) + copy(dAtA[i:], m.Username) + i = encodeVarintService(dAtA, i, uint64(len(m.Username))) + i-- + dAtA[i] = 0x22 + } + if m.Port != 0 { + i = encodeVarintService(dAtA, i, uint64(m.Port)) + i-- + dAtA[i] = 0x18 + } + if len(m.Host) > 0 { + i -= len(m.Host) + copy(dAtA[i:], m.Host) + i = encodeVarintService(dAtA, i, uint64(len(m.Host))) + i-- + dAtA[i] = 0x12 + } + if len(m.Hostname) > 0 { + i -= len(m.Hostname) + copy(dAtA[i:], m.Hostname) + i = encodeVarintService(dAtA, i, uint64(len(m.Hostname))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *QueryRequest) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -2616,6 +2849,10 @@ func (m *Configs) Size() (n int) { l = m.PluginConfig.Size() n += 1 + l + sovService(uint64(l)) } + if m.ClickhouseConfig != nil { + l = m.ClickhouseConfig.Size() + n += 1 + l + sovService(uint64(l)) + } return n } @@ -2661,6 +2898,38 @@ func (m *Configs_PluginConfig) Size() (n int) { return n } +func (m *Configs_ClickHouseConfig) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Hostname) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + if m.Port != 0 { + n += 1 + sovService(uint64(m.Port)) + } + l = len(m.Username) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + l = len(m.Password) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + l = len(m.Database) + if l > 0 { + n += 1 + l + sovService(uint64(l)) + } + return n +} + func (m *QueryRequest) Size() (n int) { if m == nil { return 0 @@ -2908,6 +3177,7 @@ func (this *Configs) String() string { s := strings.Join([]string{`&Configs{`, `OTelEndpointConfig:` + strings.Replace(fmt.Sprintf("%v", this.OTelEndpointConfig), "Configs_OTelEndpointConfig", "Configs_OTelEndpointConfig", 1) + `,`, `PluginConfig:` + strings.Replace(fmt.Sprintf("%v", this.PluginConfig), "Configs_PluginConfig", "Configs_PluginConfig", 1) + `,`, + `ClickhouseConfig:` + strings.Replace(fmt.Sprintf("%v", this.ClickhouseConfig), "Configs_ClickHouseConfig", "Configs_ClickHouseConfig", 1) + `,`, `}`, }, "") return s @@ -2946,6 +3216,21 @@ func (this *Configs_PluginConfig) String() string { }, "") return s } +func (this *Configs_ClickHouseConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Configs_ClickHouseConfig{`, + `Hostname:` + fmt.Sprintf("%v", this.Hostname) + `,`, + `Host:` + fmt.Sprintf("%v", this.Host) + `,`, + `Port:` + fmt.Sprintf("%v", this.Port) + `,`, + `Username:` + fmt.Sprintf("%v", this.Username) + `,`, + `Password:` + fmt.Sprintf("%v", this.Password) + `,`, + `Database:` + fmt.Sprintf("%v", this.Database) + `,`, + `}`, + }, "") + return s +} func (this *QueryRequest) String() string { if this == nil { return "nil" @@ -3464,6 +3749,42 @@ func (m *Configs) Unmarshal(dAtA []byte) error { return err } iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseConfig", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.ClickhouseConfig == nil { + m.ClickhouseConfig = &Configs_ClickHouseConfig{} + } + if err := m.ClickhouseConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipService(dAtA[iNdEx:]) @@ -3821,6 +4142,235 @@ func (m *Configs_PluginConfig) Unmarshal(dAtA []byte) error { } return nil } +func (m *Configs_ClickHouseConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ClickHouseConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ClickHouseConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hostname", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hostname = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Host = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + } + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Username", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Username = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 5: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Password = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 6: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Database", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowService + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthService + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthService + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Database = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipService(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthService + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} func (m *QueryRequest) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 diff --git a/src/carnot/planner/plannerpb/service.proto b/src/carnot/planner/plannerpb/service.proto index a9b33d825f8..4c3fd9a99a8 100644 --- a/src/carnot/planner/plannerpb/service.proto +++ b/src/carnot/planner/plannerpb/service.proto @@ -75,6 +75,22 @@ message Configs { int64 end_time_ns = 2; } PluginConfig plugin_config = 2; + // ClickHouseConfig contains information about ClickHouse connection parameters. + message ClickHouseConfig { + // The hostname of the node executing the query. + string hostname = 1; + // The ClickHouse server host. + string host = 2; + // The ClickHouse server port. + int32 port = 3; + // The ClickHouse username. + string username = 4; + // The ClickHouse password. + string password = 5; + // The ClickHouse database name. + string database = 6; + } + ClickHouseConfig clickhouse_config = 3; } // QueryRequest is the body of the request made to the planner. diff --git a/src/carnot/planpb/plan.pb.go b/src/carnot/planpb/plan.pb.go index ce6671091c1..bb5a6584aea 100755 --- a/src/carnot/planpb/plan.pb.go +++ b/src/carnot/planpb/plan.pb.go @@ -34,20 +34,22 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type OperatorType int32 const ( - OPERATOR_TYPE_UNKNOWN OperatorType = 0 - MEMORY_SOURCE_OPERATOR OperatorType = 1000 - GRPC_SOURCE_OPERATOR OperatorType = 1100 - UDTF_SOURCE_OPERATOR OperatorType = 1200 - EMPTY_SOURCE_OPERATOR OperatorType = 1300 - MAP_OPERATOR OperatorType = 2000 - AGGREGATE_OPERATOR OperatorType = 2100 - FILTER_OPERATOR OperatorType = 2200 - LIMIT_OPERATOR OperatorType = 2300 - UNION_OPERATOR OperatorType = 2400 - JOIN_OPERATOR OperatorType = 2500 - MEMORY_SINK_OPERATOR OperatorType = 9000 - GRPC_SINK_OPERATOR OperatorType = 9100 - OTEL_EXPORT_SINK_OPERATOR OperatorType = 9200 + OPERATOR_TYPE_UNKNOWN OperatorType = 0 + MEMORY_SOURCE_OPERATOR OperatorType = 1000 + GRPC_SOURCE_OPERATOR OperatorType = 1100 + UDTF_SOURCE_OPERATOR OperatorType = 1200 + EMPTY_SOURCE_OPERATOR OperatorType = 1300 + CLICKHOUSE_SOURCE_OPERATOR OperatorType = 1400 + MAP_OPERATOR OperatorType = 2000 + AGGREGATE_OPERATOR OperatorType = 2100 + FILTER_OPERATOR OperatorType = 2200 + LIMIT_OPERATOR OperatorType = 2300 + UNION_OPERATOR OperatorType = 2400 + JOIN_OPERATOR OperatorType = 2500 + MEMORY_SINK_OPERATOR OperatorType = 9000 + GRPC_SINK_OPERATOR OperatorType = 9100 + OTEL_EXPORT_SINK_OPERATOR OperatorType = 9200 + CLICKHOUSE_EXPORT_SINK_OPERATOR OperatorType = 9300 ) var OperatorType_name = map[int32]string{ @@ -56,6 +58,7 @@ var OperatorType_name = map[int32]string{ 1100: "GRPC_SOURCE_OPERATOR", 1200: "UDTF_SOURCE_OPERATOR", 1300: "EMPTY_SOURCE_OPERATOR", + 1400: "CLICKHOUSE_SOURCE_OPERATOR", 2000: "MAP_OPERATOR", 2100: "AGGREGATE_OPERATOR", 2200: "FILTER_OPERATOR", @@ -65,23 +68,26 @@ var OperatorType_name = map[int32]string{ 9000: "MEMORY_SINK_OPERATOR", 9100: "GRPC_SINK_OPERATOR", 9200: "OTEL_EXPORT_SINK_OPERATOR", + 9300: "CLICKHOUSE_EXPORT_SINK_OPERATOR", } var OperatorType_value = map[string]int32{ - "OPERATOR_TYPE_UNKNOWN": 0, - "MEMORY_SOURCE_OPERATOR": 1000, - "GRPC_SOURCE_OPERATOR": 1100, - "UDTF_SOURCE_OPERATOR": 1200, - "EMPTY_SOURCE_OPERATOR": 1300, - "MAP_OPERATOR": 2000, - "AGGREGATE_OPERATOR": 2100, - "FILTER_OPERATOR": 2200, - "LIMIT_OPERATOR": 2300, - "UNION_OPERATOR": 2400, - "JOIN_OPERATOR": 2500, - "MEMORY_SINK_OPERATOR": 9000, - "GRPC_SINK_OPERATOR": 9100, - "OTEL_EXPORT_SINK_OPERATOR": 9200, + "OPERATOR_TYPE_UNKNOWN": 0, + "MEMORY_SOURCE_OPERATOR": 1000, + "GRPC_SOURCE_OPERATOR": 1100, + "UDTF_SOURCE_OPERATOR": 1200, + "EMPTY_SOURCE_OPERATOR": 1300, + "CLICKHOUSE_SOURCE_OPERATOR": 1400, + "MAP_OPERATOR": 2000, + "AGGREGATE_OPERATOR": 2100, + "FILTER_OPERATOR": 2200, + "LIMIT_OPERATOR": 2300, + "UNION_OPERATOR": 2400, + "JOIN_OPERATOR": 2500, + "MEMORY_SINK_OPERATOR": 9000, + "GRPC_SINK_OPERATOR": 9100, + "OTEL_EXPORT_SINK_OPERATOR": 9200, + "CLICKHOUSE_EXPORT_SINK_OPERATOR": 9300, } func (OperatorType) EnumDescriptor() ([]byte, []int) { @@ -526,6 +532,8 @@ type Operator struct { // *Operator_UdtfSourceOp // *Operator_EmptySourceOp // *Operator_OTelSinkOp + // *Operator_ClickhouseSourceOp + // *Operator_ClickhouseSinkOp Op isOperator_Op `protobuf_oneof:"op"` } @@ -607,20 +615,28 @@ type Operator_EmptySourceOp struct { type Operator_OTelSinkOp struct { OTelSinkOp *OTelExportSinkOperator `protobuf:"bytes,14,opt,name=otel_sink_op,json=otelSinkOp,proto3,oneof" json:"otel_sink_op,omitempty"` } - -func (*Operator_MemSourceOp) isOperator_Op() {} -func (*Operator_MapOp) isOperator_Op() {} -func (*Operator_AggOp) isOperator_Op() {} -func (*Operator_MemSinkOp) isOperator_Op() {} -func (*Operator_FilterOp) isOperator_Op() {} -func (*Operator_LimitOp) isOperator_Op() {} -func (*Operator_UnionOp) isOperator_Op() {} -func (*Operator_GRPCSourceOp) isOperator_Op() {} -func (*Operator_GRPCSinkOp) isOperator_Op() {} -func (*Operator_JoinOp) isOperator_Op() {} -func (*Operator_UdtfSourceOp) isOperator_Op() {} -func (*Operator_EmptySourceOp) isOperator_Op() {} -func (*Operator_OTelSinkOp) isOperator_Op() {} +type Operator_ClickhouseSourceOp struct { + ClickhouseSourceOp *ClickHouseSourceOperator `protobuf:"bytes,15,opt,name=clickhouse_source_op,json=clickhouseSourceOp,proto3,oneof" json:"clickhouse_source_op,omitempty"` +} +type Operator_ClickhouseSinkOp struct { + ClickhouseSinkOp *ClickHouseExportSinkOperator `protobuf:"bytes,16,opt,name=clickhouse_sink_op,json=clickhouseSinkOp,proto3,oneof" json:"clickhouse_sink_op,omitempty"` +} + +func (*Operator_MemSourceOp) isOperator_Op() {} +func (*Operator_MapOp) isOperator_Op() {} +func (*Operator_AggOp) isOperator_Op() {} +func (*Operator_MemSinkOp) isOperator_Op() {} +func (*Operator_FilterOp) isOperator_Op() {} +func (*Operator_LimitOp) isOperator_Op() {} +func (*Operator_UnionOp) isOperator_Op() {} +func (*Operator_GRPCSourceOp) isOperator_Op() {} +func (*Operator_GRPCSinkOp) isOperator_Op() {} +func (*Operator_JoinOp) isOperator_Op() {} +func (*Operator_UdtfSourceOp) isOperator_Op() {} +func (*Operator_EmptySourceOp) isOperator_Op() {} +func (*Operator_OTelSinkOp) isOperator_Op() {} +func (*Operator_ClickhouseSourceOp) isOperator_Op() {} +func (*Operator_ClickhouseSinkOp) isOperator_Op() {} func (m *Operator) GetOp() isOperator_Op { if m != nil { @@ -727,6 +743,20 @@ func (m *Operator) GetOTelSinkOp() *OTelExportSinkOperator { return nil } +func (m *Operator) GetClickhouseSourceOp() *ClickHouseSourceOperator { + if x, ok := m.GetOp().(*Operator_ClickhouseSourceOp); ok { + return x.ClickhouseSourceOp + } + return nil +} + +func (m *Operator) GetClickhouseSinkOp() *ClickHouseExportSinkOperator { + if x, ok := m.GetOp().(*Operator_ClickhouseSinkOp); ok { + return x.ClickhouseSinkOp + } + return nil +} + // XXX_OneofWrappers is for the internal use of the proto package. func (*Operator) XXX_OneofWrappers() []interface{} { return []interface{}{ @@ -743,6 +773,8 @@ func (*Operator) XXX_OneofWrappers() []interface{} { (*Operator_UdtfSourceOp)(nil), (*Operator_EmptySourceOp)(nil), (*Operator_OTelSinkOp)(nil), + (*Operator_ClickhouseSourceOp)(nil), + (*Operator_ClickhouseSinkOp)(nil), } } @@ -1810,6 +1842,153 @@ func (m *EmptySourceOperator) GetColumnTypes() []typespb.DataType { return nil } +type ClickHouseSourceOperator struct { + Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` + Port int32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + Username string `protobuf:"bytes,3,opt,name=username,proto3" json:"username,omitempty"` + Password string `protobuf:"bytes,4,opt,name=password,proto3" json:"password,omitempty"` + Database string `protobuf:"bytes,5,opt,name=database,proto3" json:"database,omitempty"` + Query string `protobuf:"bytes,6,opt,name=query,proto3" json:"query,omitempty"` + ColumnNames []string `protobuf:"bytes,7,rep,name=column_names,json=columnNames,proto3" json:"column_names,omitempty"` + ColumnTypes []typespb.DataType `protobuf:"varint,8,rep,packed,name=column_types,json=columnTypes,proto3,enum=px.types.DataType" json:"column_types,omitempty"` + BatchSize int32 `protobuf:"varint,9,opt,name=batch_size,json=batchSize,proto3" json:"batch_size,omitempty"` + Streaming bool `protobuf:"varint,10,opt,name=streaming,proto3" json:"streaming,omitempty"` + TimestampColumn string `protobuf:"bytes,11,opt,name=timestamp_column,json=timestampColumn,proto3" json:"timestamp_column,omitempty"` + PartitionColumn string `protobuf:"bytes,12,opt,name=partition_column,json=partitionColumn,proto3" json:"partition_column,omitempty"` + StartTime int64 `protobuf:"varint,13,opt,name=start_time,json=startTime,proto3" json:"start_time,omitempty"` + EndTime int64 `protobuf:"varint,14,opt,name=end_time,json=endTime,proto3" json:"end_time,omitempty"` +} + +func (m *ClickHouseSourceOperator) Reset() { *m = ClickHouseSourceOperator{} } +func (*ClickHouseSourceOperator) ProtoMessage() {} +func (*ClickHouseSourceOperator) Descriptor() ([]byte, []int) { + return fileDescriptor_e5dcfc8666ec3f33, []int{18} +} +func (m *ClickHouseSourceOperator) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClickHouseSourceOperator) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClickHouseSourceOperator.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClickHouseSourceOperator) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClickHouseSourceOperator.Merge(m, src) +} +func (m *ClickHouseSourceOperator) XXX_Size() int { + return m.Size() +} +func (m *ClickHouseSourceOperator) XXX_DiscardUnknown() { + xxx_messageInfo_ClickHouseSourceOperator.DiscardUnknown(m) +} + +var xxx_messageInfo_ClickHouseSourceOperator proto.InternalMessageInfo + +func (m *ClickHouseSourceOperator) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *ClickHouseSourceOperator) GetPort() int32 { + if m != nil { + return m.Port + } + return 0 +} + +func (m *ClickHouseSourceOperator) GetUsername() string { + if m != nil { + return m.Username + } + return "" +} + +func (m *ClickHouseSourceOperator) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + +func (m *ClickHouseSourceOperator) GetDatabase() string { + if m != nil { + return m.Database + } + return "" +} + +func (m *ClickHouseSourceOperator) GetQuery() string { + if m != nil { + return m.Query + } + return "" +} + +func (m *ClickHouseSourceOperator) GetColumnNames() []string { + if m != nil { + return m.ColumnNames + } + return nil +} + +func (m *ClickHouseSourceOperator) GetColumnTypes() []typespb.DataType { + if m != nil { + return m.ColumnTypes + } + return nil +} + +func (m *ClickHouseSourceOperator) GetBatchSize() int32 { + if m != nil { + return m.BatchSize + } + return 0 +} + +func (m *ClickHouseSourceOperator) GetStreaming() bool { + if m != nil { + return m.Streaming + } + return false +} + +func (m *ClickHouseSourceOperator) GetTimestampColumn() string { + if m != nil { + return m.TimestampColumn + } + return "" +} + +func (m *ClickHouseSourceOperator) GetPartitionColumn() string { + if m != nil { + return m.PartitionColumn + } + return "" +} + +func (m *ClickHouseSourceOperator) GetStartTime() int64 { + if m != nil { + return m.StartTime + } + return 0 +} + +func (m *ClickHouseSourceOperator) GetEndTime() int64 { + if m != nil { + return m.EndTime + } + return 0 +} + type OTelLog struct { Attributes []*OTelAttribute `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty"` TimeColumnIndex int64 `protobuf:"varint,2,opt,name=time_column_index,json=timeColumnIndex,proto3" json:"time_column_index,omitempty"` @@ -1822,7 +2001,7 @@ type OTelLog struct { func (m *OTelLog) Reset() { *m = OTelLog{} } func (*OTelLog) ProtoMessage() {} func (*OTelLog) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{18} + return fileDescriptor_e5dcfc8666ec3f33, []int{19} } func (m *OTelLog) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -1911,7 +2090,7 @@ type OTelSpan struct { func (m *OTelSpan) Reset() { *m = OTelSpan{} } func (*OTelSpan) ProtoMessage() {} func (*OTelSpan) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{19} + return fileDescriptor_e5dcfc8666ec3f33, []int{20} } func (m *OTelSpan) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2046,7 +2225,7 @@ type OTelMetricGauge struct { func (m *OTelMetricGauge) Reset() { *m = OTelMetricGauge{} } func (*OTelMetricGauge) ProtoMessage() {} func (*OTelMetricGauge) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{20} + return fileDescriptor_e5dcfc8666ec3f33, []int{21} } func (m *OTelMetricGauge) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2130,7 +2309,7 @@ type OTelMetricSummary struct { func (m *OTelMetricSummary) Reset() { *m = OTelMetricSummary{} } func (*OTelMetricSummary) ProtoMessage() {} func (*OTelMetricSummary) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{21} + return fileDescriptor_e5dcfc8666ec3f33, []int{22} } func (m *OTelMetricSummary) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2188,7 +2367,7 @@ type OTelMetricSummary_ValueAtQuantile struct { func (m *OTelMetricSummary_ValueAtQuantile) Reset() { *m = OTelMetricSummary_ValueAtQuantile{} } func (*OTelMetricSummary_ValueAtQuantile) ProtoMessage() {} func (*OTelMetricSummary_ValueAtQuantile) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{21, 0} + return fileDescriptor_e5dcfc8666ec3f33, []int{22, 0} } func (m *OTelMetricSummary_ValueAtQuantile) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2243,7 +2422,7 @@ type OTelAttribute struct { func (m *OTelAttribute) Reset() { *m = OTelAttribute{} } func (*OTelAttribute) ProtoMessage() {} func (*OTelAttribute) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{22} + return fileDescriptor_e5dcfc8666ec3f33, []int{23} } func (m *OTelAttribute) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2334,7 +2513,7 @@ type OTelAttribute_Column struct { func (m *OTelAttribute_Column) Reset() { *m = OTelAttribute_Column{} } func (*OTelAttribute_Column) ProtoMessage() {} func (*OTelAttribute_Column) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{22, 0} + return fileDescriptor_e5dcfc8666ec3f33, []int{23, 0} } func (m *OTelAttribute_Column) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2400,7 +2579,7 @@ type OTelMetric struct { func (m *OTelMetric) Reset() { *m = OTelMetric{} } func (*OTelMetric) ProtoMessage() {} func (*OTelMetric) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{23} + return fileDescriptor_e5dcfc8666ec3f33, []int{24} } func (m *OTelMetric) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2520,7 +2699,7 @@ type OTelEndpointConfig struct { func (m *OTelEndpointConfig) Reset() { *m = OTelEndpointConfig{} } func (*OTelEndpointConfig) ProtoMessage() {} func (*OTelEndpointConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{24} + return fileDescriptor_e5dcfc8666ec3f33, []int{25} } func (m *OTelEndpointConfig) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2577,6 +2756,89 @@ func (m *OTelEndpointConfig) GetTimeout() int64 { return 0 } +type ClickHouseConfig struct { + Hostname string `protobuf:"bytes,1,opt,name=hostname,proto3" json:"hostname,omitempty"` + Host string `protobuf:"bytes,2,opt,name=host,proto3" json:"host,omitempty"` + Port int32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + Username string `protobuf:"bytes,4,opt,name=username,proto3" json:"username,omitempty"` + Password string `protobuf:"bytes,5,opt,name=password,proto3" json:"password,omitempty"` + Database string `protobuf:"bytes,6,opt,name=database,proto3" json:"database,omitempty"` +} + +func (m *ClickHouseConfig) Reset() { *m = ClickHouseConfig{} } +func (*ClickHouseConfig) ProtoMessage() {} +func (*ClickHouseConfig) Descriptor() ([]byte, []int) { + return fileDescriptor_e5dcfc8666ec3f33, []int{26} +} +func (m *ClickHouseConfig) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClickHouseConfig) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClickHouseConfig.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClickHouseConfig) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClickHouseConfig.Merge(m, src) +} +func (m *ClickHouseConfig) XXX_Size() int { + return m.Size() +} +func (m *ClickHouseConfig) XXX_DiscardUnknown() { + xxx_messageInfo_ClickHouseConfig.DiscardUnknown(m) +} + +var xxx_messageInfo_ClickHouseConfig proto.InternalMessageInfo + +func (m *ClickHouseConfig) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *ClickHouseConfig) GetHost() string { + if m != nil { + return m.Host + } + return "" +} + +func (m *ClickHouseConfig) GetPort() int32 { + if m != nil { + return m.Port + } + return 0 +} + +func (m *ClickHouseConfig) GetUsername() string { + if m != nil { + return m.Username + } + return "" +} + +func (m *ClickHouseConfig) GetPassword() string { + if m != nil { + return m.Password + } + return "" +} + +func (m *ClickHouseConfig) GetDatabase() string { + if m != nil { + return m.Database + } + return "" +} + type OTelResource struct { Attributes []*OTelAttribute `protobuf:"bytes,1,rep,name=attributes,proto3" json:"attributes,omitempty"` } @@ -2584,7 +2846,7 @@ type OTelResource struct { func (m *OTelResource) Reset() { *m = OTelResource{} } func (*OTelResource) ProtoMessage() {} func (*OTelResource) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{25} + return fileDescriptor_e5dcfc8666ec3f33, []int{27} } func (m *OTelResource) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2631,7 +2893,7 @@ type OTelExportSinkOperator struct { func (m *OTelExportSinkOperator) Reset() { *m = OTelExportSinkOperator{} } func (*OTelExportSinkOperator) ProtoMessage() {} func (*OTelExportSinkOperator) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{26} + return fileDescriptor_e5dcfc8666ec3f33, []int{28} } func (m *OTelExportSinkOperator) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2695,6 +2957,126 @@ func (m *OTelExportSinkOperator) GetLogs() []*OTelLog { return nil } +type ClickHouseExportSinkOperator struct { + ClickhouseConfig *ClickHouseConfig `protobuf:"bytes,1,opt,name=clickhouse_config,json=clickhouseConfig,proto3" json:"clickhouse_config,omitempty"` + TableName string `protobuf:"bytes,2,opt,name=table_name,json=tableName,proto3" json:"table_name,omitempty"` + ColumnMappings []*ClickHouseExportSinkOperator_ColumnMapping `protobuf:"bytes,3,rep,name=column_mappings,json=columnMappings,proto3" json:"column_mappings,omitempty"` +} + +func (m *ClickHouseExportSinkOperator) Reset() { *m = ClickHouseExportSinkOperator{} } +func (*ClickHouseExportSinkOperator) ProtoMessage() {} +func (*ClickHouseExportSinkOperator) Descriptor() ([]byte, []int) { + return fileDescriptor_e5dcfc8666ec3f33, []int{29} +} +func (m *ClickHouseExportSinkOperator) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClickHouseExportSinkOperator) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClickHouseExportSinkOperator.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClickHouseExportSinkOperator) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClickHouseExportSinkOperator.Merge(m, src) +} +func (m *ClickHouseExportSinkOperator) XXX_Size() int { + return m.Size() +} +func (m *ClickHouseExportSinkOperator) XXX_DiscardUnknown() { + xxx_messageInfo_ClickHouseExportSinkOperator.DiscardUnknown(m) +} + +var xxx_messageInfo_ClickHouseExportSinkOperator proto.InternalMessageInfo + +func (m *ClickHouseExportSinkOperator) GetClickhouseConfig() *ClickHouseConfig { + if m != nil { + return m.ClickhouseConfig + } + return nil +} + +func (m *ClickHouseExportSinkOperator) GetTableName() string { + if m != nil { + return m.TableName + } + return "" +} + +func (m *ClickHouseExportSinkOperator) GetColumnMappings() []*ClickHouseExportSinkOperator_ColumnMapping { + if m != nil { + return m.ColumnMappings + } + return nil +} + +type ClickHouseExportSinkOperator_ColumnMapping struct { + InputColumnIndex int32 `protobuf:"varint,1,opt,name=input_column_index,json=inputColumnIndex,proto3" json:"input_column_index,omitempty"` + ClickhouseColumnName string `protobuf:"bytes,2,opt,name=clickhouse_column_name,json=clickhouseColumnName,proto3" json:"clickhouse_column_name,omitempty"` + ColumnType typespb.DataType `protobuf:"varint,3,opt,name=column_type,json=columnType,proto3,enum=px.types.DataType" json:"column_type,omitempty"` +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) Reset() { + *m = ClickHouseExportSinkOperator_ColumnMapping{} +} +func (*ClickHouseExportSinkOperator_ColumnMapping) ProtoMessage() {} +func (*ClickHouseExportSinkOperator_ColumnMapping) Descriptor() ([]byte, []int) { + return fileDescriptor_e5dcfc8666ec3f33, []int{29, 0} +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) XXX_Unmarshal(b []byte) error { + return m.Unmarshal(b) +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + if deterministic { + return xxx_messageInfo_ClickHouseExportSinkOperator_ColumnMapping.Marshal(b, m, deterministic) + } else { + b = b[:cap(b)] + n, err := m.MarshalToSizedBuffer(b) + if err != nil { + return nil, err + } + return b[:n], nil + } +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) XXX_Merge(src proto.Message) { + xxx_messageInfo_ClickHouseExportSinkOperator_ColumnMapping.Merge(m, src) +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) XXX_Size() int { + return m.Size() +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) XXX_DiscardUnknown() { + xxx_messageInfo_ClickHouseExportSinkOperator_ColumnMapping.DiscardUnknown(m) +} + +var xxx_messageInfo_ClickHouseExportSinkOperator_ColumnMapping proto.InternalMessageInfo + +func (m *ClickHouseExportSinkOperator_ColumnMapping) GetInputColumnIndex() int32 { + if m != nil { + return m.InputColumnIndex + } + return 0 +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) GetClickhouseColumnName() string { + if m != nil { + return m.ClickhouseColumnName + } + return "" +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) GetColumnType() typespb.DataType { + if m != nil { + return m.ColumnType + } + return typespb.DATA_TYPE_UNKNOWN +} + type ScalarExpression struct { // Types that are valid to be assigned to Value: // @@ -2707,7 +3089,7 @@ type ScalarExpression struct { func (m *ScalarExpression) Reset() { *m = ScalarExpression{} } func (*ScalarExpression) ProtoMessage() {} func (*ScalarExpression) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{27} + return fileDescriptor_e5dcfc8666ec3f33, []int{30} } func (m *ScalarExpression) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2810,7 +3192,7 @@ type ScalarValue struct { func (m *ScalarValue) Reset() { *m = ScalarValue{} } func (*ScalarValue) ProtoMessage() {} func (*ScalarValue) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{28} + return fileDescriptor_e5dcfc8666ec3f33, []int{31} } func (m *ScalarValue) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -2951,7 +3333,7 @@ type ScalarFunc struct { func (m *ScalarFunc) Reset() { *m = ScalarFunc{} } func (*ScalarFunc) ProtoMessage() {} func (*ScalarFunc) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{29} + return fileDescriptor_e5dcfc8666ec3f33, []int{32} } func (m *ScalarFunc) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3026,7 +3408,7 @@ type AggregateExpression struct { func (m *AggregateExpression) Reset() { *m = AggregateExpression{} } func (*AggregateExpression) ProtoMessage() {} func (*AggregateExpression) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{30} + return fileDescriptor_e5dcfc8666ec3f33, []int{33} } func (m *AggregateExpression) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3101,7 +3483,7 @@ type AggregateExpression_Arg struct { func (m *AggregateExpression_Arg) Reset() { *m = AggregateExpression_Arg{} } func (*AggregateExpression_Arg) ProtoMessage() {} func (*AggregateExpression_Arg) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{30, 0} + return fileDescriptor_e5dcfc8666ec3f33, []int{33, 0} } func (m *AggregateExpression_Arg) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3184,7 +3566,7 @@ type Column struct { func (m *Column) Reset() { *m = Column{} } func (*Column) ProtoMessage() {} func (*Column) Descriptor() ([]byte, []int) { - return fileDescriptor_e5dcfc8666ec3f33, []int{31} + return fileDescriptor_e5dcfc8666ec3f33, []int{34} } func (m *Column) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -3255,6 +3637,7 @@ func init() { proto.RegisterType((*JoinOperator_ParentColumn)(nil), "px.carnot.planpb.JoinOperator.ParentColumn") proto.RegisterType((*UDTFSourceOperator)(nil), "px.carnot.planpb.UDTFSourceOperator") proto.RegisterType((*EmptySourceOperator)(nil), "px.carnot.planpb.EmptySourceOperator") + proto.RegisterType((*ClickHouseSourceOperator)(nil), "px.carnot.planpb.ClickHouseSourceOperator") proto.RegisterType((*OTelLog)(nil), "px.carnot.planpb.OTelLog") proto.RegisterType((*OTelSpan)(nil), "px.carnot.planpb.OTelSpan") proto.RegisterType((*OTelMetricGauge)(nil), "px.carnot.planpb.OTelMetricGauge") @@ -3265,8 +3648,11 @@ func init() { proto.RegisterType((*OTelMetric)(nil), "px.carnot.planpb.OTelMetric") proto.RegisterType((*OTelEndpointConfig)(nil), "px.carnot.planpb.OTelEndpointConfig") proto.RegisterMapType((map[string]string)(nil), "px.carnot.planpb.OTelEndpointConfig.HeadersEntry") + proto.RegisterType((*ClickHouseConfig)(nil), "px.carnot.planpb.ClickHouseConfig") proto.RegisterType((*OTelResource)(nil), "px.carnot.planpb.OTelResource") proto.RegisterType((*OTelExportSinkOperator)(nil), "px.carnot.planpb.OTelExportSinkOperator") + proto.RegisterType((*ClickHouseExportSinkOperator)(nil), "px.carnot.planpb.ClickHouseExportSinkOperator") + proto.RegisterType((*ClickHouseExportSinkOperator_ColumnMapping)(nil), "px.carnot.planpb.ClickHouseExportSinkOperator.ColumnMapping") proto.RegisterType((*ScalarExpression)(nil), "px.carnot.planpb.ScalarExpression") proto.RegisterType((*ScalarValue)(nil), "px.carnot.planpb.ScalarValue") proto.RegisterType((*ScalarFunc)(nil), "px.carnot.planpb.ScalarFunc") @@ -3278,213 +3664,238 @@ func init() { func init() { proto.RegisterFile("src/carnot/planpb/plan.proto", fileDescriptor_e5dcfc8666ec3f33) } var fileDescriptor_e5dcfc8666ec3f33 = []byte{ - // 3294 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x39, 0x4b, 0x6c, 0x1b, 0xc7, - 0xd9, 0x5c, 0x92, 0xe2, 0xe3, 0xe3, 0x53, 0x63, 0xc9, 0x96, 0x69, 0x9b, 0x72, 0x18, 0xfb, 0xb7, - 0xe2, 0x3f, 0xa1, 0x6c, 0xd9, 0xf1, 0xef, 0x38, 0xce, 0x9f, 0x50, 0x12, 0x25, 0x51, 0x91, 0x44, - 0x75, 0x44, 0x25, 0x4d, 0x1b, 0x74, 0xb1, 0xe2, 0x8e, 0xd6, 0x1b, 0x93, 0xbb, 0x9b, 0x7d, 0xd8, - 0x52, 0x80, 0xa2, 0x29, 0x7a, 0xe9, 0x21, 0x87, 0x1e, 0x7a, 0x28, 0x7a, 0x6f, 0x91, 0x4b, 0x8b, - 0x1c, 0x7a, 0xec, 0xa1, 0x05, 0x0a, 0xa4, 0x87, 0x22, 0x70, 0x7b, 0xca, 0xc9, 0x88, 0x95, 0x8b, - 0x0f, 0x45, 0x91, 0xde, 0x7b, 0x28, 0xe6, 0xb1, 0xe4, 0x52, 0xbb, 0xb2, 0x94, 0xb4, 0x28, 0xd0, - 0x83, 0xc4, 0x9d, 0xef, 0x35, 0xdf, 0x7b, 0xbe, 0xd9, 0x85, 0xf3, 0x8e, 0xdd, 0x9d, 0xed, 0x2a, - 0xb6, 0x61, 0xba, 0xb3, 0x56, 0x4f, 0x31, 0xac, 0x1d, 0xf6, 0x53, 0xb7, 0x6c, 0xd3, 0x35, 0x51, - 0xd9, 0xda, 0xab, 0x73, 0x64, 0x9d, 0x23, 0x2b, 0x13, 0x9a, 0xa9, 0x99, 0x0c, 0x39, 0x4b, 0x9f, - 0x38, 0x5d, 0xa5, 0xaa, 0x99, 0xa6, 0xd6, 0x23, 0xb3, 0x6c, 0xb5, 0xe3, 0xed, 0xce, 0x3e, 0xb4, - 0x15, 0xcb, 0x22, 0xb6, 0x23, 0xf0, 0xd3, 0x74, 0x17, 0xc5, 0xd2, 0x39, 0xc1, 0xac, 0xe7, 0xe9, - 0xaa, 0xb5, 0xc3, 0x7e, 0x04, 0xc1, 0x25, 0x4a, 0xe0, 0xdc, 0x53, 0x6c, 0xa2, 0xce, 0xba, 0xfb, - 0x16, 0x71, 0xf8, 0x7f, 0x6b, 0x87, 0xff, 0x72, 0xaa, 0xda, 0x0f, 0x25, 0xc8, 0x6d, 0xf6, 0x14, - 0xa3, 0x6d, 0xb9, 0xba, 0x69, 0x38, 0x68, 0x0a, 0xd2, 0x64, 0xcf, 0xea, 0x29, 0xba, 0x31, 0x15, - 0xbf, 0x28, 0xcd, 0x64, 0xb0, 0xbf, 0xa4, 0x18, 0xc5, 0x50, 0x7a, 0xfb, 0x1f, 0x90, 0xa9, 0x04, - 0xc7, 0x88, 0x25, 0xba, 0x0d, 0x67, 0xfb, 0xca, 0x9e, 0x6c, 0x7a, 0xae, 0xe5, 0xb9, 0xb2, 0x6d, - 0x3e, 0x74, 0x64, 0x8b, 0xd8, 0xb2, 0xab, 0xec, 0xf4, 0xc8, 0x54, 0xf2, 0xa2, 0x34, 0x93, 0xc0, - 0x93, 0x7d, 0x65, 0xaf, 0xcd, 0xf0, 0xd8, 0x7c, 0xe8, 0x6c, 0x12, 0xbb, 0x43, 0x91, 0xab, 0xc9, - 0x8c, 0x54, 0x8e, 0xd7, 0x9e, 0x24, 0x20, 0x49, 0x75, 0x40, 0x57, 0x20, 0xa1, 0x2a, 0xda, 0x94, - 0x74, 0x51, 0x9a, 0xc9, 0xcd, 0x4d, 0xd6, 0x0f, 0x7b, 0xaa, 0xbe, 0xd8, 0x58, 0xc6, 0x94, 0x02, - 0xdd, 0x84, 0x31, 0xc3, 0x54, 0x89, 0x33, 0x15, 0xbf, 0x98, 0x98, 0xc9, 0xcd, 0x55, 0xc3, 0xa4, - 0x54, 0xde, 0x92, 0xad, 0x68, 0x7d, 0x62, 0xb8, 0x98, 0x13, 0xa3, 0x37, 0x20, 0x4f, 0xb1, 0xb2, - 0xc9, 0x6d, 0x65, 0xaa, 0xe5, 0xe6, 0x2e, 0x44, 0x33, 0x0b, 0x87, 0xe0, 0x9c, 0x15, 0xf0, 0xce, - 0x16, 0x20, 0xdd, 0xe8, 0x9a, 0x7d, 0xdd, 0xd0, 0x64, 0x45, 0x23, 0x86, 0x2b, 0xeb, 0xaa, 0x33, - 0x35, 0xc6, 0x94, 0x28, 0x51, 0x39, 0x3c, 0x0c, 0xf5, 0xed, 0xed, 0xd6, 0xe2, 0xfc, 0xc4, 0xc1, - 0xe3, 0xe9, 0x72, 0x4b, 0x90, 0x37, 0x28, 0x75, 0x6b, 0xd1, 0xc1, 0x65, 0x7d, 0x04, 0xa2, 0x3a, - 0xc8, 0x83, 0x0b, 0x64, 0x8f, 0x74, 0x3d, 0xba, 0x85, 0xec, 0xb8, 0x8a, 0xeb, 0x39, 0xb2, 0x4a, - 0x1c, 0x57, 0x37, 0x14, 0xae, 0x67, 0x8a, 0xc9, 0xbf, 0x1e, 0xad, 0x67, 0xbd, 0xe9, 0xf3, 0x6e, - 0x31, 0xd6, 0xc5, 0x21, 0x27, 0x3e, 0x47, 0x8e, 0xc4, 0x39, 0x95, 0x5d, 0xa8, 0x1c, 0xcd, 0x8a, - 0x9e, 0x83, 0xbc, 0x66, 0x5b, 0x5d, 0x59, 0x51, 0x55, 0x9b, 0x38, 0x0e, 0x8b, 0x49, 0x16, 0xe7, - 0x28, 0xac, 0xc1, 0x41, 0xe8, 0x32, 0x14, 0x1d, 0xa7, 0x27, 0xbb, 0x8a, 0xad, 0x11, 0xd7, 0x50, - 0xfa, 0x84, 0x65, 0x4c, 0x16, 0x17, 0x1c, 0xa7, 0xd7, 0x19, 0x00, 0x57, 0x93, 0x99, 0x44, 0x39, - 0x59, 0xdb, 0x87, 0x7c, 0x30, 0x24, 0xa8, 0x08, 0x71, 0x5d, 0x65, 0x52, 0x93, 0x38, 0xae, 0xab, - 0x7e, 0xe8, 0xe3, 0xc7, 0x86, 0xfe, 0x9a, 0x1f, 0xfa, 0x04, 0xf3, 0x4a, 0x25, 0xda, 0x2b, 0x1b, - 0xa6, 0x4a, 0x44, 0xd8, 0x6b, 0xbf, 0x90, 0x20, 0xb1, 0xd8, 0x58, 0x46, 0x37, 0x7c, 0x4e, 0x89, - 0x71, 0x5e, 0x88, 0xdc, 0x84, 0xfe, 0x05, 0x98, 0x2b, 0x3a, 0xa4, 0x05, 0x24, 0xa4, 0x32, 0xb5, - 0xdf, 0xb4, 0x5d, 0xa2, 0xca, 0x96, 0x62, 0x13, 0xc3, 0xa5, 0x09, 0x95, 0x98, 0x49, 0xe2, 0x02, - 0x87, 0x6e, 0x72, 0x20, 0xba, 0x02, 0x25, 0x41, 0xd6, 0xbd, 0xa7, 0xf7, 0x54, 0x9b, 0x18, 0x4c, - 0xf5, 0x24, 0x16, 0xdc, 0x0b, 0x02, 0x5a, 0x5b, 0x82, 0x8c, 0xaf, 0x7a, 0x68, 0xaf, 0xab, 0x10, - 0x37, 0x2d, 0xe1, 0x9d, 0x08, 0x93, 0xdb, 0x16, 0xb1, 0x15, 0xd7, 0xb4, 0x71, 0xdc, 0xb4, 0x6a, - 0x3f, 0xca, 0x40, 0xc6, 0x07, 0xa0, 0xff, 0x83, 0xb4, 0x69, 0xc9, 0xb4, 0xe2, 0x99, 0xb4, 0x62, - 0x54, 0xad, 0xf8, 0xc4, 0x9d, 0x7d, 0x8b, 0xe0, 0x94, 0x69, 0xd1, 0x5f, 0xb4, 0x06, 0x85, 0x3e, - 0xe9, 0xcb, 0x8e, 0xe9, 0xd9, 0x5d, 0x22, 0x0f, 0x36, 0xff, 0x9f, 0x30, 0xfb, 0x3a, 0xe9, 0x9b, - 0xf6, 0xfe, 0x16, 0x23, 0xf4, 0x45, 0xad, 0xc4, 0x70, 0xae, 0x4f, 0xfa, 0x3e, 0x10, 0xdd, 0x82, - 0x54, 0x5f, 0xb1, 0xa8, 0x98, 0xc4, 0x51, 0x45, 0xb7, 0xae, 0x58, 0x01, 0xee, 0xb1, 0x3e, 0x5d, - 0xa2, 0xbb, 0x90, 0x52, 0x34, 0x8d, 0xf2, 0xf1, 0x62, 0x7d, 0x3e, 0xcc, 0xd7, 0xd0, 0x34, 0x9b, - 0x68, 0x8a, 0x1b, 0xdc, 0x7b, 0x4c, 0xd1, 0xb4, 0xb6, 0x85, 0x96, 0x20, 0xc7, 0x6c, 0xd0, 0x8d, - 0xfb, 0x54, 0xc4, 0x18, 0x13, 0x71, 0xe9, 0x48, 0x0b, 0x74, 0xe3, 0x7e, 0x40, 0x46, 0x96, 0xea, - 0xcf, 0x40, 0xe8, 0x75, 0xc8, 0xee, 0xea, 0x3d, 0x97, 0xd8, 0x54, 0x4a, 0x8a, 0x49, 0xb9, 0x18, - 0x96, 0xb2, 0xc4, 0x48, 0x02, 0x12, 0x32, 0xbb, 0x02, 0x82, 0xee, 0x42, 0xa6, 0xa7, 0xf7, 0x75, - 0x97, 0xf2, 0xa7, 0x19, 0xff, 0x74, 0x98, 0x7f, 0x8d, 0x52, 0x04, 0xd8, 0xd3, 0x3d, 0x0e, 0xa0, - 0xdc, 0x9e, 0x41, 0x9b, 0x83, 0x69, 0x4d, 0x65, 0x8e, 0xe2, 0xde, 0xa6, 0x14, 0x41, 0x6e, 0x8f, - 0x03, 0xd0, 0xf7, 0xa0, 0xc8, 0x2a, 0x79, 0x18, 0xc9, 0xec, 0x51, 0x7e, 0x58, 0xc6, 0x9b, 0x0b, - 0xa3, 0x71, 0x9c, 0x2f, 0x1f, 0x3c, 0x9e, 0xce, 0x07, 0xe1, 0x2b, 0x31, 0xcc, 0x3a, 0xc3, 0x20, - 0xb4, 0x6f, 0x8b, 0x4e, 0xe1, 0x7b, 0xf9, 0x29, 0x37, 0xb0, 0x76, 0x84, 0xf8, 0x80, 0x93, 0xe7, - 0x8b, 0x07, 0x8f, 0xa7, 0x61, 0x08, 0x5d, 0x89, 0x61, 0x60, 0xa2, 0xb9, 0xd7, 0x5f, 0x81, 0xf4, - 0x7b, 0xa6, 0xce, 0xac, 0xce, 0x31, 0x91, 0x11, 0xa9, 0xbb, 0x6a, 0xea, 0x41, 0xa3, 0x53, 0xef, - 0xb1, 0x35, 0x5a, 0x83, 0xa2, 0xa7, 0xba, 0xbb, 0x01, 0x9b, 0xf3, 0x47, 0xd9, 0xbc, 0xbd, 0xd8, - 0x59, 0x0a, 0xe5, 0x6e, 0x9e, 0x72, 0x0f, 0x2c, 0x6c, 0x43, 0x89, 0xf4, 0x2d, 0x77, 0x3f, 0x20, - 0xae, 0xc0, 0xc4, 0x5d, 0x0e, 0x8b, 0x6b, 0x52, 0xc2, 0x90, 0xbc, 0x02, 0x09, 0x82, 0xd1, 0xbb, - 0x90, 0x37, 0x5d, 0xd2, 0x1b, 0xb8, 0xac, 0xc8, 0xa4, 0xcd, 0x44, 0x54, 0x66, 0x87, 0xf4, 0x9a, - 0x7b, 0x96, 0x69, 0xbb, 0x61, 0xbf, 0x51, 0xdc, 0xd0, 0x6f, 0x54, 0x1e, 0x5f, 0xcd, 0x27, 0x69, - 0xaf, 0xa8, 0xfd, 0x39, 0x0e, 0x13, 0x51, 0x95, 0x89, 0x10, 0x24, 0x59, 0xb3, 0xe6, 0x1d, 0x9d, - 0x3d, 0xa3, 0x69, 0xc8, 0x75, 0xcd, 0x9e, 0xd7, 0x37, 0x64, 0x5d, 0xdd, 0xe3, 0xa7, 0x6a, 0x02, - 0x03, 0x07, 0xb5, 0xd4, 0x3d, 0x87, 0x1e, 0x07, 0x82, 0x80, 0xd2, 0xf3, 0xe6, 0x9b, 0xc5, 0x82, - 0x69, 0x83, 0x82, 0xd0, 0xcb, 0x03, 0x12, 0x36, 0x5f, 0xb0, 0x66, 0x58, 0x9c, 0x43, 0xd4, 0x28, - 0x3e, 0x70, 0x2c, 0x2a, 0xae, 0xc2, 0x5a, 0x8c, 0x60, 0xa3, 0xcf, 0x0e, 0xba, 0x03, 0xe0, 0xb8, - 0x8a, 0xed, 0xca, 0xae, 0xde, 0x27, 0xa2, 0x44, 0xcf, 0xd5, 0xf9, 0xf0, 0x53, 0xf7, 0x87, 0x9f, - 0x7a, 0xcb, 0x70, 0x6f, 0xdd, 0x7c, 0x4b, 0xe9, 0x79, 0x04, 0x67, 0x19, 0x79, 0x47, 0xef, 0xd3, - 0xc1, 0x23, 0xeb, 0xb8, 0xb4, 0xbd, 0x51, 0xd6, 0xd4, 0xf1, 0xac, 0x19, 0x4a, 0xcd, 0x38, 0x4f, - 0x43, 0x8a, 0x8d, 0x27, 0x2e, 0x2b, 0xc7, 0x2c, 0x16, 0x2b, 0x74, 0x9e, 0x4a, 0xb4, 0x89, 0x42, - 0x0f, 0x68, 0x56, 0x6b, 0x19, 0x3c, 0x04, 0xd4, 0x3e, 0x93, 0x00, 0x85, 0x7b, 0x45, 0xa4, 0x47, - 0x0f, 0x7b, 0x23, 0x7e, 0x32, 0x6f, 0x9c, 0xc0, 0xcf, 0xab, 0x30, 0x29, 0x48, 0x1c, 0xd2, 0x57, - 0x0c, 0x57, 0xef, 0x8e, 0x38, 0xfc, 0xf4, 0x70, 0x8b, 0x2d, 0x81, 0x67, 0xdb, 0x9c, 0xe2, 0x4c, - 0x41, 0x98, 0x53, 0x33, 0x00, 0x85, 0x6b, 0x3e, 0xa4, 0xbb, 0xf4, 0xcd, 0x74, 0x8f, 0x87, 0x74, - 0xaf, 0x7d, 0x96, 0x84, 0xf2, 0xe1, 0x2e, 0xc0, 0x06, 0xcb, 0x91, 0x29, 0xc3, 0x5f, 0xa2, 0xdb, - 0xa3, 0xad, 0x4b, 0x57, 0xd9, 0xe9, 0x91, 0x3c, 0xdc, 0x94, 0x5a, 0x8b, 0xa3, 0x4d, 0xa9, 0xa5, - 0xa2, 0x2d, 0xc8, 0x8b, 0x71, 0x74, 0x38, 0x85, 0xe6, 0xe6, 0xea, 0xc7, 0xf7, 0xa4, 0x3a, 0x26, - 0x8e, 0xd7, 0x73, 0xd9, 0x78, 0x4a, 0x0f, 0x31, 0x2e, 0x85, 0x2d, 0x91, 0x06, 0xa8, 0x6b, 0x1a, - 0x06, 0xe9, 0xba, 0xbc, 0x19, 0xf3, 0xe9, 0x8c, 0xa7, 0xec, 0xed, 0x13, 0x88, 0xa6, 0x80, 0x85, - 0x81, 0x00, 0x7f, 0xc0, 0x1c, 0xef, 0x1e, 0x06, 0x55, 0xfe, 0x22, 0x41, 0x2e, 0xa0, 0x07, 0xba, - 0x00, 0xc0, 0xcc, 0x90, 0x03, 0x69, 0x96, 0x65, 0x90, 0x8d, 0xff, 0x9a, 0x5c, 0xab, 0xfc, 0x3f, - 0x4c, 0x46, 0x3a, 0x20, 0x62, 0x8e, 0x94, 0x22, 0xe6, 0xc8, 0xf9, 0x02, 0xe4, 0x02, 0x53, 0xf1, - 0x6a, 0x32, 0x13, 0x2f, 0x27, 0x6a, 0x0f, 0x20, 0x17, 0x98, 0x1b, 0xd0, 0x22, 0xe4, 0xc8, 0x9e, - 0x45, 0x73, 0x87, 0x85, 0x86, 0x0f, 0x7a, 0x11, 0x27, 0xd1, 0x56, 0x57, 0xe9, 0x29, 0x76, 0x73, - 0x40, 0x8a, 0x83, 0x6c, 0x27, 0x49, 0xe4, 0x5f, 0xc7, 0x61, 0x3c, 0x34, 0x78, 0xa0, 0xd7, 0x20, - 0xf5, 0x80, 0x36, 0x1a, 0x7f, 0xe7, 0xcb, 0xcf, 0x98, 0x56, 0x02, 0x9b, 0x0b, 0x26, 0x74, 0x0d, - 0x52, 0x9a, 0x6d, 0x7a, 0x96, 0x7f, 0xad, 0x99, 0x0a, 0xb3, 0x2f, 0x30, 0x1d, 0xb0, 0xa0, 0xa3, - 0x7d, 0x9b, 0x3d, 0x8d, 0x44, 0x10, 0x18, 0x88, 0x07, 0x70, 0x1a, 0x72, 0x4c, 0xb8, 0x20, 0x48, - 0x72, 0x02, 0x06, 0xe2, 0x04, 0x15, 0xc8, 0x3c, 0xd4, 0x0d, 0xd5, 0x7c, 0x48, 0x54, 0x96, 0xc9, - 0x19, 0x3c, 0x58, 0x53, 0x66, 0x4b, 0xb1, 0x5d, 0x5d, 0xe9, 0xc9, 0x8a, 0xa6, 0xb1, 0x06, 0x9b, - 0xc1, 0x20, 0x40, 0x0d, 0x4d, 0x43, 0x2f, 0x40, 0x79, 0x57, 0x37, 0x94, 0x9e, 0xfe, 0x01, 0x91, - 0x6d, 0x96, 0xaf, 0x0e, 0xeb, 0xa7, 0x19, 0x5c, 0xf2, 0xe1, 0x3c, 0x8d, 0x9d, 0xda, 0x8f, 0x25, - 0x28, 0x8e, 0x0e, 0x48, 0x68, 0x1e, 0x60, 0xe8, 0x75, 0x71, 0xe9, 0x3b, 0x49, 0xac, 0x02, 0x5c, - 0x68, 0x0e, 0xd2, 0x3c, 0x2c, 0xc7, 0xfb, 0xcc, 0x27, 0xac, 0x7d, 0x28, 0x41, 0x61, 0x64, 0xd6, - 0x42, 0x13, 0x30, 0xc6, 0x66, 0x2d, 0xa6, 0x44, 0x02, 0xf3, 0xc5, 0x37, 0x91, 0x4d, 0x73, 0x59, - 0xd9, 0x31, 0x6d, 0x5e, 0xad, 0x8e, 0xdd, 0x75, 0xc4, 0xac, 0x5f, 0x18, 0x40, 0xb7, 0xec, 0xae, - 0x53, 0x7b, 0x2a, 0x41, 0x61, 0x64, 0x60, 0x0b, 0xe5, 0x9c, 0x14, 0x2e, 0xc6, 0xb7, 0xa0, 0x24, - 0x48, 0xfa, 0x8a, 0x65, 0xe9, 0x86, 0xe6, 0xeb, 0xf5, 0xd2, 0x31, 0xd3, 0xa0, 0xd0, 0x72, 0x9d, - 0x73, 0xe1, 0x62, 0x37, 0xb8, 0x74, 0xd0, 0x25, 0x28, 0x0e, 0xee, 0xec, 0x3b, 0x8a, 0xdb, 0xbd, - 0xc7, 0xbb, 0x2c, 0xce, 0xdb, 0xfc, 0xaa, 0x3e, 0x4f, 0x61, 0x95, 0x5b, 0x50, 0x18, 0x11, 0x43, - 0x4d, 0xf5, 0x67, 0x06, 0x43, 0x25, 0x7b, 0x42, 0xe7, 0x04, 0x2e, 0x88, 0xb1, 0x81, 0x03, 0x6b, - 0x9f, 0x26, 0x21, 0x1f, 0x9c, 0xd2, 0xd0, 0xab, 0x90, 0x0c, 0x5c, 0x47, 0xae, 0x3c, 0x7b, 0xa6, - 0x63, 0x0b, 0xd6, 0x53, 0x18, 0x13, 0x52, 0xe0, 0x14, 0x79, 0xdf, 0x53, 0x7a, 0xba, 0xbb, 0x2f, - 0x77, 0x4d, 0x43, 0xd5, 0x79, 0x0f, 0xe6, 0x7e, 0xb8, 0x76, 0x8c, 0xac, 0xa6, 0xe0, 0x5c, 0xf0, - 0x19, 0x31, 0x22, 0x87, 0x41, 0x0e, 0xc2, 0x50, 0x14, 0x47, 0x87, 0x1f, 0x7d, 0x7e, 0xd3, 0xfc, - 0xdf, 0x63, 0xa4, 0xf3, 0xfb, 0x9e, 0x48, 0x88, 0x02, 0x17, 0xb1, 0x20, 0xd2, 0xe2, 0x70, 0x74, - 0x93, 0xe1, 0xe8, 0x86, 0xa3, 0x30, 0x16, 0x11, 0x85, 0x3e, 0x8c, 0x87, 0xac, 0x40, 0x57, 0x61, - 0xbc, 0x47, 0x76, 0x7d, 0x7d, 0x79, 0x38, 0xc4, 0xdd, 0xb1, 0x44, 0x11, 0x0b, 0xc3, 0x80, 0xa0, - 0x17, 0x01, 0xd9, 0xba, 0x76, 0xef, 0x10, 0x71, 0x9c, 0x11, 0x97, 0x19, 0x26, 0x40, 0x5d, 0xe9, - 0x40, 0x3e, 0x68, 0x16, 0xb5, 0x83, 0xdf, 0x75, 0x47, 0x36, 0xc9, 0x71, 0x18, 0xdf, 0x60, 0x68, - 0x6a, 0x50, 0x74, 0x2e, 0x90, 0x14, 0xb5, 0x97, 0x21, 0xe3, 0x87, 0x15, 0x65, 0x61, 0xac, 0xb5, - 0xb1, 0xd1, 0xc4, 0xe5, 0x18, 0x2a, 0x02, 0xac, 0x35, 0x97, 0x3a, 0x72, 0x7b, 0xbb, 0xd3, 0xc4, - 0x65, 0x89, 0xae, 0x97, 0xb6, 0xd7, 0xd6, 0xc4, 0x3a, 0x51, 0xdb, 0x05, 0x14, 0x1e, 0xd6, 0x23, - 0x87, 0xaf, 0xbb, 0x00, 0x8a, 0xad, 0xc9, 0xa2, 0x17, 0xc7, 0x8f, 0xba, 0xee, 0xf3, 0xce, 0x22, - 0xa6, 0x4a, 0xc5, 0xd6, 0xd8, 0x93, 0x53, 0x33, 0xe1, 0x54, 0xc4, 0x14, 0x7f, 0x92, 0x0a, 0xfd, - 0x66, 0x07, 0x71, 0xed, 0x57, 0x71, 0x48, 0xd3, 0x69, 0x7e, 0xcd, 0xd4, 0xd0, 0xeb, 0x00, 0x8a, - 0xeb, 0xda, 0xfa, 0x8e, 0xe7, 0x0e, 0x8e, 0x91, 0xe9, 0xe8, 0x8b, 0x41, 0xc3, 0xa7, 0xc3, 0x01, - 0x16, 0x9a, 0x0c, 0x74, 0x1c, 0x0e, 0xc7, 0x37, 0x81, 0x4b, 0x14, 0x11, 0x4c, 0x86, 0x57, 0xa1, - 0x62, 0xee, 0x38, 0xc4, 0x7e, 0x40, 0x54, 0x39, 0xcc, 0x94, 0x60, 0x4c, 0x67, 0x7c, 0x8a, 0xce, - 0x21, 0xe6, 0x2b, 0x50, 0x72, 0xc8, 0x03, 0x62, 0xd3, 0x52, 0x34, 0xbc, 0xfe, 0x0e, 0xb1, 0xc5, - 0xbb, 0xbe, 0xa2, 0x0f, 0xde, 0x60, 0x50, 0xf4, 0x3c, 0x14, 0x06, 0x84, 0x2e, 0xd9, 0x73, 0x59, - 0x62, 0x67, 0x71, 0xde, 0x07, 0x76, 0xc8, 0x9e, 0x4b, 0xd5, 0xde, 0x31, 0xd5, 0xfd, 0x51, 0x0d, - 0x52, 0x5c, 0x6d, 0x8a, 0x08, 0xec, 0x5c, 0xfb, 0x28, 0x09, 0x19, 0x76, 0xfb, 0xb1, 0x14, 0x9a, - 0x92, 0x39, 0x1a, 0x0f, 0xd9, 0x71, 0x6d, 0x3a, 0xb3, 0xb3, 0x34, 0xa0, 0x17, 0x22, 0x0a, 0xdc, - 0x62, 0x30, 0xf4, 0x22, 0x8c, 0x33, 0x92, 0xb0, 0x4b, 0x56, 0x62, 0xb8, 0x44, 0x51, 0x41, 0xbb, - 0x46, 0x23, 0x90, 0xf8, 0xfa, 0x11, 0x58, 0x84, 0x49, 0xd7, 0x56, 0xd8, 0xbc, 0x3a, 0xba, 0x25, - 0x73, 0xcf, 0xfc, 0xf8, 0xc1, 0xe3, 0xe9, 0x42, 0x87, 0x12, 0xb4, 0x16, 0x45, 0xb7, 0x40, 0x8c, - 0xbe, 0xa5, 0x06, 0xd5, 0x68, 0xc0, 0x84, 0x63, 0x29, 0x46, 0x48, 0xc8, 0x18, 0x13, 0xc2, 0x26, - 0x60, 0x6a, 0xff, 0x40, 0xc6, 0x38, 0xa5, 0x1e, 0x15, 0xd1, 0x81, 0x73, 0xa2, 0x5a, 0x23, 0x25, - 0x31, 0xef, 0xce, 0x9f, 0x3e, 0x78, 0x3c, 0x8d, 0x78, 0x91, 0x8f, 0xc8, 0x3b, 0x63, 0x0d, 0x61, - 0x23, 0x52, 0x5f, 0x86, 0x33, 0xc3, 0x0b, 0xdb, 0xa8, 0xc4, 0x34, 0x8b, 0xd7, 0xc4, 0xe0, 0x82, - 0x16, 0x64, 0xbb, 0x0e, 0x93, 0xc4, 0x88, 0x4a, 0xb3, 0x0c, 0x63, 0x42, 0xc4, 0x08, 0x65, 0xd8, - 0x05, 0x80, 0xfb, 0xba, 0xa1, 0xf2, 0x3a, 0x66, 0x6f, 0x2d, 0x12, 0x38, 0x4b, 0x21, 0xac, 0x50, - 0xe7, 0x53, 0xbc, 0xf2, 0x6b, 0xdf, 0x87, 0x12, 0x0d, 0xc6, 0x3a, 0x71, 0x6d, 0xbd, 0xbb, 0xac, - 0x78, 0x1a, 0x41, 0x75, 0x40, 0xbb, 0x3d, 0x53, 0x89, 0x68, 0x89, 0x34, 0xe4, 0x65, 0x86, 0x0b, - 0xee, 0x74, 0x15, 0xca, 0xba, 0xe1, 0x46, 0x27, 0x48, 0x51, 0x37, 0x82, 0xb4, 0xf3, 0x45, 0xc8, - 0xf3, 0x91, 0x8a, 0x53, 0xd7, 0x7e, 0x19, 0x87, 0xf1, 0xe1, 0xfe, 0x5b, 0x5e, 0xbf, 0xaf, 0xd8, - 0xfb, 0xb4, 0xcf, 0x76, 0x4d, 0xcf, 0x88, 0xd2, 0x00, 0x97, 0x19, 0x26, 0xb8, 0xff, 0x0c, 0x94, - 0x1d, 0xaf, 0x1f, 0x55, 0xb3, 0x45, 0xc7, 0xeb, 0x07, 0x29, 0xdf, 0x85, 0xd2, 0xfb, 0x1e, 0x9d, - 0xaa, 0x7b, 0xc4, 0xef, 0x6f, 0x3c, 0x45, 0x6f, 0x44, 0xa7, 0xe8, 0x88, 0x56, 0x75, 0xe6, 0xb8, - 0x86, 0xfb, 0x2d, 0x21, 0x01, 0x17, 0x7d, 0x59, 0xbc, 0xf5, 0x55, 0xbe, 0x0b, 0xa5, 0x43, 0x24, - 0x74, 0x40, 0xf4, 0x89, 0x98, 0xfa, 0x12, 0x1e, 0xac, 0xa9, 0x91, 0x41, 0x57, 0x8c, 0x28, 0x5e, - 0x66, 0x98, 0x60, 0xd9, 0x7e, 0x12, 0x87, 0xc2, 0x48, 0xd5, 0x44, 0xf6, 0xee, 0x37, 0x20, 0xc5, - 0xa5, 0x1d, 0xfd, 0xc2, 0x71, 0x44, 0x88, 0x18, 0x6e, 0x56, 0x62, 0x58, 0xf0, 0xa1, 0xe7, 0x21, - 0xcf, 0x9b, 0x81, 0x48, 0x9c, 0x84, 0x68, 0x09, 0x39, 0x0e, 0x65, 0x06, 0x56, 0x7e, 0x2e, 0x41, - 0x4a, 0x1c, 0x6a, 0x37, 0x06, 0x2f, 0x3f, 0x02, 0x73, 0x49, 0x54, 0xd3, 0x86, 0x61, 0xd3, 0x8e, - 0x3c, 0xe6, 0x12, 0x23, 0xc7, 0x1c, 0xba, 0x0d, 0x67, 0xbb, 0x8a, 0x21, 0xef, 0x10, 0xf9, 0x3d, - 0xc7, 0x34, 0x64, 0x62, 0x74, 0x4d, 0x95, 0xa8, 0xb2, 0x62, 0xdb, 0xca, 0xbe, 0xf8, 0x84, 0x32, - 0xd9, 0x55, 0x8c, 0x79, 0xb2, 0xea, 0x98, 0x46, 0x93, 0x63, 0x1b, 0x14, 0x39, 0x9f, 0x86, 0x31, - 0xa6, 0x7a, 0xed, 0xd3, 0x38, 0xc0, 0x30, 0x8a, 0x91, 0xfe, 0xba, 0xc8, 0xae, 0x45, 0x5d, 0x5b, - 0x67, 0xb7, 0x29, 0xf1, 0x0a, 0x3e, 0x08, 0xa2, 0x5c, 0x9e, 0xa1, 0xbb, 0xdc, 0x0f, 0x98, 0x3d, - 0x1f, 0x6a, 0x72, 0xc9, 0x7f, 0xd3, 0x31, 0x33, 0x16, 0x7d, 0xcc, 0xbc, 0x02, 0x63, 0x1a, 0x2d, - 0xcb, 0x29, 0xc2, 0x22, 0xfa, 0xdc, 0xb3, 0x32, 0x95, 0xd5, 0xef, 0x4a, 0x0c, 0x73, 0x0e, 0xf4, - 0x3a, 0xa4, 0x1d, 0x9e, 0xbb, 0x53, 0xbb, 0x47, 0xbd, 0x00, 0x0e, 0xa5, 0xf9, 0x4a, 0x0c, 0xfb, - 0x5c, 0xb4, 0x49, 0xa8, 0x8a, 0xab, 0xd4, 0xfe, 0x26, 0x01, 0x62, 0x6f, 0xd3, 0x0c, 0xd5, 0x32, - 0x59, 0x45, 0x1b, 0xbb, 0xba, 0x86, 0xce, 0x42, 0xc2, 0xb3, 0x7b, 0xdc, 0xa1, 0xf3, 0xe9, 0x83, - 0xc7, 0xd3, 0x89, 0x6d, 0xbc, 0x86, 0x29, 0x0c, 0xbd, 0x09, 0xe9, 0x7b, 0x44, 0x51, 0x89, 0xed, - 0x4f, 0x10, 0xd7, 0x8f, 0x78, 0x3f, 0x37, 0x22, 0xb1, 0xbe, 0xc2, 0x79, 0x9a, 0x86, 0x6b, 0xef, - 0x63, 0x5f, 0x02, 0xad, 0x22, 0xdd, 0x70, 0x48, 0xd7, 0xb3, 0xfd, 0xaf, 0x67, 0x83, 0x35, 0x9a, - 0x82, 0x34, 0xf5, 0x98, 0xe9, 0xb9, 0xe2, 0x00, 0xf5, 0x97, 0x95, 0x3b, 0x90, 0x0f, 0x8a, 0x43, - 0x65, 0x48, 0xdc, 0x27, 0xfb, 0x22, 0xfc, 0xf4, 0x91, 0xde, 0x5c, 0x78, 0x92, 0xf3, 0xb8, 0xf3, - 0xc5, 0x9d, 0xf8, 0x6d, 0xa9, 0xd6, 0x86, 0x3c, 0xd5, 0x0e, 0x13, 0xfe, 0xf2, 0xe4, 0x5f, 0x1e, - 0x2c, 0x6a, 0xbf, 0x8d, 0xc3, 0xe9, 0xe8, 0xf7, 0x91, 0x68, 0x1d, 0x4a, 0x44, 0x78, 0x81, 0x4e, - 0xe5, 0xbb, 0xba, 0xff, 0x0d, 0xef, 0xd2, 0x49, 0x5c, 0x86, 0x8b, 0x64, 0x34, 0x28, 0x77, 0x20, - 0x63, 0x0b, 0xb5, 0x45, 0x13, 0xa8, 0x46, 0xcb, 0xf1, 0x8d, 0xc3, 0x03, 0x7a, 0x74, 0x0b, 0xd2, - 0x7d, 0x96, 0x0b, 0x7e, 0x5f, 0x3c, 0xff, 0xac, 0x84, 0xc1, 0x3e, 0x31, 0xba, 0x06, 0x63, 0xf4, - 0x90, 0xf4, 0x6b, 0xa1, 0x12, 0xcd, 0x45, 0x4f, 0x43, 0xcc, 0x09, 0xd1, 0x4b, 0x90, 0xec, 0x99, - 0x9a, 0xff, 0xf5, 0xef, 0x6c, 0x34, 0xc3, 0x9a, 0xa9, 0x61, 0x46, 0x56, 0xfb, 0x9d, 0x04, 0xe5, - 0xc3, 0x57, 0x59, 0xf4, 0x2a, 0x64, 0xba, 0xa6, 0xe1, 0xb8, 0x8a, 0xe1, 0x0a, 0x8f, 0x3d, 0x7b, - 0x4c, 0x5d, 0x89, 0xe1, 0x01, 0x03, 0x9a, 0x3b, 0xd4, 0x29, 0x8f, 0xbc, 0x9e, 0x06, 0x7a, 0xe3, - 0x1c, 0x24, 0x77, 0x3d, 0xa3, 0x2b, 0xbe, 0xc2, 0x9c, 0x3f, 0x6a, 0xb3, 0x25, 0xcf, 0xe8, 0xae, - 0xc4, 0x30, 0xa3, 0x1d, 0x76, 0xa3, 0xdf, 0xc7, 0x21, 0x17, 0x50, 0x06, 0xcd, 0x42, 0x96, 0xd6, - 0xd6, 0x71, 0x6d, 0x33, 0xa3, 0x8a, 0x27, 0x34, 0x0d, 0xb0, 0x63, 0x9a, 0x3d, 0x79, 0x98, 0xb2, - 0x99, 0x95, 0x18, 0xce, 0x52, 0x18, 0x97, 0xf8, 0x1c, 0xe4, 0x74, 0xc3, 0xbd, 0x75, 0x33, 0xd0, - 0xb9, 0xe9, 0x11, 0x0c, 0xfa, 0xe0, 0x1d, 0x2e, 0xba, 0x0c, 0x05, 0x76, 0x7c, 0x0f, 0x88, 0x68, - 0xcd, 0x48, 0x2b, 0x31, 0x9c, 0x17, 0x60, 0x4e, 0x76, 0xf8, 0x10, 0x18, 0x8b, 0x38, 0x04, 0xd0, - 0x0c, 0xb0, 0x5e, 0x75, 0xeb, 0xa6, 0x6c, 0x38, 0x82, 0x2e, 0x25, 0xb6, 0x2c, 0x70, 0xc4, 0x86, - 0xc3, 0x29, 0x6f, 0x43, 0xc1, 0xd3, 0x0d, 0xf7, 0xfa, 0xdc, 0x6d, 0x41, 0xc7, 0x3f, 0x72, 0x8c, - 0x0f, 0xcd, 0xdd, 0x6e, 0x31, 0x34, 0xfb, 0x78, 0xc0, 0x29, 0xf9, 0x94, 0xe2, 0x7b, 0x6f, 0x35, - 0x99, 0xc9, 0x94, 0xb3, 0xb5, 0x2f, 0x24, 0x80, 0xa1, 0x8f, 0x23, 0x3b, 0xfa, 0x1d, 0xc8, 0xea, - 0x86, 0xee, 0xca, 0x8a, 0xad, 0x9d, 0xf0, 0xf2, 0x92, 0xa1, 0xf4, 0x0d, 0x5b, 0x73, 0xd0, 0x2d, - 0x48, 0x32, 0xb6, 0xc4, 0x89, 0xdf, 0x7c, 0x31, 0x7a, 0xf1, 0xbd, 0x91, 0xb7, 0x9f, 0xb8, 0xae, - 0xa2, 0x3b, 0x50, 0xa2, 0x70, 0x79, 0x10, 0x5f, 0x9e, 0xe7, 0xd1, 0x01, 0x2e, 0x50, 0x52, 0x7f, - 0xe5, 0xd4, 0xfe, 0x1e, 0x87, 0x53, 0x11, 0xaf, 0xb9, 0x06, 0xb6, 0x26, 0x8e, 0xb2, 0x35, 0xf9, - 0xf5, 0x6c, 0x7d, 0x4d, 0xd8, 0xca, 0x0b, 0xf0, 0x85, 0x13, 0xbd, 0x6b, 0xab, 0x37, 0x6c, 0x6d, - 0xc4, 0xe4, 0xd4, 0xb3, 0x4c, 0x4e, 0x9f, 0xd0, 0xe4, 0xca, 0x0f, 0x20, 0xd1, 0xb0, 0xb5, 0xff, - 0x78, 0x39, 0x0f, 0x4b, 0x73, 0x6e, 0x30, 0xcd, 0x50, 0x2f, 0x9b, 0x2a, 0x11, 0x57, 0x73, 0xf6, - 0x4c, 0x4f, 0x89, 0xe0, 0x65, 0x9c, 0x2f, 0xae, 0xfe, 0x35, 0x0e, 0xf9, 0xe0, 0xa7, 0x5f, 0x74, - 0x16, 0x26, 0xdb, 0x9b, 0x4d, 0xdc, 0xe8, 0xb4, 0xb1, 0xdc, 0x79, 0x67, 0xb3, 0x29, 0x6f, 0x6f, - 0xbc, 0xb9, 0xd1, 0x7e, 0x7b, 0xa3, 0x1c, 0x43, 0xe7, 0xe0, 0xf4, 0x7a, 0x73, 0xbd, 0x8d, 0xdf, - 0x91, 0xb7, 0xda, 0xdb, 0x78, 0xa1, 0x29, 0xfb, 0x84, 0xe5, 0xa7, 0x69, 0x74, 0x16, 0x26, 0x96, - 0xf1, 0xe6, 0x42, 0x08, 0xf5, 0xa7, 0x0c, 0x45, 0xd1, 0x3b, 0x7b, 0x08, 0xf5, 0x49, 0x16, 0x55, - 0x60, 0xb2, 0xb9, 0xbe, 0xd9, 0x09, 0x4b, 0xfc, 0x29, 0xa0, 0x71, 0xc8, 0xaf, 0x37, 0x36, 0x87, - 0xa0, 0x47, 0x25, 0x74, 0x06, 0x50, 0x63, 0x79, 0x19, 0x37, 0x97, 0x1b, 0x9d, 0x00, 0xed, 0x6f, - 0xca, 0x68, 0x02, 0x4a, 0x4b, 0xad, 0xb5, 0x4e, 0x13, 0x0f, 0xa1, 0x3f, 0x1b, 0x47, 0xa7, 0xa0, - 0xb8, 0xd6, 0x5a, 0x6f, 0x75, 0x86, 0xc0, 0x7f, 0x30, 0xe0, 0xf6, 0x46, 0xab, 0xbd, 0x31, 0x04, - 0x7e, 0x81, 0x10, 0x82, 0xc2, 0x6a, 0xbb, 0x15, 0x80, 0xfd, 0xe1, 0x14, 0x55, 0xdb, 0x37, 0xb7, - 0xb5, 0xf1, 0xe6, 0x10, 0xf5, 0xf1, 0x12, 0xd5, 0x83, 0x1b, 0x3b, 0x82, 0xf8, 0x68, 0x19, 0x55, - 0xe1, 0x6c, 0xbb, 0xd3, 0x5c, 0x93, 0x9b, 0xdf, 0xde, 0x6c, 0xe3, 0xce, 0x21, 0xfc, 0x57, 0xcb, - 0xf3, 0x77, 0x1f, 0x3d, 0xa9, 0xc6, 0x3e, 0x7f, 0x52, 0x8d, 0x7d, 0xf5, 0xa4, 0x2a, 0x7d, 0x78, - 0x50, 0x95, 0x3e, 0x3e, 0xa8, 0x4a, 0x7f, 0x3c, 0xa8, 0x4a, 0x8f, 0x0e, 0xaa, 0xd2, 0x17, 0x07, - 0x55, 0xe9, 0xe9, 0x41, 0x35, 0xf6, 0xd5, 0x41, 0x55, 0xfa, 0xc9, 0x97, 0xd5, 0xd8, 0xa3, 0x2f, - 0xab, 0xb1, 0xcf, 0xbf, 0xac, 0xc6, 0xbe, 0x93, 0xe2, 0xa1, 0xdf, 0x49, 0xb1, 0xef, 0x59, 0x37, - 0xfe, 0x19, 0x00, 0x00, 0xff, 0xff, 0xd2, 0xa5, 0x24, 0xbc, 0x5d, 0x24, 0x00, 0x00, + // 3683 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x3a, 0x4b, 0x6c, 0x1c, 0x47, + 0x76, 0x33, 0xd3, 0xf3, 0x7d, 0xf3, 0x65, 0x89, 0x94, 0xa9, 0xb1, 0x35, 0x94, 0xdb, 0x72, 0x2c, + 0x2b, 0x5e, 0xca, 0xa6, 0x65, 0x45, 0x2b, 0x6b, 0xe3, 0x1d, 0x92, 0x43, 0x72, 0x64, 0x92, 0xc3, + 0x14, 0x87, 0xbb, 0xd9, 0x64, 0xe1, 0x46, 0x73, 0xba, 0xd8, 0x6a, 0x6b, 0xa6, 0xbb, 0xdd, 0x1f, + 0x8b, 0x34, 0x10, 0x64, 0x73, 0xcb, 0x61, 0x0f, 0x39, 0xe4, 0x10, 0xe4, 0x94, 0x4b, 0x02, 0x23, + 0x40, 0x82, 0x05, 0x92, 0x63, 0x0e, 0x09, 0x12, 0x60, 0x73, 0x08, 0x16, 0xce, 0xe7, 0xb0, 0x27, + 0xc1, 0xa6, 0x2f, 0x3e, 0x05, 0xce, 0x2d, 0x87, 0x1c, 0x82, 0xfa, 0x74, 0x4f, 0xf7, 0x74, 0x0f, + 0x45, 0x3b, 0x41, 0x80, 0x1c, 0x6c, 0x4e, 0xbd, 0x5f, 0xbd, 0x5f, 0xbd, 0xf7, 0xaa, 0x5a, 0xf0, + 0x92, 0xeb, 0x8c, 0xee, 0x8c, 0x54, 0xc7, 0xb4, 0xbc, 0x3b, 0xf6, 0x58, 0x35, 0xed, 0x63, 0xf6, + 0x67, 0xd5, 0x76, 0x2c, 0xcf, 0x42, 0x2d, 0xfb, 0x74, 0x95, 0x23, 0x57, 0x39, 0xb2, 0xbd, 0xa8, + 0x5b, 0xba, 0xc5, 0x90, 0x77, 0xe8, 0x2f, 0x4e, 0xd7, 0xee, 0xe8, 0x96, 0xa5, 0x8f, 0xc9, 0x1d, + 0xb6, 0x3a, 0xf6, 0x4f, 0xee, 0x3c, 0x75, 0x54, 0xdb, 0x26, 0x8e, 0x2b, 0xf0, 0x2b, 0x74, 0x17, + 0xd5, 0x36, 0x38, 0xc1, 0x1d, 0xdf, 0x37, 0x34, 0xfb, 0x98, 0xfd, 0x11, 0x04, 0x37, 0x29, 0x81, + 0xfb, 0x58, 0x75, 0x88, 0x76, 0xc7, 0x3b, 0xb3, 0x89, 0xcb, 0xff, 0x6f, 0x1f, 0xf3, 0xbf, 0x9c, + 0x4a, 0xfe, 0xbd, 0x2c, 0x54, 0x0f, 0xc6, 0xaa, 0x39, 0xb0, 0x3d, 0xc3, 0x32, 0x5d, 0xb4, 0x0c, + 0x25, 0x72, 0x6a, 0x8f, 0x55, 0xc3, 0x5c, 0xce, 0xdd, 0xc8, 0xde, 0x2a, 0xe3, 0x60, 0x49, 0x31, + 0xaa, 0xa9, 0x8e, 0xcf, 0x3e, 0x21, 0xcb, 0x12, 0xc7, 0x88, 0x25, 0xba, 0x0f, 0xd7, 0x26, 0xea, + 0xa9, 0x62, 0xf9, 0x9e, 0xed, 0x7b, 0x8a, 0x63, 0x3d, 0x75, 0x15, 0x9b, 0x38, 0x8a, 0xa7, 0x1e, + 0x8f, 0xc9, 0x72, 0xfe, 0x46, 0xf6, 0x96, 0x84, 0x97, 0x26, 0xea, 0xe9, 0x80, 0xe1, 0xb1, 0xf5, + 0xd4, 0x3d, 0x20, 0xce, 0x90, 0x22, 0x1f, 0xe5, 0xcb, 0xd9, 0x56, 0x4e, 0xfe, 0x42, 0x82, 0x3c, + 0xd5, 0x01, 0xbd, 0x06, 0x92, 0xa6, 0xea, 0xcb, 0xd9, 0x1b, 0xd9, 0x5b, 0xd5, 0xb5, 0xa5, 0xd5, + 0x59, 0x4f, 0xad, 0x6e, 0x76, 0xb7, 0x31, 0xa5, 0x40, 0x77, 0xa1, 0x60, 0x5a, 0x1a, 0x71, 0x97, + 0x73, 0x37, 0xa4, 0x5b, 0xd5, 0xb5, 0x4e, 0x92, 0x94, 0xca, 0xdb, 0x72, 0x54, 0x7d, 0x42, 0x4c, + 0x0f, 0x73, 0x62, 0xf4, 0x7d, 0xa8, 0x51, 0xac, 0x62, 0x71, 0x5b, 0x99, 0x6a, 0xd5, 0xb5, 0xeb, + 0xe9, 0xcc, 0xc2, 0x21, 0xb8, 0x6a, 0x47, 0xbc, 0x73, 0x08, 0xc8, 0x30, 0x47, 0xd6, 0xc4, 0x30, + 0x75, 0x45, 0xd5, 0x89, 0xe9, 0x29, 0x86, 0xe6, 0x2e, 0x17, 0x98, 0x12, 0x4d, 0x2a, 0x87, 0x87, + 0x61, 0xf5, 0xe8, 0xa8, 0xbf, 0xb9, 0xbe, 0x78, 0xfe, 0x6c, 0xa5, 0xd5, 0x17, 0xe4, 0x5d, 0x4a, + 0xdd, 0xdf, 0x74, 0x71, 0xcb, 0x88, 0x41, 0x34, 0x17, 0xf9, 0x70, 0x9d, 0x9c, 0x92, 0x91, 0x4f, + 0xb7, 0x50, 0x5c, 0x4f, 0xf5, 0x7c, 0x57, 0xd1, 0x88, 0xeb, 0x19, 0xa6, 0xca, 0xf5, 0x2c, 0x32, + 0xf9, 0x6f, 0xa5, 0xeb, 0xb9, 0xda, 0x0b, 0x78, 0x0f, 0x19, 0xeb, 0xe6, 0x94, 0x13, 0xbf, 0x48, + 0xe6, 0xe2, 0xdc, 0xf6, 0x09, 0xb4, 0xe7, 0xb3, 0xa2, 0x97, 0xa1, 0xa6, 0x3b, 0xf6, 0x48, 0x51, + 0x35, 0xcd, 0x21, 0xae, 0xcb, 0x62, 0x52, 0xc1, 0x55, 0x0a, 0xeb, 0x72, 0x10, 0x7a, 0x15, 0x1a, + 0xae, 0x3b, 0x56, 0x3c, 0xd5, 0xd1, 0x89, 0x67, 0xaa, 0x13, 0xc2, 0x32, 0xa6, 0x82, 0xeb, 0xae, + 0x3b, 0x1e, 0x86, 0xc0, 0x47, 0xf9, 0xb2, 0xd4, 0xca, 0xcb, 0x67, 0x50, 0x8b, 0x86, 0x04, 0x35, + 0x20, 0x67, 0x68, 0x4c, 0x6a, 0x1e, 0xe7, 0x0c, 0x2d, 0x08, 0x7d, 0xee, 0xb9, 0xa1, 0x7f, 0x33, + 0x08, 0xbd, 0xc4, 0xbc, 0xd2, 0x4e, 0xf7, 0xca, 0xbe, 0xa5, 0x11, 0x11, 0x76, 0xf9, 0x4f, 0xb3, + 0x20, 0x6d, 0x76, 0xb7, 0xd1, 0xdb, 0x01, 0x67, 0x96, 0x71, 0x5e, 0x4f, 0xdd, 0x84, 0xfe, 0x17, + 0x61, 0x6e, 0x1b, 0x50, 0x12, 0x90, 0x84, 0xca, 0xd4, 0x7e, 0xcb, 0xf1, 0x88, 0xa6, 0xd8, 0xaa, + 0x43, 0x4c, 0x8f, 0x26, 0x94, 0x74, 0x2b, 0x8f, 0xeb, 0x1c, 0x7a, 0xc0, 0x81, 0xe8, 0x35, 0x68, + 0x0a, 0xb2, 0xd1, 0x63, 0x63, 0xac, 0x39, 0xc4, 0x64, 0xaa, 0xe7, 0xb1, 0xe0, 0xde, 0x10, 0x50, + 0x79, 0x0b, 0xca, 0x81, 0xea, 0x89, 0xbd, 0x6e, 0x43, 0xce, 0xb2, 0x85, 0x77, 0x52, 0x4c, 0x1e, + 0xd8, 0xc4, 0x51, 0x3d, 0xcb, 0xc1, 0x39, 0xcb, 0x96, 0xff, 0xbe, 0x02, 0xe5, 0x00, 0x80, 0x7e, + 0x0d, 0x4a, 0x96, 0xad, 0xd0, 0x13, 0xcf, 0xa4, 0x35, 0xd2, 0xce, 0x4a, 0x40, 0x3c, 0x3c, 0xb3, + 0x09, 0x2e, 0x5a, 0x36, 0xfd, 0x8b, 0x76, 0xa1, 0x3e, 0x21, 0x13, 0xc5, 0xb5, 0x7c, 0x67, 0x44, + 0x94, 0x70, 0xf3, 0x5f, 0x49, 0xb2, 0xef, 0x91, 0x89, 0xe5, 0x9c, 0x1d, 0x32, 0xc2, 0x40, 0xd4, + 0x4e, 0x06, 0x57, 0x27, 0x64, 0x12, 0x00, 0xd1, 0x3d, 0x28, 0x4e, 0x54, 0x9b, 0x8a, 0x91, 0xe6, + 0x1d, 0xba, 0x3d, 0xd5, 0x8e, 0x70, 0x17, 0x26, 0x74, 0x89, 0x1e, 0x42, 0x51, 0xd5, 0x75, 0xca, + 0xc7, 0x0f, 0xeb, 0x2b, 0x49, 0xbe, 0xae, 0xae, 0x3b, 0x44, 0x57, 0xbd, 0xe8, 0xde, 0x05, 0x55, + 0xd7, 0x07, 0x36, 0xda, 0x82, 0x2a, 0xb3, 0xc1, 0x30, 0x9f, 0x50, 0x11, 0x05, 0x26, 0xe2, 0xe6, + 0x5c, 0x0b, 0x0c, 0xf3, 0x49, 0x44, 0x46, 0x85, 0xea, 0xcf, 0x40, 0xe8, 0x3d, 0xa8, 0x9c, 0x18, + 0x63, 0x8f, 0x38, 0x54, 0x4a, 0x91, 0x49, 0xb9, 0x91, 0x94, 0xb2, 0xc5, 0x48, 0x22, 0x12, 0xca, + 0x27, 0x02, 0x82, 0x1e, 0x42, 0x79, 0x6c, 0x4c, 0x0c, 0x8f, 0xf2, 0x97, 0x18, 0xff, 0x4a, 0x92, + 0x7f, 0x97, 0x52, 0x44, 0xd8, 0x4b, 0x63, 0x0e, 0xa0, 0xdc, 0xbe, 0x49, 0x8b, 0x83, 0x65, 0x2f, + 0x97, 0xe7, 0x71, 0x1f, 0x51, 0x8a, 0x28, 0xb7, 0xcf, 0x01, 0xe8, 0x03, 0x68, 0xb0, 0x93, 0x3c, + 0x8d, 0x64, 0x65, 0x9e, 0x1f, 0xb6, 0xf1, 0xc1, 0x46, 0x3c, 0x8e, 0xeb, 0xad, 0xf3, 0x67, 0x2b, + 0xb5, 0x28, 0x7c, 0x27, 0x83, 0x59, 0x65, 0x08, 0x43, 0xfb, 0x43, 0x51, 0x29, 0x02, 0x2f, 0x7f, + 0xc5, 0x0d, 0x94, 0xe7, 0x88, 0x8f, 0x38, 0x79, 0xbd, 0x71, 0xfe, 0x6c, 0x05, 0xa6, 0xd0, 0x9d, + 0x0c, 0x06, 0x26, 0x9a, 0x7b, 0xfd, 0xbb, 0x50, 0xfa, 0xd0, 0x32, 0x98, 0xd5, 0x55, 0x26, 0x32, + 0x25, 0x75, 0x1f, 0x59, 0x46, 0xd4, 0xe8, 0xe2, 0x87, 0x6c, 0x8d, 0x76, 0xa1, 0xe1, 0x6b, 0xde, + 0x49, 0xc4, 0xe6, 0xda, 0x3c, 0x9b, 0x8f, 0x36, 0x87, 0x5b, 0x89, 0xdc, 0xad, 0x51, 0xee, 0xd0, + 0xc2, 0x01, 0x34, 0xc9, 0xc4, 0xf6, 0xce, 0x22, 0xe2, 0xea, 0x4c, 0xdc, 0xab, 0x49, 0x71, 0x3d, + 0x4a, 0x98, 0x90, 0x57, 0x27, 0x51, 0x30, 0xfa, 0x31, 0xd4, 0x2c, 0x8f, 0x8c, 0x43, 0x97, 0x35, + 0x98, 0xb4, 0x5b, 0x29, 0x27, 0x73, 0x48, 0xc6, 0xbd, 0x53, 0xdb, 0x72, 0xbc, 0xa4, 0xdf, 0x28, + 0x6e, 0xea, 0x37, 0x2a, 0x4f, 0xf8, 0xed, 0x03, 0x58, 0x1c, 0x8d, 0x8d, 0xd1, 0x93, 0xc7, 0x96, + 0xef, 0x92, 0x88, 0xce, 0x4d, 0xb6, 0xcb, 0xed, 0xe4, 0x2e, 0x1b, 0x94, 0x7a, 0x87, 0x52, 0x27, + 0x14, 0x47, 0x53, 0x49, 0xa1, 0xf6, 0x1f, 0x00, 0x8a, 0xca, 0x17, 0x36, 0xb4, 0x98, 0xf4, 0xd5, + 0x8b, 0xa4, 0x27, 0x2d, 0xd9, 0xc9, 0xe0, 0x56, 0x64, 0x07, 0x86, 0x59, 0xcf, 0xd3, 0x5a, 0x27, + 0xff, 0x73, 0x0e, 0x16, 0xd3, 0x2a, 0x0b, 0x42, 0x90, 0x67, 0xcd, 0x86, 0x77, 0x24, 0xf6, 0x1b, + 0xad, 0x40, 0x75, 0x64, 0x8d, 0xfd, 0x89, 0xa9, 0x18, 0xda, 0x29, 0x9f, 0x0a, 0x24, 0x0c, 0x1c, + 0xd4, 0xd7, 0x4e, 0x5d, 0xda, 0xce, 0x04, 0x01, 0xa5, 0xe7, 0xcd, 0xa3, 0x82, 0x05, 0xd3, 0x3e, + 0x05, 0xa1, 0x77, 0x42, 0x12, 0x36, 0x1f, 0xb1, 0x62, 0xde, 0x58, 0x43, 0xd4, 0x20, 0x3e, 0x30, + 0x6d, 0xaa, 0x9e, 0xca, 0x4a, 0xa4, 0x60, 0xa3, 0xbf, 0x5d, 0xf4, 0x00, 0xc0, 0xf5, 0x54, 0xc7, + 0x53, 0x3c, 0x63, 0x42, 0x44, 0x89, 0x79, 0x71, 0x95, 0x0f, 0x6f, 0xab, 0xc1, 0xf0, 0xb6, 0xda, + 0x37, 0xbd, 0x7b, 0x77, 0x7f, 0xa0, 0x8e, 0x7d, 0x82, 0x2b, 0x8c, 0x7c, 0x68, 0x4c, 0xe8, 0xe0, + 0x54, 0x71, 0x3d, 0x5a, 0x9e, 0x29, 0x6b, 0xf1, 0xf9, 0xac, 0x65, 0x4a, 0xcd, 0x38, 0xaf, 0x42, + 0x91, 0x8d, 0x57, 0x1e, 0x2b, 0x27, 0x15, 0x2c, 0x56, 0xe8, 0x25, 0x2a, 0xd1, 0x21, 0x2a, 0x1d, + 0x30, 0x58, 0xad, 0x28, 0xe3, 0x29, 0x40, 0xfe, 0x45, 0x16, 0x50, 0xb2, 0xd6, 0xa5, 0x7a, 0x74, + 0xd6, 0x1b, 0xb9, 0xcb, 0x79, 0xe3, 0x12, 0x7e, 0x7e, 0x04, 0x4b, 0x82, 0xc4, 0x25, 0x13, 0xd5, + 0xf4, 0x8c, 0x51, 0xcc, 0xe1, 0x57, 0xa7, 0x5b, 0x1c, 0x0a, 0x3c, 0xdb, 0xe6, 0x0a, 0x67, 0x8a, + 0xc2, 0x5c, 0xd9, 0x04, 0x94, 0xac, 0x59, 0x09, 0xdd, 0xb3, 0xdf, 0x4e, 0xf7, 0x5c, 0x42, 0x77, + 0xf9, 0x17, 0x79, 0x68, 0xcd, 0x56, 0x31, 0x36, 0x18, 0xc7, 0xa6, 0xa4, 0x60, 0x89, 0xee, 0xc7, + 0x4b, 0xaf, 0xa1, 0xb1, 0xee, 0x97, 0x9f, 0x2d, 0xaa, 0xfd, 0xcd, 0x78, 0x51, 0xed, 0x6b, 0xe8, + 0x10, 0x6a, 0x62, 0x9c, 0x9e, 0x4e, 0xd1, 0xa9, 0xa7, 0x6b, 0x56, 0x9b, 0x55, 0x4c, 0x5c, 0x7f, + 0xec, 0xb1, 0xf1, 0x9a, 0x36, 0x61, 0x2e, 0x85, 0x2d, 0x91, 0x0e, 0x68, 0x64, 0x99, 0x26, 0x19, + 0x79, 0xbc, 0x99, 0xf0, 0xe9, 0x92, 0xa7, 0xec, 0xfd, 0x4b, 0x88, 0xa6, 0x80, 0x8d, 0x50, 0x40, + 0x30, 0x20, 0x2f, 0x8c, 0x66, 0x41, 0xed, 0x7f, 0xc9, 0x42, 0x35, 0xa2, 0x07, 0xba, 0x0e, 0xc0, + 0xcc, 0x50, 0x22, 0x69, 0x56, 0x61, 0x90, 0xfd, 0xff, 0x37, 0xb9, 0xd6, 0xfe, 0x75, 0x58, 0x4a, + 0x75, 0x40, 0xca, 0x1c, 0x9c, 0x4d, 0x99, 0x83, 0xd7, 0xeb, 0x50, 0x8d, 0x4c, 0xf5, 0x8f, 0xf2, + 0xe5, 0x5c, 0x4b, 0x92, 0x3f, 0x86, 0x6a, 0x64, 0xee, 0x41, 0x9b, 0x50, 0x25, 0xa7, 0x36, 0xcd, + 0x1d, 0x16, 0x1a, 0x3e, 0xa8, 0xa6, 0x74, 0xd2, 0xc3, 0x91, 0x3a, 0x56, 0x9d, 0x5e, 0x48, 0x8a, + 0xa3, 0x6c, 0x97, 0x49, 0xe4, 0xbf, 0xcc, 0xc1, 0x42, 0x62, 0x70, 0x42, 0xdf, 0x83, 0xe2, 0xc7, + 0xb4, 0xd0, 0x04, 0x3b, 0xbf, 0x7a, 0xc1, 0xb4, 0x15, 0xd9, 0x5c, 0x30, 0xa1, 0x37, 0xa1, 0xa8, + 0x3b, 0x96, 0x6f, 0x07, 0xd7, 0xb2, 0xe5, 0x94, 0x66, 0xc0, 0x74, 0xc0, 0x82, 0x8e, 0xd6, 0x6d, + 0xf6, 0x2b, 0x16, 0x41, 0x60, 0x20, 0x1e, 0xc0, 0x15, 0xa8, 0x32, 0xe1, 0x82, 0x20, 0xcf, 0x09, + 0x18, 0x88, 0x13, 0xb4, 0xa1, 0xfc, 0xd4, 0x30, 0x35, 0xeb, 0x29, 0xd1, 0x58, 0x26, 0x97, 0x71, + 0xb8, 0xa6, 0xcc, 0xb6, 0xea, 0x78, 0x86, 0x3a, 0x56, 0x54, 0x5d, 0x67, 0x05, 0xb6, 0x8c, 0x41, + 0x80, 0xba, 0xba, 0x8e, 0x5e, 0x87, 0xd6, 0x89, 0x61, 0xaa, 0x63, 0xe3, 0x13, 0xa2, 0x38, 0x2c, + 0x5f, 0x5d, 0x56, 0x4f, 0xcb, 0xb8, 0x19, 0xc0, 0x79, 0x1a, 0xbb, 0xf2, 0xef, 0x67, 0xa1, 0x11, + 0x1f, 0xf0, 0xd0, 0x3a, 0xc0, 0xd4, 0xeb, 0xe2, 0xd2, 0x7a, 0x99, 0x58, 0x45, 0xb8, 0xd0, 0x1a, + 0x94, 0x78, 0x58, 0x9e, 0xef, 0xb3, 0x80, 0x50, 0xfe, 0x49, 0x16, 0xea, 0xb1, 0x59, 0x11, 0x2d, + 0x42, 0x81, 0xcd, 0x8a, 0x4c, 0x09, 0x09, 0xf3, 0xc5, 0xb7, 0x91, 0x4d, 0x73, 0x59, 0x3d, 0xb6, + 0x1c, 0x7e, 0x5a, 0x5d, 0x67, 0xe4, 0x8a, 0xbb, 0x4a, 0x3d, 0x84, 0x1e, 0x3a, 0x23, 0x57, 0xfe, + 0x2a, 0x0b, 0xf5, 0xd8, 0xc0, 0x99, 0xc8, 0xb9, 0x6c, 0xf2, 0x30, 0xfe, 0x00, 0x9a, 0x82, 0x64, + 0xa2, 0xda, 0xb6, 0x61, 0xea, 0x81, 0x5e, 0xdf, 0x79, 0xce, 0x34, 0x2b, 0xb4, 0xdc, 0xe3, 0x5c, + 0xb8, 0x31, 0x8a, 0x2e, 0x5d, 0x74, 0x13, 0x1a, 0xe1, 0x9b, 0xc3, 0xb1, 0xea, 0x8d, 0x1e, 0xf3, + 0x2a, 0x8b, 0x6b, 0x0e, 0x7f, 0x6a, 0x58, 0xa7, 0xb0, 0xf6, 0x3d, 0xa8, 0xc7, 0xc4, 0x50, 0x53, + 0x83, 0x99, 0xc1, 0xd4, 0xc8, 0xa9, 0xd0, 0x59, 0xc2, 0x75, 0x31, 0x36, 0x70, 0xa0, 0xfc, 0xf3, + 0x3c, 0xd4, 0xa2, 0x53, 0x26, 0x7a, 0x17, 0xf2, 0x91, 0xeb, 0xd4, 0x6b, 0x17, 0xcf, 0xa4, 0x6c, + 0xc1, 0x6a, 0x0a, 0x63, 0x42, 0x2a, 0x5c, 0x21, 0x1f, 0xf9, 0xea, 0xd8, 0xf0, 0xce, 0x94, 0x91, + 0x65, 0x6a, 0x06, 0xaf, 0xc1, 0xdc, 0x0f, 0x6f, 0x3e, 0x47, 0x56, 0x4f, 0x70, 0x6e, 0x04, 0x8c, + 0x18, 0x91, 0x59, 0x90, 0x8b, 0x30, 0x34, 0x44, 0xeb, 0x08, 0xa2, 0xcf, 0x6f, 0xca, 0xbf, 0xfa, + 0x1c, 0xe9, 0xfc, 0xbe, 0x2a, 0x12, 0xa2, 0xce, 0x45, 0x6c, 0x88, 0xb4, 0x98, 0x8d, 0x6e, 0x3e, + 0x19, 0xdd, 0x64, 0x14, 0x0a, 0x29, 0x51, 0x98, 0xc0, 0x42, 0xc2, 0x0a, 0x74, 0x1b, 0x16, 0xc6, + 0xe4, 0x24, 0xd0, 0x97, 0x87, 0x43, 0xdc, 0x7d, 0x9b, 0x14, 0xb1, 0x31, 0x0d, 0x08, 0x7a, 0x03, + 0x90, 0x63, 0xe8, 0x8f, 0x67, 0x88, 0x73, 0x8c, 0xb8, 0xc5, 0x30, 0x11, 0xea, 0xf6, 0x10, 0x6a, + 0x51, 0xb3, 0xa8, 0x1d, 0xfc, 0xae, 0x1e, 0xdb, 0xa4, 0xca, 0x61, 0x7c, 0x83, 0xa9, 0xa9, 0x51, + 0xd1, 0xd5, 0x48, 0x52, 0xc8, 0xef, 0x40, 0x39, 0x08, 0x2b, 0xaa, 0x40, 0xa1, 0xbf, 0xbf, 0xdf, + 0xc3, 0xad, 0x0c, 0x6a, 0x00, 0xec, 0xf6, 0xb6, 0x86, 0xca, 0xe0, 0x68, 0xd8, 0xc3, 0xad, 0x2c, + 0x5d, 0x6f, 0x1d, 0xed, 0xee, 0x8a, 0xb5, 0x24, 0x9f, 0x00, 0x4a, 0x5e, 0x36, 0x52, 0x87, 0xaf, + 0x87, 0x00, 0xaa, 0xa3, 0x2b, 0xa2, 0x16, 0xe7, 0xe6, 0x3d, 0x57, 0xf0, 0xca, 0x22, 0xa6, 0x4a, + 0xd5, 0xd1, 0xd9, 0x2f, 0x57, 0xb6, 0xe0, 0x4a, 0xca, 0x2d, 0xe4, 0x32, 0x27, 0xf4, 0xdb, 0x35, + 0x62, 0xf9, 0x5f, 0x25, 0x58, 0x9e, 0x77, 0x87, 0xa0, 0xf6, 0x3d, 0xb6, 0x5c, 0x2f, 0xb0, 0x8f, + 0xfe, 0xa6, 0x30, 0x7a, 0x13, 0x60, 0xbe, 0x2d, 0x60, 0xf6, 0x9b, 0x16, 0x72, 0xdf, 0x25, 0x0e, + 0xf3, 0x85, 0xc4, 0x68, 0xc3, 0x35, 0xc5, 0xd9, 0xaa, 0xeb, 0x3e, 0xb5, 0x1c, 0x8d, 0x4d, 0x42, + 0x15, 0x1c, 0xae, 0x29, 0x4e, 0x53, 0x3d, 0xf5, 0x58, 0x75, 0xf9, 0xf4, 0x5d, 0xc1, 0xe1, 0x9a, + 0xd6, 0xc5, 0x8f, 0x7c, 0xe2, 0x9c, 0xb1, 0xd2, 0x5f, 0xc1, 0x7c, 0x91, 0x70, 0x44, 0xe9, 0xf9, + 0x8e, 0x28, 0x5f, 0x6e, 0x22, 0xb9, 0x0e, 0xc0, 0x52, 0x5f, 0x71, 0x8d, 0x4f, 0x08, 0xbb, 0x66, + 0x17, 0x70, 0x85, 0x41, 0x0e, 0x8d, 0x4f, 0x48, 0x7c, 0x38, 0x87, 0x99, 0xe1, 0x9c, 0x36, 0x23, + 0x7a, 0x0f, 0x70, 0x3d, 0x75, 0x62, 0x8b, 0xec, 0x66, 0xf7, 0xde, 0x0a, 0x6e, 0x86, 0x70, 0x91, + 0xc6, 0xaf, 0x43, 0x8b, 0x75, 0x31, 0x36, 0xc7, 0x09, 0xd2, 0x1a, 0x27, 0x0d, 0xe1, 0x82, 0xf4, + 0x7a, 0xec, 0x7a, 0x52, 0x67, 0xfd, 0x21, 0x72, 0x03, 0xb9, 0x06, 0x65, 0x62, 0x6a, 0x1c, 0xd9, + 0x60, 0xc8, 0x12, 0x31, 0x35, 0x8a, 0x92, 0xff, 0x22, 0x07, 0x25, 0x7a, 0xc7, 0xdc, 0xb5, 0x74, + 0xf4, 0x1e, 0x80, 0xea, 0x79, 0x8e, 0x71, 0xec, 0x7b, 0xe1, 0x70, 0xb0, 0x92, 0x7e, 0x5d, 0xed, + 0x06, 0x74, 0x38, 0xc2, 0x42, 0x8f, 0x38, 0xdd, 0x23, 0x79, 0x6a, 0x25, 0x6e, 0x5d, 0xf4, 0x88, + 0xbf, 0x0b, 0x6d, 0xeb, 0xd8, 0x25, 0xce, 0xc7, 0x84, 0x2b, 0x16, 0x67, 0x92, 0x18, 0xd3, 0x0b, + 0x01, 0xc5, 0x70, 0x86, 0xf9, 0x35, 0x68, 0xba, 0xe4, 0x63, 0xe2, 0xd0, 0x02, 0x6b, 0xfa, 0x93, + 0x63, 0xe2, 0x88, 0x17, 0xe8, 0x46, 0x00, 0xde, 0x67, 0x50, 0xf4, 0x0a, 0xd4, 0x43, 0x42, 0x8f, + 0x9c, 0x7a, 0x22, 0x79, 0x6a, 0x01, 0x70, 0x48, 0x4e, 0x3d, 0xaa, 0xf6, 0xb1, 0xa5, 0x9d, 0xc5, + 0x35, 0x28, 0x72, 0xb5, 0x29, 0x22, 0xb2, 0xb3, 0xfc, 0xd3, 0x3c, 0x94, 0xd9, 0x9d, 0xdc, 0x56, + 0x69, 0xa1, 0xa9, 0xd2, 0xe4, 0x52, 0x5c, 0xcf, 0xa1, 0xc1, 0x66, 0xc9, 0x4f, 0xaf, 0xe9, 0x14, + 0x78, 0xc8, 0x60, 0xe8, 0x0d, 0x58, 0x60, 0x24, 0x49, 0x97, 0xec, 0x64, 0x70, 0x93, 0xa2, 0xa2, + 0x76, 0xc5, 0x23, 0x20, 0x7d, 0xf3, 0x08, 0x6c, 0xc2, 0x92, 0xe7, 0xa8, 0xec, 0x16, 0x12, 0xdf, + 0x92, 0xb9, 0x67, 0x7d, 0xe1, 0xfc, 0xd9, 0x4a, 0x7d, 0x48, 0x09, 0xfa, 0x9b, 0xa2, 0x07, 0x20, + 0x46, 0xdf, 0xd7, 0xa2, 0x6a, 0x74, 0x61, 0xd1, 0xb5, 0x55, 0x33, 0x21, 0xa4, 0xc0, 0x84, 0xb0, + 0x7b, 0x0d, 0xb5, 0x3f, 0x94, 0xb1, 0x40, 0xa9, 0xe3, 0x22, 0x86, 0xf0, 0xa2, 0xa8, 0xc1, 0xa9, + 0x92, 0x98, 0x77, 0xd7, 0xaf, 0x9e, 0x3f, 0x5b, 0x41, 0xbc, 0x74, 0xc7, 0xe4, 0xbd, 0x60, 0x4f, + 0x61, 0x31, 0xa9, 0xef, 0xc0, 0x0b, 0xd3, 0x3c, 0x8f, 0x4b, 0x2c, 0xb1, 0x78, 0x2d, 0x86, 0x49, + 0x1f, 0x65, 0x7b, 0x0b, 0x96, 0x82, 0xfc, 0x8f, 0x33, 0x95, 0x19, 0x13, 0x12, 0x87, 0x21, 0xca, + 0x72, 0x1d, 0xe0, 0x89, 0x61, 0x6a, 0xbc, 0x3a, 0xb3, 0x43, 0x2e, 0xe1, 0x0a, 0x85, 0xb0, 0xf2, + 0xbb, 0x5e, 0xe4, 0xf5, 0x5c, 0xfe, 0x1d, 0x68, 0xd2, 0x60, 0xec, 0x11, 0xcf, 0x31, 0x46, 0xdb, + 0xaa, 0xaf, 0x13, 0xb4, 0x0a, 0xe8, 0x64, 0x6c, 0xa9, 0x29, 0x8d, 0x8e, 0x86, 0xbc, 0xc5, 0x70, + 0xd1, 0x9d, 0x6e, 0x43, 0xcb, 0x30, 0xbd, 0xf4, 0x04, 0x69, 0x18, 0x66, 0x94, 0x76, 0xbd, 0x01, + 0x35, 0x3e, 0x28, 0x73, 0x6a, 0xf9, 0xcf, 0x72, 0xb0, 0x30, 0xdd, 0xff, 0xd0, 0x9f, 0x4c, 0x54, + 0xe7, 0x8c, 0x76, 0xcf, 0x91, 0xe5, 0x9b, 0x69, 0x1a, 0xe0, 0x16, 0xc3, 0x44, 0xf7, 0xbf, 0x05, + 0x2d, 0xd7, 0x9f, 0xa4, 0x9d, 0xd9, 0x86, 0xeb, 0x4f, 0xa2, 0x94, 0x3f, 0x86, 0xe6, 0x47, 0x3e, + 0xbd, 0x2b, 0x8d, 0x49, 0xd0, 0xb5, 0x78, 0x8a, 0xbe, 0x9d, 0x9e, 0xa2, 0x31, 0xad, 0x56, 0x99, + 0xe3, 0xba, 0xde, 0x6f, 0x08, 0x09, 0xb8, 0x11, 0xc8, 0xe2, 0x0d, 0xad, 0xfd, 0xdb, 0xd0, 0x9c, + 0x21, 0xa1, 0x55, 0x3f, 0x20, 0x62, 0xea, 0x67, 0x71, 0xb8, 0xa6, 0x46, 0x46, 0x5d, 0x11, 0x53, + 0xbc, 0xc5, 0x30, 0xd1, 0x63, 0xfb, 0xb3, 0x1c, 0xd4, 0x63, 0xa7, 0x26, 0xb5, 0x23, 0x7f, 0x1f, + 0x8a, 0xa2, 0xce, 0xce, 0x7d, 0x06, 0x8f, 0x09, 0x11, 0x23, 0xeb, 0x4e, 0x06, 0x0b, 0x3e, 0xf4, + 0x0a, 0xd4, 0x78, 0x31, 0x10, 0x89, 0x23, 0x89, 0x92, 0x50, 0xe5, 0x50, 0x66, 0x60, 0xfb, 0x8f, + 0xb3, 0x50, 0x14, 0x85, 0xfb, 0xed, 0xf0, 0x49, 0x2b, 0x32, 0x6d, 0xa6, 0x75, 0x20, 0x98, 0x76, + 0xa0, 0xd4, 0xe1, 0x45, 0x8a, 0x0d, 0x2f, 0xe8, 0x3e, 0x5c, 0x1b, 0xa9, 0xa6, 0x72, 0x4c, 0x94, + 0x0f, 0x5d, 0xcb, 0x54, 0x88, 0x39, 0xb2, 0x34, 0xa2, 0x29, 0xaa, 0xe3, 0xa8, 0x67, 0xe2, 0xc3, + 0xde, 0xd2, 0x48, 0x35, 0xd7, 0xc9, 0x23, 0xd7, 0x32, 0x7b, 0x1c, 0xdb, 0xa5, 0xc8, 0xf5, 0x12, + 0x14, 0x98, 0xea, 0xf2, 0xcf, 0x73, 0x00, 0xd3, 0x28, 0xa6, 0xfa, 0xeb, 0x06, 0xbb, 0xec, 0x8e, + 0x1c, 0x83, 0xdd, 0x91, 0xc5, 0x87, 0xa1, 0x28, 0x88, 0x72, 0xf9, 0xa6, 0xe1, 0x89, 0x5e, 0xcf, + 0x7e, 0xcf, 0x14, 0xb9, 0xfc, 0xff, 0x52, 0x9b, 0x29, 0xa4, 0xb7, 0x99, 0xef, 0x42, 0x41, 0xa7, + 0xc7, 0x72, 0x99, 0xb0, 0x88, 0xbe, 0x7c, 0x51, 0xa6, 0xb2, 0xf3, 0xbb, 0x93, 0xc1, 0x9c, 0x03, + 0xbd, 0x07, 0x25, 0x97, 0xe7, 0xee, 0xf2, 0xc9, 0xbc, 0xcf, 0x12, 0x89, 0x34, 0xdf, 0xc9, 0xe0, + 0x80, 0x8b, 0x16, 0x09, 0x3a, 0xa4, 0xc8, 0xff, 0x9e, 0x05, 0xc4, 0xde, 0x78, 0x4d, 0xcd, 0xb6, + 0xd8, 0x89, 0x36, 0x4f, 0x0c, 0x1d, 0x5d, 0x03, 0xc9, 0x77, 0xc6, 0xdc, 0xa1, 0xeb, 0xa5, 0xf3, + 0x67, 0x2b, 0xd2, 0x11, 0xde, 0xc5, 0x14, 0x86, 0xde, 0x87, 0xd2, 0x63, 0xa2, 0x6a, 0xc4, 0x09, + 0xe6, 0xc2, 0xb7, 0xe6, 0xbc, 0x1a, 0xc7, 0x24, 0xae, 0xee, 0x70, 0x9e, 0x9e, 0xe9, 0x39, 0x67, + 0x38, 0x90, 0x40, 0x4f, 0x91, 0x61, 0xba, 0x64, 0xe4, 0x3b, 0xc1, 0x37, 0xdd, 0x70, 0x8d, 0x96, + 0xa1, 0x44, 0x3d, 0x66, 0xf9, 0x9e, 0x68, 0xa0, 0xc1, 0xb2, 0xfd, 0x00, 0x6a, 0x51, 0x71, 0xa8, + 0x05, 0xd2, 0x13, 0x72, 0x26, 0xc2, 0x4f, 0x7f, 0xd2, 0xb9, 0x8b, 0x27, 0x39, 0x8f, 0x3b, 0x5f, + 0x3c, 0xc8, 0xdd, 0xcf, 0xca, 0x7f, 0x9e, 0x85, 0xd6, 0x74, 0x54, 0x14, 0xe6, 0xb6, 0xa1, 0x4c, + 0xc7, 0xc2, 0x48, 0x12, 0x85, 0xeb, 0x70, 0x7c, 0xcc, 0xa5, 0x8c, 0x8f, 0xd2, 0x9c, 0xf1, 0x31, + 0x7f, 0xc1, 0xf8, 0x58, 0xb8, 0x60, 0x7c, 0x2c, 0xc6, 0xc7, 0x47, 0x79, 0x00, 0x35, 0xea, 0x4a, + 0x4c, 0xf8, 0xfb, 0xdd, 0xff, 0x78, 0x0a, 0x92, 0xff, 0x26, 0x07, 0x57, 0xd3, 0x9f, 0xf4, 0xd1, + 0x1e, 0x34, 0x89, 0x08, 0x19, 0xbd, 0x18, 0x9e, 0x18, 0xc1, 0x67, 0xf0, 0x9b, 0x97, 0x89, 0x2f, + 0x6e, 0x90, 0x78, 0x06, 0x3d, 0x80, 0xb2, 0x23, 0xd4, 0x16, 0x15, 0xab, 0x93, 0x2e, 0x27, 0x30, + 0x0e, 0x87, 0xf4, 0xe8, 0x1e, 0x94, 0x26, 0x2c, 0x71, 0x83, 0x22, 0xfe, 0xd2, 0x45, 0xd9, 0x8d, + 0x03, 0x62, 0xf4, 0x26, 0x14, 0x68, 0x47, 0x0f, 0x0e, 0x6e, 0x3b, 0x9d, 0x8b, 0xb6, 0x6e, 0xcc, + 0x09, 0xd1, 0x77, 0x20, 0x3f, 0xb6, 0xf4, 0xe0, 0x03, 0xfa, 0xb5, 0x74, 0x86, 0x5d, 0x4b, 0xc7, + 0x8c, 0x4c, 0xfe, 0x13, 0x09, 0x5e, 0xba, 0xe8, 0x6b, 0x02, 0x1a, 0xc0, 0x42, 0xe4, 0xcb, 0x44, + 0xcc, 0x8d, 0xf2, 0x45, 0x1f, 0x26, 0x84, 0x13, 0x23, 0x9f, 0x22, 0x84, 0x1b, 0xe3, 0x0f, 0x97, + 0xb9, 0xd9, 0x87, 0x4b, 0x92, 0x7c, 0xd1, 0xe0, 0x1e, 0x7b, 0xf8, 0xcd, 0x3e, 0x83, 0x5c, 0xfc, + 0xc0, 0xd1, 0xfe, 0x34, 0x3b, 0xfb, 0x76, 0xf1, 0x06, 0x20, 0xc3, 0x9c, 0x5e, 0xf1, 0x23, 0x7d, + 0xbc, 0x80, 0x5b, 0x0c, 0x13, 0xad, 0x74, 0x77, 0xe1, 0x6a, 0xcc, 0x2d, 0xe1, 0xdd, 0x47, 0x58, + 0xb4, 0x18, 0xb5, 0x3b, 0xb8, 0x04, 0xcd, 0x36, 0x20, 0xe9, 0x32, 0x0d, 0x48, 0xfe, 0xdb, 0x2c, + 0xb4, 0x66, 0x1f, 0xbc, 0xd0, 0xbb, 0x50, 0x1e, 0x59, 0xa6, 0xeb, 0xa9, 0xa6, 0x27, 0xa2, 0x71, + 0xf1, 0x65, 0x76, 0x27, 0x83, 0x43, 0x06, 0xb4, 0x36, 0xd3, 0x79, 0xe7, 0x3e, 0x62, 0x45, 0x7a, + 0xed, 0x1a, 0xe4, 0x4f, 0x7c, 0x73, 0x24, 0xbe, 0x35, 0xbf, 0x34, 0x6f, 0xb3, 0x2d, 0xdf, 0x1c, + 0xed, 0x64, 0x30, 0xa3, 0x9d, 0x76, 0xb7, 0xbf, 0xcb, 0x41, 0x35, 0xa2, 0x0c, 0xba, 0x03, 0x15, + 0x5a, 0x11, 0x9e, 0xd7, 0x86, 0x59, 0xd9, 0x60, 0x4d, 0x78, 0x05, 0xe0, 0xd8, 0xb2, 0xc6, 0xca, + 0xb4, 0x04, 0x96, 0x77, 0x32, 0xb8, 0x42, 0x61, 0x5c, 0xe2, 0xcb, 0x50, 0x35, 0x4c, 0xef, 0xde, + 0xdd, 0xc8, 0x24, 0x40, 0x47, 0x3a, 0x30, 0xc2, 0x2f, 0x3d, 0xe8, 0x55, 0xa8, 0xb3, 0x71, 0x30, + 0x24, 0xa2, 0x35, 0x2d, 0xbb, 0x93, 0xc1, 0x35, 0x01, 0xe6, 0x64, 0xb3, 0x43, 0x45, 0x21, 0x65, + 0xa8, 0x40, 0xb7, 0x80, 0xf5, 0xbe, 0x7b, 0x77, 0x15, 0xd3, 0x15, 0x74, 0x45, 0xb1, 0x65, 0x9d, + 0x23, 0xf6, 0x5d, 0x4e, 0x79, 0x1f, 0xea, 0xbe, 0x61, 0x7a, 0x6f, 0xad, 0xdd, 0x17, 0x74, 0xfc, + 0x53, 0xee, 0xc2, 0xd4, 0xdc, 0xa3, 0x3e, 0x43, 0xb3, 0x4f, 0xa4, 0x9c, 0x92, 0x4f, 0xbd, 0x81, + 0xf7, 0x1e, 0xe5, 0xcb, 0xe5, 0x56, 0x45, 0xfe, 0x3c, 0x0b, 0x30, 0xf5, 0x71, 0xea, 0x84, 0xf0, + 0x00, 0x2a, 0x86, 0x69, 0x78, 0x8a, 0xea, 0xe8, 0x97, 0x7c, 0xe2, 0x28, 0x53, 0xfa, 0xae, 0xa3, + 0xbb, 0xe8, 0x1e, 0xe4, 0x19, 0x9b, 0x74, 0xe9, 0xf7, 0x71, 0x46, 0x2f, 0xfe, 0x55, 0x05, 0x6f, + 0x67, 0x39, 0x43, 0x43, 0x0f, 0xa0, 0x49, 0xe1, 0x4a, 0x18, 0x5f, 0x5e, 0x8a, 0xd2, 0x03, 0x5c, + 0xa7, 0xa4, 0xc1, 0xca, 0x95, 0xff, 0x23, 0x07, 0x57, 0x52, 0x1e, 0xc3, 0x43, 0x5b, 0xa5, 0x79, + 0xb6, 0xe6, 0xbf, 0x99, 0xad, 0xdf, 0x13, 0xb6, 0xf2, 0x1a, 0xf9, 0xfa, 0xa5, 0x5e, 0xe4, 0x57, + 0xbb, 0x8e, 0x1e, 0x33, 0xb9, 0x78, 0x91, 0xc9, 0xa5, 0x4b, 0x9a, 0xdc, 0xfe, 0x5d, 0x90, 0xba, + 0x8e, 0xfe, 0x7f, 0x7e, 0x9c, 0xa7, 0x47, 0x73, 0x2d, 0x9c, 0x8e, 0xa9, 0x97, 0x2d, 0x8d, 0x88, + 0x07, 0x3c, 0xf6, 0x9b, 0x4e, 0x1d, 0xd1, 0x27, 0x3b, 0xbe, 0xb8, 0xfd, 0x57, 0x12, 0xd4, 0xa2, + 0xff, 0xc0, 0x05, 0x5d, 0x83, 0xa5, 0xc1, 0x41, 0x0f, 0x77, 0x87, 0x03, 0xac, 0x0c, 0x7f, 0x74, + 0xd0, 0x53, 0x8e, 0xf6, 0xdf, 0xdf, 0x1f, 0xfc, 0x70, 0xbf, 0x95, 0x41, 0x2f, 0xc2, 0xd5, 0xbd, + 0xde, 0xde, 0x00, 0xff, 0x48, 0x39, 0x1c, 0x1c, 0xe1, 0x8d, 0x9e, 0x12, 0x10, 0xb6, 0xbe, 0x2a, + 0xa1, 0x6b, 0xb0, 0xb8, 0x8d, 0x0f, 0x36, 0x12, 0xa8, 0x7f, 0x2a, 0x53, 0xd4, 0xd1, 0xe6, 0x70, + 0x2b, 0x81, 0xfa, 0x59, 0x05, 0xb5, 0x61, 0xa9, 0xb7, 0x77, 0x30, 0x4c, 0x4a, 0xfc, 0x43, 0x40, + 0x2b, 0xd0, 0xde, 0xd8, 0xed, 0x6f, 0xbc, 0xbf, 0x33, 0x38, 0x3a, 0xec, 0x25, 0x08, 0xfe, 0x13, + 0xd0, 0x02, 0xd4, 0xf6, 0xba, 0x07, 0x53, 0xd0, 0x67, 0x4d, 0xf4, 0x02, 0xa0, 0xee, 0xf6, 0x36, + 0xee, 0x6d, 0x77, 0x87, 0x11, 0xda, 0xbf, 0x6e, 0xa1, 0x45, 0x68, 0x6e, 0xf5, 0x77, 0x87, 0x3d, + 0x3c, 0x85, 0xfe, 0xd1, 0x02, 0xba, 0x02, 0x8d, 0xdd, 0xfe, 0x5e, 0x7f, 0x38, 0x05, 0xfe, 0x17, + 0x03, 0x1e, 0xed, 0xf7, 0x07, 0xfb, 0x53, 0xe0, 0xe7, 0x08, 0x21, 0xa8, 0x3f, 0x1a, 0xf4, 0x23, + 0xb0, 0x7f, 0xb8, 0x42, 0xed, 0x0a, 0xfc, 0xd1, 0xdf, 0x7f, 0x7f, 0x8a, 0xfa, 0x74, 0x8b, 0xea, + 0xc1, 0xbd, 0x11, 0x43, 0xfc, 0x74, 0x1b, 0x75, 0xe0, 0xda, 0x60, 0xd8, 0xdb, 0x55, 0x7a, 0xbf, + 0x79, 0x30, 0xc0, 0xc3, 0x19, 0xfc, 0xd7, 0xdb, 0xe8, 0x26, 0xac, 0x44, 0x8c, 0x4e, 0xa5, 0xfa, + 0xb7, 0x9d, 0xf5, 0x87, 0x9f, 0x7d, 0xd1, 0xc9, 0xfc, 0xf2, 0x8b, 0x4e, 0xe6, 0xeb, 0x2f, 0x3a, + 0xd9, 0x9f, 0x9c, 0x77, 0xb2, 0x9f, 0x9e, 0x77, 0xb2, 0xff, 0x78, 0xde, 0xc9, 0x7e, 0x76, 0xde, + 0xc9, 0x7e, 0x7e, 0xde, 0xc9, 0x7e, 0x75, 0xde, 0xc9, 0x7c, 0x7d, 0xde, 0xc9, 0xfe, 0xc1, 0x97, + 0x9d, 0xcc, 0x67, 0x5f, 0x76, 0x32, 0xbf, 0xfc, 0xb2, 0x93, 0xf9, 0xad, 0x22, 0xcf, 0xa0, 0xe3, + 0x22, 0xfb, 0x78, 0xfe, 0xf6, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0xa6, 0x01, 0x0f, 0xea, 0x8a, + 0x29, 0x00, 0x00, } func (x OperatorType) String() string { @@ -4085,6 +4496,54 @@ func (this *Operator_OTelSinkOp) Equal(that interface{}) bool { } return true } +func (this *Operator_ClickhouseSourceOp) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Operator_ClickhouseSourceOp) + if !ok { + that2, ok := that.(Operator_ClickhouseSourceOp) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.ClickhouseSourceOp.Equal(that1.ClickhouseSourceOp) { + return false + } + return true +} +func (this *Operator_ClickhouseSinkOp) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*Operator_ClickhouseSinkOp) + if !ok { + that2, ok := that.(Operator_ClickhouseSinkOp) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.ClickhouseSinkOp.Equal(that1.ClickhouseSinkOp) { + return false + } + return true +} func (this *MemorySourceOperator) Equal(that interface{}) bool { if that == nil { return this == nil @@ -4800,6 +5259,79 @@ func (this *EmptySourceOperator) Equal(that interface{}) bool { } return true } +func (this *ClickHouseSourceOperator) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ClickHouseSourceOperator) + if !ok { + that2, ok := that.(ClickHouseSourceOperator) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Host != that1.Host { + return false + } + if this.Port != that1.Port { + return false + } + if this.Username != that1.Username { + return false + } + if this.Password != that1.Password { + return false + } + if this.Database != that1.Database { + return false + } + if this.Query != that1.Query { + return false + } + if len(this.ColumnNames) != len(that1.ColumnNames) { + return false + } + for i := range this.ColumnNames { + if this.ColumnNames[i] != that1.ColumnNames[i] { + return false + } + } + if len(this.ColumnTypes) != len(that1.ColumnTypes) { + return false + } + for i := range this.ColumnTypes { + if this.ColumnTypes[i] != that1.ColumnTypes[i] { + return false + } + } + if this.BatchSize != that1.BatchSize { + return false + } + if this.Streaming != that1.Streaming { + return false + } + if this.TimestampColumn != that1.TimestampColumn { + return false + } + if this.PartitionColumn != that1.PartitionColumn { + return false + } + if this.StartTime != that1.StartTime { + return false + } + if this.EndTime != that1.EndTime { + return false + } + return true +} func (this *OTelLog) Equal(that interface{}) bool { if that == nil { return this == nil @@ -5335,6 +5867,45 @@ func (this *OTelEndpointConfig) Equal(that interface{}) bool { } return true } +func (this *ClickHouseConfig) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ClickHouseConfig) + if !ok { + that2, ok := that.(ClickHouseConfig) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.Hostname != that1.Hostname { + return false + } + if this.Host != that1.Host { + return false + } + if this.Port != that1.Port { + return false + } + if this.Username != that1.Username { + return false + } + if this.Password != that1.Password { + return false + } + if this.Database != that1.Database { + return false + } + return true +} func (this *OTelResource) Equal(that interface{}) bool { if that == nil { return this == nil @@ -5415,6 +5986,71 @@ func (this *OTelExportSinkOperator) Equal(that interface{}) bool { } return true } +func (this *ClickHouseExportSinkOperator) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ClickHouseExportSinkOperator) + if !ok { + that2, ok := that.(ClickHouseExportSinkOperator) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if !this.ClickhouseConfig.Equal(that1.ClickhouseConfig) { + return false + } + if this.TableName != that1.TableName { + return false + } + if len(this.ColumnMappings) != len(that1.ColumnMappings) { + return false + } + for i := range this.ColumnMappings { + if !this.ColumnMappings[i].Equal(that1.ColumnMappings[i]) { + return false + } + } + return true +} +func (this *ClickHouseExportSinkOperator_ColumnMapping) Equal(that interface{}) bool { + if that == nil { + return this == nil + } + + that1, ok := that.(*ClickHouseExportSinkOperator_ColumnMapping) + if !ok { + that2, ok := that.(ClickHouseExportSinkOperator_ColumnMapping) + if ok { + that1 = &that2 + } else { + return false + } + } + if that1 == nil { + return this == nil + } else if this == nil { + return false + } + if this.InputColumnIndex != that1.InputColumnIndex { + return false + } + if this.ClickhouseColumnName != that1.ClickhouseColumnName { + return false + } + if this.ColumnType != that1.ColumnType { + return false + } + return true +} func (this *ScalarExpression) Equal(that interface{}) bool { if that == nil { return this == nil @@ -6005,7 +6641,7 @@ func (this *Operator) GoString() string { if this == nil { return "nil" } - s := make([]string, 0, 18) + s := make([]string, 0, 20) s = append(s, "&planpb.Operator{") s = append(s, "OpType: "+fmt.Sprintf("%#v", this.OpType)+",\n") if this.Op != nil { @@ -6118,6 +6754,22 @@ func (this *Operator_OTelSinkOp) GoString() string { `OTelSinkOp:` + fmt.Sprintf("%#v", this.OTelSinkOp) + `}`}, ", ") return s } +func (this *Operator_ClickhouseSourceOp) GoString() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&planpb.Operator_ClickhouseSourceOp{` + + `ClickhouseSourceOp:` + fmt.Sprintf("%#v", this.ClickhouseSourceOp) + `}`}, ", ") + return s +} +func (this *Operator_ClickhouseSinkOp) GoString() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&planpb.Operator_ClickhouseSinkOp{` + + `ClickhouseSinkOp:` + fmt.Sprintf("%#v", this.ClickhouseSinkOp) + `}`}, ", ") + return s +} func (this *MemorySourceOperator) GoString() string { if this == nil { return "nil" @@ -6368,6 +7020,29 @@ func (this *EmptySourceOperator) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *ClickHouseSourceOperator) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 18) + s = append(s, "&planpb.ClickHouseSourceOperator{") + s = append(s, "Host: "+fmt.Sprintf("%#v", this.Host)+",\n") + s = append(s, "Port: "+fmt.Sprintf("%#v", this.Port)+",\n") + s = append(s, "Username: "+fmt.Sprintf("%#v", this.Username)+",\n") + s = append(s, "Password: "+fmt.Sprintf("%#v", this.Password)+",\n") + s = append(s, "Database: "+fmt.Sprintf("%#v", this.Database)+",\n") + s = append(s, "Query: "+fmt.Sprintf("%#v", this.Query)+",\n") + s = append(s, "ColumnNames: "+fmt.Sprintf("%#v", this.ColumnNames)+",\n") + s = append(s, "ColumnTypes: "+fmt.Sprintf("%#v", this.ColumnTypes)+",\n") + s = append(s, "BatchSize: "+fmt.Sprintf("%#v", this.BatchSize)+",\n") + s = append(s, "Streaming: "+fmt.Sprintf("%#v", this.Streaming)+",\n") + s = append(s, "TimestampColumn: "+fmt.Sprintf("%#v", this.TimestampColumn)+",\n") + s = append(s, "PartitionColumn: "+fmt.Sprintf("%#v", this.PartitionColumn)+",\n") + s = append(s, "StartTime: "+fmt.Sprintf("%#v", this.StartTime)+",\n") + s = append(s, "EndTime: "+fmt.Sprintf("%#v", this.EndTime)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *OTelLog) GoString() string { if this == nil { return "nil" @@ -6576,6 +7251,21 @@ func (this *OTelEndpointConfig) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *ClickHouseConfig) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 10) + s = append(s, "&planpb.ClickHouseConfig{") + s = append(s, "Hostname: "+fmt.Sprintf("%#v", this.Hostname)+",\n") + s = append(s, "Host: "+fmt.Sprintf("%#v", this.Host)+",\n") + s = append(s, "Port: "+fmt.Sprintf("%#v", this.Port)+",\n") + s = append(s, "Username: "+fmt.Sprintf("%#v", this.Username)+",\n") + s = append(s, "Password: "+fmt.Sprintf("%#v", this.Password)+",\n") + s = append(s, "Database: "+fmt.Sprintf("%#v", this.Database)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *OTelResource) GoString() string { if this == nil { return "nil" @@ -6612,6 +7302,34 @@ func (this *OTelExportSinkOperator) GoString() string { s = append(s, "}") return strings.Join(s, "") } +func (this *ClickHouseExportSinkOperator) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 7) + s = append(s, "&planpb.ClickHouseExportSinkOperator{") + if this.ClickhouseConfig != nil { + s = append(s, "ClickhouseConfig: "+fmt.Sprintf("%#v", this.ClickhouseConfig)+",\n") + } + s = append(s, "TableName: "+fmt.Sprintf("%#v", this.TableName)+",\n") + if this.ColumnMappings != nil { + s = append(s, "ColumnMappings: "+fmt.Sprintf("%#v", this.ColumnMappings)+",\n") + } + s = append(s, "}") + return strings.Join(s, "") +} +func (this *ClickHouseExportSinkOperator_ColumnMapping) GoString() string { + if this == nil { + return "nil" + } + s := make([]string, 0, 7) + s = append(s, "&planpb.ClickHouseExportSinkOperator_ColumnMapping{") + s = append(s, "InputColumnIndex: "+fmt.Sprintf("%#v", this.InputColumnIndex)+",\n") + s = append(s, "ClickhouseColumnName: "+fmt.Sprintf("%#v", this.ClickhouseColumnName)+",\n") + s = append(s, "ColumnType: "+fmt.Sprintf("%#v", this.ColumnType)+",\n") + s = append(s, "}") + return strings.Join(s, "") +} func (this *ScalarExpression) GoString() string { if this == nil { return "nil" @@ -7450,16 +8168,16 @@ func (m *Operator_OTelSinkOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { } return len(dAtA) - i, nil } -func (m *Operator_GRPCSinkOp) MarshalTo(dAtA []byte) (int, error) { +func (m *Operator_ClickhouseSourceOp) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Operator_GRPCSinkOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Operator_ClickhouseSourceOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) - if m.GRPCSinkOp != nil { + if m.ClickhouseSourceOp != nil { { - size, err := m.GRPCSinkOp.MarshalToSizedBuffer(dAtA[:i]) + size, err := m.ClickhouseSourceOp.MarshalToSizedBuffer(dAtA[:i]) if err != nil { return 0, err } @@ -7467,30 +8185,74 @@ func (m *Operator_GRPCSinkOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i = encodeVarintPlan(dAtA, i, uint64(size)) } i-- - dAtA[i] = 0x3e - i-- - dAtA[i] = 0xc2 + dAtA[i] = 0x7a } return len(dAtA) - i, nil } -func (m *MemorySourceOperator) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *MemorySourceOperator) MarshalTo(dAtA []byte) (int, error) { +func (m *Operator_ClickhouseSinkOp) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *MemorySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *Operator_ClickhouseSinkOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) - _ = i + if m.ClickhouseSinkOp != nil { + { + size, err := m.ClickhouseSinkOp.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPlan(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1 + i-- + dAtA[i] = 0x82 + } + return len(dAtA) - i, nil +} +func (m *Operator_GRPCSinkOp) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *Operator_GRPCSinkOp) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + if m.GRPCSinkOp != nil { + { + size, err := m.GRPCSinkOp.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPlan(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x3e + i-- + dAtA[i] = 0xc2 + } + return len(dAtA) - i, nil +} +func (m *MemorySourceOperator) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *MemorySourceOperator) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *MemorySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i var l int _ = l if m.Streaming { @@ -7535,20 +8297,20 @@ func (m *MemorySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { dAtA[i] = 0x2a } if len(m.ColumnTypes) > 0 { - dAtA25 := make([]byte, len(m.ColumnTypes)*10) - var j24 int + dAtA27 := make([]byte, len(m.ColumnTypes)*10) + var j26 int for _, num := range m.ColumnTypes { for num >= 1<<7 { - dAtA25[j24] = uint8(uint64(num)&0x7f | 0x80) + dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j24++ + j26++ } - dAtA25[j24] = uint8(num) - j24++ + dAtA27[j26] = uint8(num) + j26++ } - i -= j24 - copy(dAtA[i:], dAtA25[:j24]) - i = encodeVarintPlan(dAtA, i, uint64(j24)) + i -= j26 + copy(dAtA[i:], dAtA27[:j26]) + i = encodeVarintPlan(dAtA, i, uint64(j26)) i-- dAtA[i] = 0x22 } @@ -7562,21 +8324,21 @@ func (m *MemorySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.ColumnIdxs) > 0 { - dAtA27 := make([]byte, len(m.ColumnIdxs)*10) - var j26 int + dAtA29 := make([]byte, len(m.ColumnIdxs)*10) + var j28 int for _, num1 := range m.ColumnIdxs { num := uint64(num1) for num >= 1<<7 { - dAtA27[j26] = uint8(uint64(num)&0x7f | 0x80) + dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j26++ + j28++ } - dAtA27[j26] = uint8(num) - j26++ + dAtA29[j28] = uint8(num) + j28++ } - i -= j26 - copy(dAtA[i:], dAtA27[:j26]) - i = encodeVarintPlan(dAtA, i, uint64(j26)) + i -= j28 + copy(dAtA[i:], dAtA29[:j28]) + i = encodeVarintPlan(dAtA, i, uint64(j28)) i-- dAtA[i] = 0x12 } @@ -7611,20 +8373,20 @@ func (m *MemorySinkOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.ColumnSemanticTypes) > 0 { - dAtA29 := make([]byte, len(m.ColumnSemanticTypes)*10) - var j28 int + dAtA31 := make([]byte, len(m.ColumnSemanticTypes)*10) + var j30 int for _, num := range m.ColumnSemanticTypes { for num >= 1<<7 { - dAtA29[j28] = uint8(uint64(num)&0x7f | 0x80) + dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j28++ + j30++ } - dAtA29[j28] = uint8(num) - j28++ + dAtA31[j30] = uint8(num) + j30++ } - i -= j28 - copy(dAtA[i:], dAtA29[:j28]) - i = encodeVarintPlan(dAtA, i, uint64(j28)) + i -= j30 + copy(dAtA[i:], dAtA31[:j30]) + i = encodeVarintPlan(dAtA, i, uint64(j30)) i-- dAtA[i] = 0x22 } @@ -7638,20 +8400,20 @@ func (m *MemorySinkOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.ColumnTypes) > 0 { - dAtA31 := make([]byte, len(m.ColumnTypes)*10) - var j30 int + dAtA33 := make([]byte, len(m.ColumnTypes)*10) + var j32 int for _, num := range m.ColumnTypes { for num >= 1<<7 { - dAtA31[j30] = uint8(uint64(num)&0x7f | 0x80) + dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j30++ + j32++ } - dAtA31[j30] = uint8(num) - j30++ + dAtA33[j32] = uint8(num) + j32++ } - i -= j30 - copy(dAtA[i:], dAtA31[:j30]) - i = encodeVarintPlan(dAtA, i, uint64(j30)) + i -= j32 + copy(dAtA[i:], dAtA33[:j32]) + i = encodeVarintPlan(dAtA, i, uint64(j32)) i-- dAtA[i] = 0x12 } @@ -7695,20 +8457,20 @@ func (m *GRPCSourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { } } if len(m.ColumnTypes) > 0 { - dAtA33 := make([]byte, len(m.ColumnTypes)*10) - var j32 int + dAtA35 := make([]byte, len(m.ColumnTypes)*10) + var j34 int for _, num := range m.ColumnTypes { for num >= 1<<7 { - dAtA33[j32] = uint8(uint64(num)&0x7f | 0x80) + dAtA35[j34] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j32++ + j34++ } - dAtA33[j32] = uint8(num) - j32++ + dAtA35[j34] = uint8(num) + j34++ } - i -= j32 - copy(dAtA[i:], dAtA33[:j32]) - i = encodeVarintPlan(dAtA, i, uint64(j32)) + i -= j34 + copy(dAtA[i:], dAtA35[:j34]) + i = encodeVarintPlan(dAtA, i, uint64(j34)) i-- dAtA[i] = 0xa } @@ -7820,20 +8582,20 @@ func (m *GRPCSinkOperator_ResultTable) MarshalToSizedBuffer(dAtA []byte) (int, e var l int _ = l if len(m.ColumnSemanticTypes) > 0 { - dAtA37 := make([]byte, len(m.ColumnSemanticTypes)*10) - var j36 int + dAtA39 := make([]byte, len(m.ColumnSemanticTypes)*10) + var j38 int for _, num := range m.ColumnSemanticTypes { for num >= 1<<7 { - dAtA37[j36] = uint8(uint64(num)&0x7f | 0x80) + dAtA39[j38] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j36++ + j38++ } - dAtA37[j36] = uint8(num) - j36++ + dAtA39[j38] = uint8(num) + j38++ } - i -= j36 - copy(dAtA[i:], dAtA37[:j36]) - i = encodeVarintPlan(dAtA, i, uint64(j36)) + i -= j38 + copy(dAtA[i:], dAtA39[:j38]) + i = encodeVarintPlan(dAtA, i, uint64(j38)) i-- dAtA[i] = 0x22 } @@ -7847,20 +8609,20 @@ func (m *GRPCSinkOperator_ResultTable) MarshalToSizedBuffer(dAtA []byte) (int, e } } if len(m.ColumnTypes) > 0 { - dAtA39 := make([]byte, len(m.ColumnTypes)*10) - var j38 int + dAtA41 := make([]byte, len(m.ColumnTypes)*10) + var j40 int for _, num := range m.ColumnTypes { for num >= 1<<7 { - dAtA39[j38] = uint8(uint64(num)&0x7f | 0x80) + dAtA41[j40] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j38++ + j40++ } - dAtA39[j38] = uint8(num) - j38++ + dAtA41[j40] = uint8(num) + j40++ } - i -= j38 - copy(dAtA[i:], dAtA39[:j38]) - i = encodeVarintPlan(dAtA, i, uint64(j38)) + i -= j40 + copy(dAtA[i:], dAtA41[:j40]) + i = encodeVarintPlan(dAtA, i, uint64(j40)) i-- dAtA[i] = 0x12 } @@ -8119,20 +8881,20 @@ func (m *LimitOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.AbortableSrcs) > 0 { - dAtA42 := make([]byte, len(m.AbortableSrcs)*10) - var j41 int + dAtA44 := make([]byte, len(m.AbortableSrcs)*10) + var j43 int for _, num := range m.AbortableSrcs { for num >= 1<<7 { - dAtA42[j41] = uint8(uint64(num)&0x7f | 0x80) + dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j41++ + j43++ } - dAtA42[j41] = uint8(num) - j41++ + dAtA44[j43] = uint8(num) + j43++ } - i -= j41 - copy(dAtA[i:], dAtA42[:j41]) - i = encodeVarintPlan(dAtA, i, uint64(j41)) + i -= j43 + copy(dAtA[i:], dAtA44[:j43]) + i = encodeVarintPlan(dAtA, i, uint64(j43)) i-- dAtA[i] = 0x1a } @@ -8230,21 +8992,21 @@ func (m *UnionOperator_ColumnMapping) MarshalToSizedBuffer(dAtA []byte) (int, er var l int _ = l if len(m.ColumnIndexes) > 0 { - dAtA44 := make([]byte, len(m.ColumnIndexes)*10) - var j43 int + dAtA46 := make([]byte, len(m.ColumnIndexes)*10) + var j45 int for _, num1 := range m.ColumnIndexes { num := uint64(num1) for num >= 1<<7 { - dAtA44[j43] = uint8(uint64(num)&0x7f | 0x80) + dAtA46[j45] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j43++ + j45++ } - dAtA44[j43] = uint8(num) - j43++ + dAtA46[j45] = uint8(num) + j45++ } - i -= j43 - copy(dAtA[i:], dAtA44[:j43]) - i = encodeVarintPlan(dAtA, i, uint64(j43)) + i -= j45 + copy(dAtA[i:], dAtA46[:j45]) + i = encodeVarintPlan(dAtA, i, uint64(j45)) i-- dAtA[i] = 0xa } @@ -8452,20 +9214,20 @@ func (m *EmptySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.ColumnTypes) > 0 { - dAtA46 := make([]byte, len(m.ColumnTypes)*10) - var j45 int + dAtA48 := make([]byte, len(m.ColumnTypes)*10) + var j47 int for _, num := range m.ColumnTypes { for num >= 1<<7 { - dAtA46[j45] = uint8(uint64(num)&0x7f | 0x80) + dAtA48[j47] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j45++ + j47++ } - dAtA46[j45] = uint8(num) - j45++ + dAtA48[j47] = uint8(num) + j47++ } - i -= j45 - copy(dAtA[i:], dAtA46[:j45]) - i = encodeVarintPlan(dAtA, i, uint64(j45)) + i -= j47 + copy(dAtA[i:], dAtA48[:j47]) + i = encodeVarintPlan(dAtA, i, uint64(j47)) i-- dAtA[i] = 0x12 } @@ -8481,6 +9243,135 @@ func (m *EmptySourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } +func (m *ClickHouseSourceOperator) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClickHouseSourceOperator) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ClickHouseSourceOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.EndTime != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.EndTime)) + i-- + dAtA[i] = 0x70 + } + if m.StartTime != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.StartTime)) + i-- + dAtA[i] = 0x68 + } + if len(m.PartitionColumn) > 0 { + i -= len(m.PartitionColumn) + copy(dAtA[i:], m.PartitionColumn) + i = encodeVarintPlan(dAtA, i, uint64(len(m.PartitionColumn))) + i-- + dAtA[i] = 0x62 + } + if len(m.TimestampColumn) > 0 { + i -= len(m.TimestampColumn) + copy(dAtA[i:], m.TimestampColumn) + i = encodeVarintPlan(dAtA, i, uint64(len(m.TimestampColumn))) + i-- + dAtA[i] = 0x5a + } + if m.Streaming { + i-- + if m.Streaming { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i-- + dAtA[i] = 0x50 + } + if m.BatchSize != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.BatchSize)) + i-- + dAtA[i] = 0x48 + } + if len(m.ColumnTypes) > 0 { + dAtA50 := make([]byte, len(m.ColumnTypes)*10) + var j49 int + for _, num := range m.ColumnTypes { + for num >= 1<<7 { + dAtA50[j49] = uint8(uint64(num)&0x7f | 0x80) + num >>= 7 + j49++ + } + dAtA50[j49] = uint8(num) + j49++ + } + i -= j49 + copy(dAtA[i:], dAtA50[:j49]) + i = encodeVarintPlan(dAtA, i, uint64(j49)) + i-- + dAtA[i] = 0x42 + } + if len(m.ColumnNames) > 0 { + for iNdEx := len(m.ColumnNames) - 1; iNdEx >= 0; iNdEx-- { + i -= len(m.ColumnNames[iNdEx]) + copy(dAtA[i:], m.ColumnNames[iNdEx]) + i = encodeVarintPlan(dAtA, i, uint64(len(m.ColumnNames[iNdEx]))) + i-- + dAtA[i] = 0x3a + } + } + if len(m.Query) > 0 { + i -= len(m.Query) + copy(dAtA[i:], m.Query) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Query))) + i-- + dAtA[i] = 0x32 + } + if len(m.Database) > 0 { + i -= len(m.Database) + copy(dAtA[i:], m.Database) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Database))) + i-- + dAtA[i] = 0x2a + } + if len(m.Password) > 0 { + i -= len(m.Password) + copy(dAtA[i:], m.Password) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Password))) + i-- + dAtA[i] = 0x22 + } + if len(m.Username) > 0 { + i -= len(m.Username) + copy(dAtA[i:], m.Username) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Username))) + i-- + dAtA[i] = 0x1a + } + if m.Port != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.Port)) + i-- + dAtA[i] = 0x10 + } + if len(m.Host) > 0 { + i -= len(m.Host) + copy(dAtA[i:], m.Host) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Host))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + func (m *OTelLog) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -9083,7 +9974,7 @@ func (m *OTelEndpointConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *OTelResource) Marshal() (dAtA []byte, err error) { +func (m *ClickHouseConfig) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -9093,25 +9984,88 @@ func (m *OTelResource) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *OTelResource) MarshalTo(dAtA []byte) (int, error) { +func (m *ClickHouseConfig) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *OTelResource) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *ClickHouseConfig) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Attributes) > 0 { - for iNdEx := len(m.Attributes) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Attributes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintPlan(dAtA, i, uint64(size)) + if len(m.Database) > 0 { + i -= len(m.Database) + copy(dAtA[i:], m.Database) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Database))) + i-- + dAtA[i] = 0x32 + } + if len(m.Password) > 0 { + i -= len(m.Password) + copy(dAtA[i:], m.Password) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Password))) + i-- + dAtA[i] = 0x2a + } + if len(m.Username) > 0 { + i -= len(m.Username) + copy(dAtA[i:], m.Username) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Username))) + i-- + dAtA[i] = 0x22 + } + if m.Port != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.Port)) + i-- + dAtA[i] = 0x18 + } + if len(m.Host) > 0 { + i -= len(m.Host) + copy(dAtA[i:], m.Host) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Host))) + i-- + dAtA[i] = 0x12 + } + if len(m.Hostname) > 0 { + i -= len(m.Hostname) + copy(dAtA[i:], m.Hostname) + i = encodeVarintPlan(dAtA, i, uint64(len(m.Hostname))) + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *OTelResource) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *OTelResource) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *OTelResource) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.Attributes) > 0 { + for iNdEx := len(m.Attributes) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.Attributes[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPlan(dAtA, i, uint64(size)) } i-- dAtA[i] = 0xa @@ -9209,6 +10163,102 @@ func (m *OTelExportSinkOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) return len(dAtA) - i, nil } +func (m *ClickHouseExportSinkOperator) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClickHouseExportSinkOperator) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ClickHouseExportSinkOperator) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if len(m.ColumnMappings) > 0 { + for iNdEx := len(m.ColumnMappings) - 1; iNdEx >= 0; iNdEx-- { + { + size, err := m.ColumnMappings[iNdEx].MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPlan(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0x1a + } + } + if len(m.TableName) > 0 { + i -= len(m.TableName) + copy(dAtA[i:], m.TableName) + i = encodeVarintPlan(dAtA, i, uint64(len(m.TableName))) + i-- + dAtA[i] = 0x12 + } + if m.ClickhouseConfig != nil { + { + size, err := m.ClickhouseConfig.MarshalToSizedBuffer(dAtA[:i]) + if err != nil { + return 0, err + } + i -= size + i = encodeVarintPlan(dAtA, i, uint64(size)) + } + i-- + dAtA[i] = 0xa + } + return len(dAtA) - i, nil +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) Marshal() (dAtA []byte, err error) { + size := m.Size() + dAtA = make([]byte, size) + n, err := m.MarshalToSizedBuffer(dAtA[:size]) + if err != nil { + return nil, err + } + return dAtA[:n], nil +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) MarshalTo(dAtA []byte) (int, error) { + size := m.Size() + return m.MarshalToSizedBuffer(dAtA[:size]) +} + +func (m *ClickHouseExportSinkOperator_ColumnMapping) MarshalToSizedBuffer(dAtA []byte) (int, error) { + i := len(dAtA) + _ = i + var l int + _ = l + if m.ColumnType != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.ColumnType)) + i-- + dAtA[i] = 0x18 + } + if len(m.ClickhouseColumnName) > 0 { + i -= len(m.ClickhouseColumnName) + copy(dAtA[i:], m.ClickhouseColumnName) + i = encodeVarintPlan(dAtA, i, uint64(len(m.ClickhouseColumnName))) + i-- + dAtA[i] = 0x12 + } + if m.InputColumnIndex != 0 { + i = encodeVarintPlan(dAtA, i, uint64(m.InputColumnIndex)) + i-- + dAtA[i] = 0x8 + } + return len(dAtA) - i, nil +} + func (m *ScalarExpression) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) @@ -9451,20 +10501,20 @@ func (m *ScalarFunc) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.ArgsDataTypes) > 0 { - dAtA57 := make([]byte, len(m.ArgsDataTypes)*10) - var j56 int + dAtA62 := make([]byte, len(m.ArgsDataTypes)*10) + var j61 int for _, num := range m.ArgsDataTypes { for num >= 1<<7 { - dAtA57[j56] = uint8(uint64(num)&0x7f | 0x80) + dAtA62[j61] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j56++ + j61++ } - dAtA57[j56] = uint8(num) - j56++ + dAtA62[j61] = uint8(num) + j61++ } - i -= j56 - copy(dAtA[i:], dAtA57[:j56]) - i = encodeVarintPlan(dAtA, i, uint64(j56)) + i -= j61 + copy(dAtA[i:], dAtA62[:j61]) + i = encodeVarintPlan(dAtA, i, uint64(j61)) i-- dAtA[i] = 0x2a } @@ -9532,20 +10582,20 @@ func (m *AggregateExpression) MarshalToSizedBuffer(dAtA []byte) (int, error) { var l int _ = l if len(m.ArgsDataTypes) > 0 { - dAtA59 := make([]byte, len(m.ArgsDataTypes)*10) - var j58 int + dAtA64 := make([]byte, len(m.ArgsDataTypes)*10) + var j63 int for _, num := range m.ArgsDataTypes { for num >= 1<<7 { - dAtA59[j58] = uint8(uint64(num)&0x7f | 0x80) + dAtA64[j63] = uint8(uint64(num)&0x7f | 0x80) num >>= 7 - j58++ + j63++ } - dAtA59[j58] = uint8(num) - j58++ + dAtA64[j63] = uint8(num) + j63++ } - i -= j58 - copy(dAtA[i:], dAtA59[:j58]) - i = encodeVarintPlan(dAtA, i, uint64(j58)) + i -= j63 + copy(dAtA[i:], dAtA64[:j63]) + i = encodeVarintPlan(dAtA, i, uint64(j63)) i-- dAtA[i] = 0x3a } @@ -10018,6 +11068,30 @@ func (m *Operator_OTelSinkOp) Size() (n int) { } return n } +func (m *Operator_ClickhouseSourceOp) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ClickhouseSourceOp != nil { + l = m.ClickhouseSourceOp.Size() + n += 1 + l + sovPlan(uint64(l)) + } + return n +} +func (m *Operator_ClickhouseSinkOp) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.ClickhouseSinkOp != nil { + l = m.ClickhouseSinkOp.Size() + n += 2 + l + sovPlan(uint64(l)) + } + return n +} func (m *Operator_GRPCSinkOp) Size() (n int) { if m == nil { return 0 @@ -10471,6 +11545,71 @@ func (m *EmptySourceOperator) Size() (n int) { return n } +func (m *ClickHouseSourceOperator) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + if m.Port != 0 { + n += 1 + sovPlan(uint64(m.Port)) + } + l = len(m.Username) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Password) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Database) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Query) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + if len(m.ColumnNames) > 0 { + for _, s := range m.ColumnNames { + l = len(s) + n += 1 + l + sovPlan(uint64(l)) + } + } + if len(m.ColumnTypes) > 0 { + l = 0 + for _, e := range m.ColumnTypes { + l += sovPlan(uint64(e)) + } + n += 1 + sovPlan(uint64(l)) + l + } + if m.BatchSize != 0 { + n += 1 + sovPlan(uint64(m.BatchSize)) + } + if m.Streaming { + n += 2 + } + l = len(m.TimestampColumn) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.PartitionColumn) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + if m.StartTime != 0 { + n += 1 + sovPlan(uint64(m.StartTime)) + } + if m.EndTime != 0 { + n += 1 + sovPlan(uint64(m.EndTime)) + } + return n +} + func (m *OTelLog) Size() (n int) { if m == nil { return 0 @@ -10763,6 +11902,38 @@ func (m *OTelEndpointConfig) Size() (n int) { return n } +func (m *ClickHouseConfig) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + l = len(m.Hostname) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Host) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + if m.Port != 0 { + n += 1 + sovPlan(uint64(m.Port)) + } + l = len(m.Username) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Password) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.Database) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + return n +} + func (m *OTelResource) Size() (n int) { if m == nil { return 0 @@ -10813,43 +11984,85 @@ func (m *OTelExportSinkOperator) Size() (n int) { return n } -func (m *ScalarExpression) Size() (n int) { +func (m *ClickHouseExportSinkOperator) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Value != nil { - n += m.Value.Size() + if m.ClickhouseConfig != nil { + l = m.ClickhouseConfig.Size() + n += 1 + l + sovPlan(uint64(l)) + } + l = len(m.TableName) + if l > 0 { + n += 1 + l + sovPlan(uint64(l)) + } + if len(m.ColumnMappings) > 0 { + for _, e := range m.ColumnMappings { + l = e.Size() + n += 1 + l + sovPlan(uint64(l)) + } } return n } -func (m *ScalarExpression_Constant) Size() (n int) { +func (m *ClickHouseExportSinkOperator_ColumnMapping) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Constant != nil { - l = m.Constant.Size() + if m.InputColumnIndex != 0 { + n += 1 + sovPlan(uint64(m.InputColumnIndex)) + } + l = len(m.ClickhouseColumnName) + if l > 0 { n += 1 + l + sovPlan(uint64(l)) } + if m.ColumnType != 0 { + n += 1 + sovPlan(uint64(m.ColumnType)) + } return n } -func (m *ScalarExpression_Column) Size() (n int) { + +func (m *ScalarExpression) Size() (n int) { if m == nil { return 0 } var l int _ = l - if m.Column != nil { - l = m.Column.Size() - n += 1 + l + sovPlan(uint64(l)) + if m.Value != nil { + n += m.Value.Size() } return n } -func (m *ScalarExpression_Func) Size() (n int) { + +func (m *ScalarExpression_Constant) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Constant != nil { + l = m.Constant.Size() + n += 1 + l + sovPlan(uint64(l)) + } + return n +} +func (m *ScalarExpression_Column) Size() (n int) { + if m == nil { + return 0 + } + var l int + _ = l + if m.Column != nil { + l = m.Column.Size() + n += 1 + l + sovPlan(uint64(l)) + } + return n +} +func (m *ScalarExpression_Func) Size() (n int) { if m == nil { return 0 } @@ -11299,6 +12512,26 @@ func (this *Operator_OTelSinkOp) String() string { }, "") return s } +func (this *Operator_ClickhouseSourceOp) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Operator_ClickhouseSourceOp{`, + `ClickhouseSourceOp:` + strings.Replace(fmt.Sprintf("%v", this.ClickhouseSourceOp), "ClickHouseSourceOperator", "ClickHouseSourceOperator", 1) + `,`, + `}`, + }, "") + return s +} +func (this *Operator_ClickhouseSinkOp) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&Operator_ClickhouseSinkOp{`, + `ClickhouseSinkOp:` + strings.Replace(fmt.Sprintf("%v", this.ClickhouseSinkOp), "ClickHouseExportSinkOperator", "ClickHouseExportSinkOperator", 1) + `,`, + `}`, + }, "") + return s +} func (this *Operator_GRPCSinkOp) String() string { if this == nil { return "nil" @@ -11580,6 +12813,29 @@ func (this *EmptySourceOperator) String() string { }, "") return s } +func (this *ClickHouseSourceOperator) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ClickHouseSourceOperator{`, + `Host:` + fmt.Sprintf("%v", this.Host) + `,`, + `Port:` + fmt.Sprintf("%v", this.Port) + `,`, + `Username:` + fmt.Sprintf("%v", this.Username) + `,`, + `Password:` + fmt.Sprintf("%v", this.Password) + `,`, + `Database:` + fmt.Sprintf("%v", this.Database) + `,`, + `Query:` + fmt.Sprintf("%v", this.Query) + `,`, + `ColumnNames:` + fmt.Sprintf("%v", this.ColumnNames) + `,`, + `ColumnTypes:` + fmt.Sprintf("%v", this.ColumnTypes) + `,`, + `BatchSize:` + fmt.Sprintf("%v", this.BatchSize) + `,`, + `Streaming:` + fmt.Sprintf("%v", this.Streaming) + `,`, + `TimestampColumn:` + fmt.Sprintf("%v", this.TimestampColumn) + `,`, + `PartitionColumn:` + fmt.Sprintf("%v", this.PartitionColumn) + `,`, + `StartTime:` + fmt.Sprintf("%v", this.StartTime) + `,`, + `EndTime:` + fmt.Sprintf("%v", this.EndTime) + `,`, + `}`, + }, "") + return s +} func (this *OTelLog) String() string { if this == nil { return "nil" @@ -11806,6 +13062,21 @@ func (this *OTelEndpointConfig) String() string { }, "") return s } +func (this *ClickHouseConfig) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ClickHouseConfig{`, + `Hostname:` + fmt.Sprintf("%v", this.Hostname) + `,`, + `Host:` + fmt.Sprintf("%v", this.Host) + `,`, + `Port:` + fmt.Sprintf("%v", this.Port) + `,`, + `Username:` + fmt.Sprintf("%v", this.Username) + `,`, + `Password:` + fmt.Sprintf("%v", this.Password) + `,`, + `Database:` + fmt.Sprintf("%v", this.Database) + `,`, + `}`, + }, "") + return s +} func (this *OTelResource) String() string { if this == nil { return "nil" @@ -11850,6 +13121,35 @@ func (this *OTelExportSinkOperator) String() string { }, "") return s } +func (this *ClickHouseExportSinkOperator) String() string { + if this == nil { + return "nil" + } + repeatedStringForColumnMappings := "[]*ClickHouseExportSinkOperator_ColumnMapping{" + for _, f := range this.ColumnMappings { + repeatedStringForColumnMappings += strings.Replace(fmt.Sprintf("%v", f), "ClickHouseExportSinkOperator_ColumnMapping", "ClickHouseExportSinkOperator_ColumnMapping", 1) + "," + } + repeatedStringForColumnMappings += "}" + s := strings.Join([]string{`&ClickHouseExportSinkOperator{`, + `ClickhouseConfig:` + strings.Replace(this.ClickhouseConfig.String(), "ClickHouseConfig", "ClickHouseConfig", 1) + `,`, + `TableName:` + fmt.Sprintf("%v", this.TableName) + `,`, + `ColumnMappings:` + repeatedStringForColumnMappings + `,`, + `}`, + }, "") + return s +} +func (this *ClickHouseExportSinkOperator_ColumnMapping) String() string { + if this == nil { + return "nil" + } + s := strings.Join([]string{`&ClickHouseExportSinkOperator_ColumnMapping{`, + `InputColumnIndex:` + fmt.Sprintf("%v", this.InputColumnIndex) + `,`, + `ClickhouseColumnName:` + fmt.Sprintf("%v", this.ClickhouseColumnName) + `,`, + `ColumnType:` + fmt.Sprintf("%v", this.ColumnType) + `,`, + `}`, + }, "") + return s +} func (this *ScalarExpression) String() string { if this == nil { return "nil" @@ -13522,6 +14822,76 @@ func (m *Operator) Unmarshal(dAtA []byte) error { } m.Op = &Operator_OTelSinkOp{v} iNdEx = postIndex + case 15: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseSourceOp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &ClickHouseSourceOperator{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Op = &Operator_ClickhouseSourceOp{v} + iNdEx = postIndex + case 16: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseSinkOp", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &ClickHouseExportSinkOperator{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Op = &Operator_ClickhouseSinkOp{v} + iNdEx = postIndex case 1000: if wireType != 2 { return fmt.Errorf("proto: wrong wireType = %d for field GRPCSinkOp", wireType) @@ -16420,7 +17790,7 @@ func (m *EmptySourceOperator) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelLog) Unmarshal(dAtA []byte) error { +func (m *ClickHouseSourceOperator) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -16443,17 +17813,17 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelLog: wiretype end group for non-group") + return fmt.Errorf("proto: ClickHouseSourceOperator: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelLog: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ClickHouseSourceOperator: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16463,31 +17833,29 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.Attributes = append(m.Attributes, &OTelAttribute{}) - if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Host = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) } - m.TimeColumnIndex = 0 + m.Port = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16497,16 +17865,16 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.TimeColumnIndex |= int64(b&0x7F) << shift + m.Port |= int32(b&0x7F) << shift if b < 0x80 { break } } case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ObservedTimeColumnIndex", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Username", wireType) } - m.ObservedTimeColumnIndex = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16516,16 +17884,29 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ObservedTimeColumnIndex |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Username = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SeverityNumber", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } - m.SeverityNumber = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16535,14 +17916,27 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.SeverityNumber |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Password = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field SeverityText", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Database", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -16570,13 +17964,13 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.SeverityText = string(dAtA[iNdEx:postIndex]) + m.Database = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field BodyColumnIndex", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Query", wireType) } - m.BodyColumnIndex = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16586,79 +17980,42 @@ func (m *OTelLog) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.BodyColumnIndex |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - default: - iNdEx = preIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { - return err + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan } - if (skippy < 0) || (iNdEx+skippy) < 0 { + postIndex := iNdEx + intStringLen + if postIndex < 0 { return ErrInvalidLengthPlan } - if (iNdEx + skippy) > l { + if postIndex > l { return io.ErrUnexpectedEOF } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OTelSpan) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan + m.Query = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 7: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnNames", wireType) } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: OTelSpan: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: OTelSpan: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NameString", wireType) - } - var stringLen uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLen |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } } intStringLen := int(stringLen) if intStringLen < 0 { @@ -16671,67 +18028,82 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Name = &OTelSpan_NameString{string(dAtA[iNdEx:postIndex])} + m.ColumnNames = append(m.ColumnNames, string(dAtA[iNdEx:postIndex])) iNdEx = postIndex - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field NameColumnIndex", wireType) - } - var v int64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan + case 8: + if wireType == 0 { + var v typespb.DataType + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= typespb.DataType(b&0x7F) << shift + if b < 0x80 { + break + } } - if iNdEx >= l { - return io.ErrUnexpectedEOF + m.ColumnTypes = append(m.ColumnTypes, v) + } else if wireType == 2 { + var packedLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + packedLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } } - b := dAtA[iNdEx] - iNdEx++ - v |= int64(b&0x7F) << shift - if b < 0x80 { - break + if packedLen < 0 { + return ErrInvalidLengthPlan } - } - m.Name = &OTelSpan_NameColumnIndex{v} - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan + postIndex := iNdEx + packedLen + if postIndex < 0 { + return ErrInvalidLengthPlan } - if iNdEx >= l { + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break + var elementCount int + if elementCount != 0 && len(m.ColumnTypes) == 0 { + m.ColumnTypes = make([]typespb.DataType, 0, elementCount) } + for iNdEx < postIndex { + var v typespb.DataType + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= typespb.DataType(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ColumnTypes = append(m.ColumnTypes, v) + } + } else { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnTypes", wireType) } - if msglen < 0 { - return ErrInvalidLengthPlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Attributes = append(m.Attributes, &OTelAttribute{}) - if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 4: + case 9: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TraceIDColumn", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BatchSize", wireType) } - m.TraceIDColumn = 0 + m.BatchSize = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16741,16 +18113,16 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.TraceIDColumn |= int64(b&0x7F) << shift + m.BatchSize |= int32(b&0x7F) << shift if b < 0x80 { break } } - case 5: + case 10: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SpanIDColumn", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Streaming", wireType) } - m.SpanIDColumn = 0 + var v int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16760,16 +18132,17 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.SpanIDColumn |= int64(b&0x7F) << shift + v |= int(b&0x7F) << shift if b < 0x80 { break } } - case 6: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ParentSpanIDColumn", wireType) + m.Streaming = bool(v != 0) + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field TimestampColumn", wireType) } - m.ParentSpanIDColumn = 0 + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16779,16 +18152,29 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ParentSpanIDColumn |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - case 7: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field StartTimeColumnIndex", wireType) + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan } - m.StartTimeColumnIndex = 0 + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.TimestampColumn = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 12: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field PartitionColumn", wireType) + } + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16798,16 +18184,29 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.StartTimeColumnIndex |= int64(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - case 8: + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.PartitionColumn = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 13: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field EndTimeColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field StartTime", wireType) } - m.EndTimeColumnIndex = 0 + m.StartTime = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16817,16 +18216,16 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.EndTimeColumnIndex |= int64(b&0x7F) << shift + m.StartTime |= int64(b&0x7F) << shift if b < 0x80 { break } } - case 9: + case 14: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field KindValue", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field EndTime", wireType) } - m.KindValue = 0 + m.EndTime = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16836,7 +18235,7 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.KindValue |= int64(b&0x7F) << shift + m.EndTime |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -16862,7 +18261,7 @@ func (m *OTelSpan) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelMetricGauge) Unmarshal(dAtA []byte) error { +func (m *OTelLog) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -16885,17 +18284,17 @@ func (m *OTelMetricGauge) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelMetricGauge: wiretype end group for non-group") + return fmt.Errorf("proto: OTelLog: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelMetricGauge: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: OTelLog: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field FloatColumnIndex", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) } - var v int64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16905,17 +18304,31 @@ func (m *OTelMetricGauge) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - m.ValueColumn = &OTelMetricGauge_FloatColumnIndex{v} + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attributes = append(m.Attributes, &OTelAttribute{}) + if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex case 2: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field IntColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TimeColumnIndex", wireType) } - var v int64 + m.TimeColumnIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16925,67 +18338,16 @@ func (m *OTelMetricGauge) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int64(b&0x7F) << shift + m.TimeColumnIndex |= int64(b&0x7F) << shift if b < 0x80 { break } } - m.ValueColumn = &OTelMetricGauge_IntColumnIndex{v} - default: - iNdEx = preIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { - return err + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ObservedTimeColumnIndex", wireType) } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthPlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OTelMetricSummary) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: OTelMetricSummary: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: OTelMetricSummary: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field CountColumnIndex", wireType) - } - m.CountColumnIndex = 0 + m.ObservedTimeColumnIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -16995,16 +18357,16 @@ func (m *OTelMetricSummary) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.CountColumnIndex |= int64(b&0x7F) << shift + m.ObservedTimeColumnIndex |= int64(b&0x7F) << shift if b < 0x80 { break } } - case 2: + case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field SumColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SeverityNumber", wireType) } - m.SumColumnIndex = 0 + m.SeverityNumber = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17014,16 +18376,16 @@ func (m *OTelMetricSummary) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.SumColumnIndex |= int64(b&0x7F) << shift + m.SeverityNumber |= int64(b&0x7F) << shift if b < 0x80 { break } } - case 3: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field QuantileValues", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SeverityText", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17033,92 +18395,29 @@ func (m *OTelMetricSummary) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.QuantileValues = append(m.QuantileValues, &OTelMetricSummary_ValueAtQuantile{}) - if err := m.QuantileValues[len(m.QuantileValues)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.SeverityText = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthPlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OTelMetricSummary_ValueAtQuantile) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: ValueAtQuantile: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: ValueAtQuantile: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 1 { - return fmt.Errorf("proto: wrong wireType = %d for field Quantile", wireType) - } - var v uint64 - if (iNdEx + 8) > l { - return io.ErrUnexpectedEOF - } - v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) - iNdEx += 8 - m.Quantile = float64(math.Float64frombits(v)) - case 2: + case 6: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ValueColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field BodyColumnIndex", wireType) } - m.ValueColumnIndex = 0 + m.BodyColumnIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17128,7 +18427,7 @@ func (m *OTelMetricSummary_ValueAtQuantile) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ValueColumnIndex |= int64(b&0x7F) << shift + m.BodyColumnIndex |= int64(b&0x7F) << shift if b < 0x80 { break } @@ -17154,7 +18453,7 @@ func (m *OTelMetricSummary_ValueAtQuantile) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelAttribute) Unmarshal(dAtA []byte) error { +func (m *OTelSpan) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -17177,15 +18476,15 @@ func (m *OTelAttribute) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelAttribute: wiretype end group for non-group") + return fmt.Errorf("proto: OTelSpan: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelAttribute: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: OTelSpan: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field NameString", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -17213,13 +18512,13 @@ func (m *OTelAttribute) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Name = string(dAtA[iNdEx:postIndex]) + m.Name = &OTelSpan_NameString{string(dAtA[iNdEx:postIndex])} iNdEx = postIndex case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Column", wireType) + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field NameColumnIndex", wireType) } - var msglen int + var v int64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17229,32 +18528,17 @@ func (m *OTelAttribute) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + v |= int64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return ErrInvalidLengthPlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - v := &OTelAttribute_Column{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - m.Value = &OTelAttribute_Column_{v} - iNdEx = postIndex + m.Name = &OTelSpan_NameColumnIndex{v} case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field StringValue", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17264,79 +18548,31 @@ func (m *OTelAttribute) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.Value = &OTelAttribute_StringValue{string(dAtA[iNdEx:postIndex])} - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { + m.Attributes = append(m.Attributes, &OTelAttribute{}) + if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthPlan - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Column: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Column: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + iNdEx = postIndex + case 4: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnType", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TraceIDColumn", wireType) } - m.ColumnType = 0 + m.TraceIDColumn = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17346,14 +18582,619 @@ func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ColumnType |= typespb.DataType(b&0x7F) << shift + m.TraceIDColumn |= int64(b&0x7F) << shift if b < 0x80 { break } } - case 2: + case 5: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field ColumnIndex", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field SpanIDColumn", wireType) + } + m.SpanIDColumn = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SpanIDColumn |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 6: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ParentSpanIDColumn", wireType) + } + m.ParentSpanIDColumn = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ParentSpanIDColumn |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 7: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field StartTimeColumnIndex", wireType) + } + m.StartTimeColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.StartTimeColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 8: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field EndTimeColumnIndex", wireType) + } + m.EndTimeColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.EndTimeColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 9: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field KindValue", wireType) + } + m.KindValue = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.KindValue |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelMetricGauge) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelMetricGauge: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelMetricGauge: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field FloatColumnIndex", wireType) + } + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ValueColumn = &OTelMetricGauge_FloatColumnIndex{v} + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field IntColumnIndex", wireType) + } + var v int64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.ValueColumn = &OTelMetricGauge_IntColumnIndex{v} + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelMetricSummary) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelMetricSummary: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelMetricSummary: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CountColumnIndex", wireType) + } + m.CountColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.CountColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field SumColumnIndex", wireType) + } + m.SumColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.SumColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field QuantileValues", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.QuantileValues = append(m.QuantileValues, &OTelMetricSummary_ValueAtQuantile{}) + if err := m.QuantileValues[len(m.QuantileValues)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelMetricSummary_ValueAtQuantile) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ValueAtQuantile: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ValueAtQuantile: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 1 { + return fmt.Errorf("proto: wrong wireType = %d for field Quantile", wireType) + } + var v uint64 + if (iNdEx + 8) > l { + return io.ErrUnexpectedEOF + } + v = uint64(encoding_binary.LittleEndian.Uint64(dAtA[iNdEx:])) + iNdEx += 8 + m.Quantile = float64(math.Float64frombits(v)) + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ValueColumnIndex", wireType) + } + m.ValueColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ValueColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelAttribute) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelAttribute: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelAttribute: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Column", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &OTelAttribute_Column{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Value = &OTelAttribute_Column_{v} + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StringValue", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Value = &OTelAttribute_StringValue{string(dAtA[iNdEx:postIndex])} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: Column: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: Column: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnType", wireType) + } + m.ColumnType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ColumnType |= typespb.DataType(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnIndex", wireType) } m.ColumnIndex = 0 for shift := uint(0); ; shift += 7 { @@ -17365,14 +19206,512 @@ func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.ColumnIndex |= int64(b&0x7F) << shift + m.ColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field CanBeJsonEncodedArray", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + m.CanBeJsonEncodedArray = bool(v != 0) + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelMetric) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelMetric: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelMetric: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Name = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Description", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Description = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 3: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Unit", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Unit = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Attributes = append(m.Attributes, &OTelAttribute{}) + if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex + case 5: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field TimeColumnIndex", wireType) + } + m.TimeColumnIndex = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.TimeColumnIndex |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 101: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Gauge", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &OTelMetricGauge{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Data = &OTelMetric_Gauge{v} + iNdEx = postIndex + case 102: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Summary", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + v := &OTelMetricSummary{} + if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + m.Data = &OTelMetric_Summary{v} + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelEndpointConfig: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelEndpointConfig: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field URL", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.URL = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + case 2: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Headers", wireType) + } + var msglen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Headers == nil { + m.Headers = make(map[string]string) + } + var mapkey string + var mapvalue string + for iNdEx < postIndex { + entryPreIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + if fieldNum == 1 { + var stringLenmapkey uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapkey |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapkey := int(stringLenmapkey) + if intStringLenmapkey < 0 { + return ErrInvalidLengthPlan + } + postStringIndexmapkey := iNdEx + intStringLenmapkey + if postStringIndexmapkey < 0 { + return ErrInvalidLengthPlan + } + if postStringIndexmapkey > l { + return io.ErrUnexpectedEOF + } + mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) + iNdEx = postStringIndexmapkey + } else if fieldNum == 2 { + var stringLenmapvalue uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLenmapvalue |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLenmapvalue := int(stringLenmapvalue) + if intStringLenmapvalue < 0 { + return ErrInvalidLengthPlan + } + postStringIndexmapvalue := iNdEx + intStringLenmapvalue + if postStringIndexmapvalue < 0 { + return ErrInvalidLengthPlan + } + if postStringIndexmapvalue > l { + return io.ErrUnexpectedEOF + } + mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) + iNdEx = postStringIndexmapvalue + } else { + iNdEx = entryPreIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err + } + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > postIndex { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + m.Headers[mapkey] = mapvalue + iNdEx = postIndex case 3: if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field CanBeJsonEncodedArray", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Insecure", wireType) } var v int for shift := uint(0); ; shift += 7 { @@ -17389,7 +19728,26 @@ func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { break } } - m.CanBeJsonEncodedArray = bool(v != 0) + m.Insecure = bool(v != 0) + case 4: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Timeout", wireType) + } + m.Timeout = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Timeout |= int64(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPlan(dAtA[iNdEx:]) @@ -17411,7 +19769,7 @@ func (m *OTelAttribute_Column) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelMetric) Unmarshal(dAtA []byte) error { +func (m *ClickHouseConfig) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -17434,15 +19792,15 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelMetric: wiretype end group for non-group") + return fmt.Errorf("proto: ClickHouseConfig: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelMetric: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ClickHouseConfig: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Name", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Hostname", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -17470,11 +19828,11 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Name = string(dAtA[iNdEx:postIndex]) + m.Hostname = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Description", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Host", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -17502,11 +19860,30 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Description = string(dAtA[iNdEx:postIndex]) + m.Host = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Port", wireType) + } + m.Port = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.Port |= int32(b&0x7F) << shift + if b < 0x80 { + break + } + } + case 4: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Unit", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Username", wireType) } var stringLen uint64 for shift := uint(0); ; shift += 7 { @@ -17534,13 +19911,13 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Unit = string(dAtA[iNdEx:postIndex]) + m.Username = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 4: + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Password", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17550,50 +19927,29 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.Attributes = append(m.Attributes, &OTelAttribute{}) - if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.Password = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex - case 5: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field TimeColumnIndex", wireType) - } - m.TimeColumnIndex = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.TimeColumnIndex |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 101: + case 6: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Gauge", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Database", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17603,30 +19959,77 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - v := &OTelMetricGauge{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Database = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { return err } - m.Data = &OTelMetric_Gauge{v} - iNdEx = postIndex - case 102: + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *OTelResource) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: OTelResource: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: OTelResource: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Summary", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -17653,11 +20056,10 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - v := &OTelMetricSummary{} - if err := v.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Attributes = append(m.Attributes, &OTelAttribute{}) + if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } - m.Data = &OTelMetric_Summary{v} iNdEx = postIndex default: iNdEx = preIndex @@ -17680,7 +20082,7 @@ func (m *OTelMetric) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { +func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -17703,17 +20105,17 @@ func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelEndpointConfig: wiretype end group for non-group") + return fmt.Errorf("proto: OTelExportSinkOperator: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelEndpointConfig: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: OTelExportSinkOperator: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field URL", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field EndpointConfig", wireType) } - var stringLen uint64 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17723,156 +20125,69 @@ func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - stringLen |= uint64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - intStringLen := int(stringLen) - if intStringLen < 0 { + if msglen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + intStringLen + postIndex := iNdEx + msglen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.URL = string(dAtA[iNdEx:postIndex]) + if m.EndpointConfig == nil { + m.EndpointConfig = &OTelEndpointConfig{} + } + if err := m.EndpointConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Headers", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Resource", wireType) } var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthPlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - if m.Headers == nil { - m.Headers = make(map[string]string) - } - var mapkey string - var mapvalue string - for iNdEx < postIndex { - entryPreIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - if fieldNum == 1 { - var stringLenmapkey uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapkey |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapkey := int(stringLenmapkey) - if intStringLenmapkey < 0 { - return ErrInvalidLengthPlan - } - postStringIndexmapkey := iNdEx + intStringLenmapkey - if postStringIndexmapkey < 0 { - return ErrInvalidLengthPlan - } - if postStringIndexmapkey > l { - return io.ErrUnexpectedEOF - } - mapkey = string(dAtA[iNdEx:postStringIndexmapkey]) - iNdEx = postStringIndexmapkey - } else if fieldNum == 2 { - var stringLenmapvalue uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - stringLenmapvalue |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - intStringLenmapvalue := int(stringLenmapvalue) - if intStringLenmapvalue < 0 { - return ErrInvalidLengthPlan - } - postStringIndexmapvalue := iNdEx + intStringLenmapvalue - if postStringIndexmapvalue < 0 { - return ErrInvalidLengthPlan - } - if postStringIndexmapvalue > l { - return io.ErrUnexpectedEOF - } - mapvalue = string(dAtA[iNdEx:postStringIndexmapvalue]) - iNdEx = postStringIndexmapvalue - } else { - iNdEx = entryPreIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthPlan - } - if (iNdEx + skippy) > postIndex { - return io.ErrUnexpectedEOF - } - iNdEx += skippy + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + msglen |= int(b&0x7F) << shift + if b < 0x80 { + break } } - m.Headers[mapkey] = mapvalue + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + if m.Resource == nil { + m.Resource = &OTelResource{} + } + if err := m.Resource.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } iNdEx = postIndex case 3: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Insecure", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Metrics", wireType) } - var v int + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17882,17 +20197,31 @@ func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - v |= int(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - m.Insecure = bool(v != 0) + if msglen < 0 { + return ErrInvalidLengthPlan + } + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Metrics = append(m.Metrics, &OTelMetric{}) + if err := m.Metrics[len(m.Metrics)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err + } + iNdEx = postIndex case 4: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Timeout", wireType) + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Spans", wireType) } - m.Timeout = 0 + var msglen int for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -17902,64 +20231,29 @@ func (m *OTelEndpointConfig) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - m.Timeout |= int64(b&0x7F) << shift + msglen |= int(b&0x7F) << shift if b < 0x80 { break } } - default: - iNdEx = preIndex - skippy, err := skipPlan(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { + if msglen < 0 { return ErrInvalidLengthPlan } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} -func (m *OTelResource) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowPlan + postIndex := iNdEx + msglen + if postIndex < 0 { + return ErrInvalidLengthPlan } - if iNdEx >= l { + if postIndex > l { return io.ErrUnexpectedEOF } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break + m.Spans = append(m.Spans, &OTelSpan{}) + if err := m.Spans[len(m.Spans)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + return err } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: OTelResource: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: OTelResource: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: + iNdEx = postIndex + case 5: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Attributes", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -17986,8 +20280,8 @@ func (m *OTelResource) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Attributes = append(m.Attributes, &OTelAttribute{}) - if err := m.Attributes[len(m.Attributes)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.Logs = append(m.Logs, &OTelLog{}) + if err := m.Logs[len(m.Logs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex @@ -18012,7 +20306,7 @@ func (m *OTelResource) Unmarshal(dAtA []byte) error { } return nil } -func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { +func (m *ClickHouseExportSinkOperator) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -18035,15 +20329,15 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: OTelExportSinkOperator: wiretype end group for non-group") + return fmt.Errorf("proto: ClickHouseExportSinkOperator: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: OTelExportSinkOperator: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: ClickHouseExportSinkOperator: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field EndpointConfig", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseConfig", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -18070,18 +20364,18 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - if m.EndpointConfig == nil { - m.EndpointConfig = &OTelEndpointConfig{} + if m.ClickhouseConfig == nil { + m.ClickhouseConfig = &ClickHouseConfig{} } - if err := m.EndpointConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + if err := m.ClickhouseConfig.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Resource", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field TableName", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -18091,31 +20385,27 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - if m.Resource == nil { - m.Resource = &OTelResource{} - } - if err := m.Resource.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.TableName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex case 3: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Metrics", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ColumnMappings", wireType) } var msglen int for shift := uint(0); ; shift += 7 { @@ -18142,16 +20432,66 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.Metrics = append(m.Metrics, &OTelMetric{}) - if err := m.Metrics[len(m.Metrics)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { + m.ColumnMappings = append(m.ColumnMappings, &ClickHouseExportSinkOperator_ColumnMapping{}) + if err := m.ColumnMappings[len(m.ColumnMappings)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err } iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Spans", wireType) + default: + iNdEx = preIndex + skippy, err := skipPlan(dAtA[iNdEx:]) + if err != nil { + return err } - var msglen int + if (skippy < 0) || (iNdEx+skippy) < 0 { + return ErrInvalidLengthPlan + } + if (iNdEx + skippy) > l { + return io.ErrUnexpectedEOF + } + iNdEx += skippy + } + } + + if iNdEx > l { + return io.ErrUnexpectedEOF + } + return nil +} +func (m *ClickHouseExportSinkOperator_ColumnMapping) Unmarshal(dAtA []byte) error { + l := len(dAtA) + iNdEx := 0 + for iNdEx < l { + preIndex := iNdEx + var wire uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + wire |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + fieldNum := int32(wire >> 3) + wireType := int(wire & 0x7) + if wireType == 4 { + return fmt.Errorf("proto: ColumnMapping: wiretype end group for non-group") + } + if fieldNum <= 0 { + return fmt.Errorf("proto: ColumnMapping: illegal tag %d (wire type %d)", fieldNum, wire) + } + switch fieldNum { + case 1: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field InputColumnIndex", wireType) + } + m.InputColumnIndex = 0 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -18161,31 +20501,16 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + m.InputColumnIndex |= int32(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { - return ErrInvalidLengthPlan - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthPlan - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Spans = append(m.Spans, &OTelSpan{}) - if err := m.Spans[len(m.Spans)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex - case 5: + case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Logs", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field ClickhouseColumnName", wireType) } - var msglen int + var stringLen uint64 for shift := uint(0); ; shift += 7 { if shift >= 64 { return ErrIntOverflowPlan @@ -18195,26 +20520,43 @@ func (m *OTelExportSinkOperator) Unmarshal(dAtA []byte) error { } b := dAtA[iNdEx] iNdEx++ - msglen |= int(b&0x7F) << shift + stringLen |= uint64(b&0x7F) << shift if b < 0x80 { break } } - if msglen < 0 { + intStringLen := int(stringLen) + if intStringLen < 0 { return ErrInvalidLengthPlan } - postIndex := iNdEx + msglen + postIndex := iNdEx + intStringLen if postIndex < 0 { return ErrInvalidLengthPlan } if postIndex > l { return io.ErrUnexpectedEOF } - m.Logs = append(m.Logs, &OTelLog{}) - if err := m.Logs[len(m.Logs)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } + m.ClickhouseColumnName = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 3: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field ColumnType", wireType) + } + m.ColumnType = 0 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowPlan + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + m.ColumnType |= typespb.DataType(b&0x7F) << shift + if b < 0x80 { + break + } + } default: iNdEx = preIndex skippy, err := skipPlan(dAtA[iNdEx:]) diff --git a/src/carnot/planpb/plan.proto b/src/carnot/planpb/plan.proto index c7bcb552dda..738b4793c08 100644 --- a/src/carnot/planpb/plan.proto +++ b/src/carnot/planpb/plan.proto @@ -104,6 +104,7 @@ enum OperatorType { GRPC_SOURCE_OPERATOR = 1100; UDTF_SOURCE_OPERATOR = 1200; EMPTY_SOURCE_OPERATOR = 1300; + CLICKHOUSE_SOURCE_OPERATOR = 1400; // Regular operators are range 2000 - 10000. MAP_OPERATOR = 2000; AGGREGATE_OPERATOR = 2100; @@ -115,6 +116,7 @@ enum OperatorType { MEMORY_SINK_OPERATOR = 9000; GRPC_SINK_OPERATOR = 9100; OTEL_EXPORT_SINK_OPERATOR = 9200; + CLICKHOUSE_EXPORT_SINK_OPERATOR = 9300; } // The Logical operation performed. Each operator needs and entry in this @@ -149,6 +151,10 @@ message Operator { EmptySourceOperator empty_source_op = 13; // OTelExportSinkOperator writes the input table to an OpenTelemetry endpoint. OTelExportSinkOperator otel_sink_op = 14 [ (gogoproto.customname) = "OTelSinkOp" ]; + // ClickHouseSourceOperator reads data from a ClickHouse database. + ClickHouseSourceOperator clickhouse_source_op = 15; + // ClickHouseExportSinkOperator writes the input table to a ClickHouse database. + ClickHouseExportSinkOperator clickhouse_sink_op = 16; } } @@ -358,6 +364,44 @@ message EmptySourceOperator { repeated px.types.DataType column_types = 2; } +// Source operator that queries a ClickHouse database. +message ClickHouseSourceOperator { + // Connection parameters + string host = 1; + int32 port = 2; + string username = 3; + string password = 4; + string database = 5; + + // Query to execute + string query = 6; + + // The names for the columns (can be auto-detected from query) + repeated string column_names = 7; + // The types of the columns (can be auto-detected from query) + repeated px.types.DataType column_types = 8; + + // Batch size for fetching results + int32 batch_size = 9; + + // Whether to stream results (future enhancement) + bool streaming = 10; + + // Column name to use for timestamp-based filtering and ordering + // This column should be of DateTime or DateTime64 type + string timestamp_column = 11; + + // Column name to use for partitioning + // The underlying ClickHouse table should be partitioned by this column + string partition_column = 12; + + // Start time for time-based filtering (nanoseconds since epoch) + int64 start_time = 13; + + // End time for time-based filtering (nanoseconds since epoch) + int64 end_time = 14; +} + // OTelLog maps operator columns to each field in the OpenTelemetry Log configuration. // The mapping ensures that each row of the table will be a separate log. // Maps to the config described here: @@ -524,6 +568,22 @@ message OTelEndpointConfig { int64 timeout = 4; } +// ClickHouseConfig contains the connection parameters for ClickHouse. +message ClickHouseConfig { + // The hostname of the node executing the query. + string hostname = 1; + // The ClickHouse server host. + string host = 2; + // The ClickHouse server port. + int32 port = 3; + // The ClickHouse username. + string username = 4; + // The ClickHouse password. + string password = 5; + // The ClickHouse database name. + string database = 6; +} + // Defines a resource. Discussed in depth in the OpenTelemetry spec. // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/resource/sdk.md message OTelResource { @@ -547,6 +607,24 @@ message OTelExportSinkOperator { repeated OTelLog logs = 5; } +message ClickHouseExportSinkOperator { + // ClickHouse connection parameters + ClickHouseConfig clickhouse_config = 1; + // Target table name to insert data into + string table_name = 2; + // Column mapping: maps input column indices to ClickHouse table columns + repeated ColumnMapping column_mappings = 3; + + message ColumnMapping { + // Index of the column in the input row batch + int32 input_column_index = 1; + // Name of the column in the ClickHouse table + string clickhouse_column_name = 2; + // Data type of the column + px.types.DataType column_type = 3; + } +} + // Scalar expression is any single valued expression. message ScalarExpression { oneof value { diff --git a/src/carnot/planpb/test_proto.h b/src/carnot/planpb/test_proto.h index 0ca5a1c37a4..227d5ad7dd2 100644 --- a/src/carnot/planpb/test_proto.h +++ b/src/carnot/planpb/test_proto.h @@ -195,6 +195,26 @@ column_names: "usage" streaming: false )"; +constexpr char kClickHouseSourceOperator[] = R"( +host: "localhost" +port: 9000 +username: "default" +password: "test_password" +database: "default" +query: "SELECT id, name, value FROM test_table" +batch_size: 2 +streaming: false +column_names: "id" +column_names: "name" +column_names: "value" +column_types: INT64 +column_types: STRING +column_types: FLOAT64 +timestamp_column: "timestamp" +start_time: 1000000000000000000 +end_time: 9223372036854775807 +)"; + constexpr char kBlockingAggOperator1[] = R"( windowed: false values { @@ -1024,6 +1044,64 @@ constexpr char kPlanWithTwoSourcesWithLimits[] = R"proto( } )proto"; +constexpr char kPlanWithOTelExport[] = R"proto( + id: 1, + dag { + nodes { + id: 1 + sorted_children: 2 + } + nodes { + id: 2 + sorted_parents: 1 + } + } + nodes { + id: 1 + op { + op_type: MEMORY_SOURCE_OPERATOR + mem_source_op { + name: "numbers" + column_idxs: 0 + column_types: INT64 + column_names: "a" + column_idxs: 1 + column_types: BOOLEAN + column_names: "b" + column_idxs: 2 + column_types: FLOAT64 + column_names: "c" + } + } + } + nodes { + id: 2 + op { + op_type: OTEL_EXPORT_SINK_OPERATOR + otel_sink_op { + endpoint_config { + url: "0.0.0.0:55690" + headers { + key: "apikey" + value: "12345" + } + timeout: 5 + } + resource { + attributes { + name: "service.name" + column { + column_type: STRING + column_index: 1 + can_be_json_encoded_array: true + } + } + } + } + } + } +)proto"; + constexpr char kOneLimit3Sources[] = R"proto( id: 1, dag { @@ -1328,6 +1406,14 @@ planpb::Operator CreateTestSource1PB(const std::string& table_name = "cpu") { return op; } +planpb::Operator CreateClickHouseSourceOperatorPB() { + planpb::Operator op; + auto op_proto = absl::Substitute(kOperatorProtoTmpl, "CLICKHOUSE_SOURCE_OPERATOR", + "clickhouse_source_op", kClickHouseSourceOperator); + CHECK(google::protobuf::TextFormat::MergeFromString(op_proto, &op)) << "Failed to parse proto"; + return op; +} + planpb::Operator CreateTestStreamingSource1PB(const std::string& table_name = "cpu") { planpb::Operator op; auto mem_proto = absl::Substitute(kStreamingMemSourceOperator1, table_name); @@ -1378,6 +1464,32 @@ planpb::Operator CreateTestSink1PB() { return op; } +// Create a test ClickHouse source operator with hardcoded values +planpb::Operator CreateTestClickHouseSourcePB() { + constexpr char kClickHouseSourceOperator[] = R"( + host: "localhost" + port: 9000 + username: "default" + password: "test_password" + database: "default" + query: "SELECT id, name, value FROM test_table ORDER BY id" + batch_size: 1024 + streaming: false + column_names: "id" + column_names: "name" + column_names: "value" + column_types: UINT64 + column_types: STRING + column_types: FLOAT64 + )"; + + planpb::Operator op; + auto op_proto = absl::Substitute(kOperatorProtoTmpl, "CLICKHOUSE_SOURCE_OPERATOR", + "clickhouse_source_op", kClickHouseSourceOperator); + CHECK(google::protobuf::TextFormat::MergeFromString(op_proto, &op)) << "Failed to parse proto"; + return op; +} + planpb::Operator CreateTestSink2PB() { planpb::Operator op; auto op_proto = absl::Substitute(kOperatorProtoTmpl, "MEMORY_SINK_OPERATOR", "mem_sink_op", diff --git a/src/common/testing/protobuf.h b/src/common/testing/protobuf.h index dfd6091a4e6..07da54be26a 100644 --- a/src/common/testing/protobuf.h +++ b/src/common/testing/protobuf.h @@ -66,7 +66,7 @@ struct ProtoMatcher { } virtual void DescribeTo(::std::ostream* os) const { - *os << "equals to text probobuf: " << expected_text_pb_; + *os << "equals to text protobuf: " << expected_text_pb_; } virtual void DescribeNegationTo(::std::ostream* os) const { @@ -97,7 +97,7 @@ struct PartiallyEqualsProtoMatcher : public ProtoMatcher { } void DescribeTo(::std::ostream* os) const override { - *os << "partially equals to text probobuf: " << expected_text_pb_; + *os << "partially equals to text protobuf: " << expected_text_pb_; } void DescribeNegationTo(::std::ostream* os) const override { diff --git a/src/common/uuid/uuid_utils.h b/src/common/uuid/uuid_utils.h index 90207d75491..792a79453e3 100644 --- a/src/common/uuid/uuid_utils.h +++ b/src/common/uuid/uuid_utils.h @@ -49,6 +49,10 @@ inline void ClearUUID(sole::uuid* uuid) { uuid->cd = 0; } +inline bool operator==(const px::uuidpb::UUID& lhs, const px::uuidpb::UUID& rhs) { + return lhs.low_bits() == rhs.low_bits() && lhs.high_bits() == rhs.high_bits(); +} + } // namespace px // Allow UUID to be logged. diff --git a/src/experimental/standalone_pem/vizier_server.h b/src/experimental/standalone_pem/vizier_server.h index ce071bf379c..44856ff585a 100644 --- a/src/experimental/standalone_pem/vizier_server.h +++ b/src/experimental/standalone_pem/vizier_server.h @@ -63,6 +63,7 @@ class VizierServer final : public api::vizierpb::VizierService::Service { LOG(INFO) << "Executing Script"; auto query_id = sole::uuid4(); + auto compiler_state = engine_state_->CreateLocalExecutionCompilerState(0); // Handle mutations. @@ -81,6 +82,7 @@ class VizierServer final : public api::vizierpb::VizierService::Service { auto deployments = mutations->Deployments(); bool tracepoints_running = true; + auto ntp_info = TracepointInfo{}; for (size_t i = 0; i < deployments.size(); i++) { carnot::planner::dynamic_tracing::ir::logical::TracepointDeployment planner_tp; auto s = deployments[i]->ToProto(&planner_tp); @@ -99,7 +101,6 @@ class VizierServer final : public api::vizierpb::VizierService::Service { if (!s.ok()) { return ::grpc::Status(grpc::StatusCode::INTERNAL, "Failed to register tracepoint"); } - auto ntp_info = TracepointInfo{}; ntp_info.name = stirling_tp.name(); ntp_info.id = tp_id; ntp_info.current_state = statuspb::PENDING_STATE; @@ -116,10 +117,6 @@ class VizierServer final : public api::vizierpb::VizierService::Service { response->Write(mutation_resp); return ::grpc::Status::CANCELLED; } - - auto m_info = mutation_resp.mutable_mutation_info(); - m_info->mutable_status()->set_code(0); - response->Write(mutation_resp); } LOG(INFO) << "Compiling and running query"; // Send schema before sending query results. diff --git a/src/shared/version/BUILD.bazel b/src/shared/version/BUILD.bazel index a94f6553cec..835730c7f4c 100644 --- a/src/shared/version/BUILD.bazel +++ b/src/shared/version/BUILD.bazel @@ -77,6 +77,7 @@ pl_cc_library_internal( # be restricted to binaries. # TODO(zasgar): Refactor dependent code so we can more precisely apply the visbility rules. visibility = [ + "//src/carnot:__pkg__", "//src/carnot/planner/docs:__pkg__", "//src/experimental:__subpackages__", "//src/vizier/funcs:__pkg__", diff --git a/src/stirling/core/source_connector.cc b/src/stirling/core/source_connector.cc index a9566daea49..54fa5137cc3 100644 --- a/src/stirling/core/source_connector.cc +++ b/src/stirling/core/source_connector.cc @@ -20,7 +20,7 @@ #include #include -#include +#include #include "src/stirling/core/source_connector.h" diff --git a/src/stirling/source_connectors/socket_tracer/BUILD.bazel b/src/stirling/source_connectors/socket_tracer/BUILD.bazel index 47301fffdb5..90154e7edc4 100644 --- a/src/stirling/source_connectors/socket_tracer/BUILD.bazel +++ b/src/stirling/source_connectors/socket_tracer/BUILD.bazel @@ -16,7 +16,10 @@ load("//bazel:pl_build_system.bzl", "pl_cc_binary", "pl_cc_bpf_test", "pl_cc_library", "pl_cc_test") -package(default_visibility = ["//src/stirling:__subpackages__"]) +package(default_visibility = [ + "//src/stirling:__subpackages__", + "//src/carnot:__subpackages__", +]) pl_cc_library( name = "cc_library", @@ -461,7 +464,7 @@ pl_cc_bpf_test( "//src/stirling/source_connectors/socket_tracer/testing/container_images:node_12_3_1_container", "//src/stirling/source_connectors/socket_tracer/testing/container_images:node_14_18_1_alpine_container", "//src/stirling/source_connectors/socket_tracer/testing/container_images:node_client_container", - "//src/stirling/source_connectors/socket_tracer/testing/container_images:python_3_10_container", + "//src/stirling/source_connectors/socket_tracer/testing/container_images:python_min_310_container", "//src/stirling/source_connectors/socket_tracer/testing/container_images:ruby_container", "//src/stirling/testing:cc_library", ], diff --git a/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel b/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel index fb6c1a02e56..2b6e32d678a 100644 --- a/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel +++ b/src/stirling/source_connectors/socket_tracer/testing/container_images/BUILD.bazel @@ -14,11 +14,15 @@ # # SPDX-License-Identifier: Apache-2.0 +load("@io_bazel_rules_docker//container:container.bzl", "container_image", "container_layer") load("//bazel:pl_build_system.bzl", "pl_boringcrypto_go_sdk", "pl_cc_test_library", "pl_go_sdk_version_template_to_label", "pl_go_test_versions", "pl_supported_go_sdk_versions") pl_all_supported_go_sdk_versions = pl_supported_go_sdk_versions + pl_boringcrypto_go_sdk -package(default_visibility = ["//src/stirling:__subpackages__"]) +package(default_visibility = [ + "//src/stirling:__subpackages__", + "//src/carnot:__subpackages__", +]) pl_cc_test_library( name = "bssl_container", @@ -318,9 +322,9 @@ pl_cc_test_library( ) pl_cc_test_library( - name = "python_3_10_container", + name = "python_min_310_container", srcs = [], - hdrs = ["python_3_10_container.h"], + hdrs = ["python_min_310_container.h"], data = [ "//src/stirling/source_connectors/socket_tracer/testing/containers/ssl:python_min_310_https_server.tar", ], @@ -416,3 +420,18 @@ pl_cc_test_library( ], deps = ["//src/common/testing/test_utils:cc_library"], ) + +# ClickHouse configuration layer for console logging +container_layer( + name = "clickhouse_config_layer", + files = ["clickhouse_logging_config.xml"], + mode = "0644", + directory = "/etc/clickhouse-server/config.d", +) + +container_image( + name = "clickhouse", + base = "@clickhouse_server_base_image//image", + layers = [":clickhouse_config_layer"], + visibility = ["//visibility:public"], +) diff --git a/src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse_logging_config.xml b/src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse_logging_config.xml new file mode 100644 index 00000000000..c2d570a3b02 --- /dev/null +++ b/src/stirling/source_connectors/socket_tracer/testing/container_images/clickhouse_logging_config.xml @@ -0,0 +1,7 @@ + + + true + + + + \ No newline at end of file diff --git a/src/stirling/source_connectors/stirling_error/stirling_error_connector.cc b/src/stirling/source_connectors/stirling_error/stirling_error_connector.cc index e186c53b60b..5489dac77f7 100644 --- a/src/stirling/source_connectors/stirling_error/stirling_error_connector.cc +++ b/src/stirling/source_connectors/stirling_error/stirling_error_connector.cc @@ -16,7 +16,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include +#include #include "src/common/base/base.h" #include "src/common/system/proc_parser.h" diff --git a/src/stirling/testing/overloads.h b/src/stirling/testing/overloads.h index f29062e857f..e38c540d000 100644 --- a/src/stirling/testing/overloads.h +++ b/src/stirling/testing/overloads.h @@ -19,7 +19,7 @@ #pragma once #include -#include +#include #include "src/stirling/utils/monitor.h" #include "src/stirling/utils/tcp_stats.h" diff --git a/src/table_store/schema/row_batch.cc b/src/table_store/schema/row_batch.cc index 4c48701cd5d..411c479b3cc 100644 --- a/src/table_store/schema/row_batch.cc +++ b/src/table_store/schema/row_batch.cc @@ -23,6 +23,7 @@ #include #include +#include #include "src/common/base/base.h" #include "src/shared/types/arrow_adapter.h" #include "src/shared/types/type_utils.h" @@ -38,13 +39,23 @@ std::shared_ptr RowBatch::ColumnAt(int64_t i) const { return colum Status RowBatch::AddColumn(const std::shared_ptr& col) { if (columns_.size() >= desc_.size()) { - return error::InvalidArgument("Schema only allows $0 columns", desc_.size()); + return error::InvalidArgument("Schema only allows $0 columns, got $1", desc_.size(), columns_.size()); } if (col->length() != num_rows_) { return error::InvalidArgument("Schema only allows $0 rows, got $1", num_rows_, col->length()); } - if (col->type_id() != types::ToArrowType(desc_.type(columns_.size()))) { - return error::InvalidArgument("Column[$0] was given incorrect type", columns_.size()); + auto expected_arrow_type = types::ToArrowType(desc_.type(columns_.size())); + if (col->type_id() != expected_arrow_type) { + auto pixie_type = desc_.type(columns_.size()); + return error::InvalidArgument( + "Column[$0] has incorrect Arrow type. " + "Got Arrow type_id=$1 (type=$2), expected Arrow type_id=$3 for Pixie DataType::$4 (enum value $5)", + columns_.size(), + static_cast(col->type_id()), + col->type()->ToString(), + static_cast(expected_arrow_type), + magic_enum::enum_name(pixie_type), + static_cast(pixie_type)); } columns_.emplace_back(col); diff --git a/src/ui/README.md b/src/ui/README.md index 088a1714bb6..09cdee3d741 100644 --- a/src/ui/README.md +++ b/src/ui/README.md @@ -2,7 +2,7 @@ ## Export environment variables for webpack ``` -export PL_GATEWAY_URL="https://$(dig +short prod.withpixie.ai @8.8.8.8)" +export PL_GATEWAY_URL="https://$(dig +short work.getcosmic.ai @8.8.8.8)" export PL_BUILD_TYPE=prod export SELFSIGN_CERT_FILE="$HOME/.prod.cert" export SELFSIGN_CERT_KEY="$HOME/.prod.key" @@ -16,13 +16,13 @@ mkcert -install mkcert \ -cert-file $SELFSIGN_CERT_FILE \ -key-file $SELFSIGN_CERT_KEY \ - prod.withpixie.ai "*.prod.withpixie.ai" localhost 127.0.0.1 ::1 + work.getcosmic.ai "*.work.getcosmic.ai" localhost 127.0.0.1 ::1 ``` ## Add the following domain to /etc/hosts, or /private/etc/hosts for Mac Replace site-name with your test site name. ``` -127.0.0.1 prod.withpixie.ai .prod.withpixie.ai id.prod.withpixie.ai +127.0.0.1 work.getcosmic.ai test.work.getcosmic.ai id.work.getcosmic.ai ``` ## Run the webpack devserver @@ -31,8 +31,30 @@ cd src/ui yarn install yarn dev ``` +This will expose the UI locally at 8080 ## Access the frontend on the browser -Navigate to https://prod.withpixie.ai:8080/ -Note the https and port. If you are not logged in, log in at work.withpixie.ai because -as of writing this, auth0 doesn't accept callbacks to prod.withpixie.ai:8080 +Navigate to https://work.getcosmic.ai:8080/ +Note the https and port. If you are not logged in, log in at work.getcosmic.ai because +as of writing this, auth0 doesn't accept callbacks to work.getcosmic.ai:8080 + +## Note if you are tunneling or get HSTS exceptions +(please do this at your own risk) +in Chrome, navigate to +chrome://net-internals/#hsts and delete the HSTS rules for work.getcosmic.ai + +This will then unblock the security feature for this domain. Please ensure to remove this once you are done. + + +## For a remote VM +### openSSH client +``` +ssh -i privkey user@IP -D 8080 +``` +### gcloud +``` +export instancename="instance-pixie-dev" +export project="gcp-project-uuid" +export zone="europe-west1-d" +gcloud compute ssh $instancename --zone $zone --project $project -- -NL 8080:localhost:8080 +``` diff --git a/src/vizier/funcs/md_udtfs/BUILD.bazel b/src/vizier/funcs/md_udtfs/BUILD.bazel index c5a966b5ca6..161ec9c3fe5 100644 --- a/src/vizier/funcs/md_udtfs/BUILD.bazel +++ b/src/vizier/funcs/md_udtfs/BUILD.bazel @@ -47,6 +47,7 @@ pl_cc_library( "//src/vizier/services/agent/shared/manager:cc_headers", "//src/vizier/services/metadata/metadatapb:service_pl_cc_proto", "@com_github_arun11299_cpp_jwt//:cpp_jwt", + "@com_github_clickhouse_clickhouse_cpp//:clickhouse_cpp", "@com_github_grpc_grpc//:grpc++", ], ) diff --git a/src/vizier/funcs/md_udtfs/md_udtfs.cc b/src/vizier/funcs/md_udtfs/md_udtfs.cc index 193c6d45dff..ec6f8926e80 100644 --- a/src/vizier/funcs/md_udtfs/md_udtfs.cc +++ b/src/vizier/funcs/md_udtfs/md_udtfs.cc @@ -58,6 +58,9 @@ void RegisterFuncsOrDie(const VizierFuncFactoryContext& ctx, carnot::udf::Regist registry ->RegisterFactoryOrDie>( "GetCronScriptHistory", ctx); + registry->RegisterFactoryOrDie>( + "CreateClickHouseSchemas", ctx); } } // namespace md diff --git a/src/vizier/funcs/md_udtfs/md_udtfs_impl.h b/src/vizier/funcs/md_udtfs/md_udtfs_impl.h index e48dd4ce790..1bc99b20b5c 100644 --- a/src/vizier/funcs/md_udtfs/md_udtfs_impl.h +++ b/src/vizier/funcs/md_udtfs/md_udtfs_impl.h @@ -28,6 +28,7 @@ #include #include +#include #include #include @@ -1073,6 +1074,259 @@ class GetCronScriptHistory final : public carnot::udf::UDTF add_context_authentication_func_; }; +namespace clickhouse_schema { + +/** + * Maps Pixie DataType to ClickHouse type string. + * Based on the mapping used in carnot_executable.cc for http_events table. + */ +inline std::string PixieTypeToClickHouseType(types::DataType pixie_type, + const std::string& column_name) { + switch (pixie_type) { + case types::DataType::INT64: + return "Int64"; + case types::DataType::FLOAT64: + return "Float64"; + case types::DataType::STRING: + return "String"; + case types::DataType::BOOLEAN: + return "UInt8"; + case types::DataType::TIME64NS: + // Use DateTime64(9) for time_ column (nanoseconds) + // Use DateTime64(3) for event_time column (milliseconds) + if (column_name == "time_") { + return "DateTime64(9)"; + } else if (column_name == "event_time") { + return "DateTime64(3)"; + } + // Default to DateTime64(9) for other time columns + return "DateTime64(9)"; + case types::DataType::UINT128: + // ClickHouse doesn't have native UINT128, use String representation (high:low format) + return "String"; + default: + return "String"; // Fallback to String for unsupported types + } +} + +} // namespace clickhouse_schema + +/** + * This UDTF creates ClickHouse schemas from Pixie DataTable schemas. + * It fetches table schemas from MDS and creates corresponding tables in ClickHouse. + */ +class CreateClickHouseSchemas final : public carnot::udf::UDTF { + public: + using MDSStub = vizier::services::metadata::MetadataService::Stub; + using SchemaResponse = vizier::services::metadata::SchemaResponse; + + CreateClickHouseSchemas() = delete; + CreateClickHouseSchemas(std::shared_ptr stub, + std::function add_context_authentication) + : idx_(0), stub_(stub), add_context_authentication_func_(add_context_authentication) {} + + static constexpr auto Executor() { return carnot::udfspb::UDTFSourceExecutor::UDTF_ONE_KELVIN; } + + static constexpr auto OutputRelation() { + return MakeArray(ColInfo("table_name", types::DataType::STRING, types::PatternType::GENERAL, + "The name of the table"), + ColInfo("status", types::DataType::STRING, types::PatternType::GENERAL, + "Status of the table creation (success/error)"), + ColInfo("message", types::DataType::STRING, types::PatternType::GENERAL, + "Additional information or error message")); + } + + static constexpr auto InitArgs() { + return MakeArray( + UDTFArg::Make("host", "ClickHouse server host", "'localhost'"), + UDTFArg::Make("port", "ClickHouse server port", 9000), + UDTFArg::Make("username", "ClickHouse username", "'default'"), + UDTFArg::Make("password", "ClickHouse password", "'test_password'"), + UDTFArg::Make("database", "ClickHouse database", "'default'"), + UDTFArg::Make("use_if_not_exists", "Whether to use IF NOT EXISTS in CREATE TABLE statements", true)); + } + + Status Init(FunctionContext*, types::StringValue host, types::Int64Value port, + types::StringValue username, types::StringValue password, + types::StringValue database, types::BoolValue use_if_not_exists) { + // Store ClickHouse connection parameters + host_ = std::string(host); + port_ = port.val; + username_ = std::string(username); + password_ = std::string(password); + database_ = std::string(database); + use_if_not_exists_ = use_if_not_exists.val; + + // Fetch schemas from MDS + px::vizier::services::metadata::SchemaRequest req; + px::vizier::services::metadata::SchemaResponse resp; + + grpc::ClientContext ctx; + add_context_authentication_func_(&ctx); + auto s = stub_->GetSchemas(&ctx, req, &resp); + if (!s.ok()) { + return error::Internal("Failed to make RPC call to metadata service: $0", + s.error_message()); + } + + // Connect to ClickHouse + clickhouse::ClientOptions client_options; + client_options.SetHost(host_); + client_options.SetPort(port_); + client_options.SetUser(username_); + client_options.SetPassword(password_); + client_options.SetDefaultDatabase(database_); + + try { + clickhouse_client_ = std::make_unique(client_options); + // Test connection + clickhouse_client_->Execute("SELECT 1"); + } catch (const std::exception& e) { + return error::Internal("Failed to connect to ClickHouse at $0:$1 - $2", + host_, port_, e.what()); + } + + for (const auto& [rel_table_name, rel] : resp.schema().relation_map()) { + TableResult result; + std::string table_name = rel_table_name; + result.table_name = table_name; + + // Check if table has a time_ column (required for partitioning) + bool has_time_column = false; + for (const auto& col : rel.columns()) { + if (col.column_name() == "time_" && + col.column_type() == types::DataType::TIME64NS) { + has_time_column = true; + break; + } + } + + if (!has_time_column) { + result.status = "skipped"; + result.message = "Table does not have a time_ TIME64NS column, skipping"; + results_.push_back(result); + continue; + } + + std::vector names = absl::StrSplit(table_name, '.'); + if (names.size() <= 0 || names.size() > 2) { + result.status = "error"; + result.message = "Invalid table name with multiple dots"; + results_.push_back(result); + continue; + } + table_name = names[0]; + + // Generate CREATE TABLE statement + std::string create_table_sql = GenerateCreateTableSQL(table_name, rel, use_if_not_exists_); + + // Execute the CREATE TABLE + try { + // Drop existing table if not using IF NOT EXISTS + if (!use_if_not_exists_) { + clickhouse_client_->Execute(absl::Substitute("DROP TABLE IF EXISTS $0", table_name)); + } + + // Create new table + clickhouse_client_->Execute(create_table_sql); + + result.status = "success"; + result.message = "Table created successfully"; + } catch (const std::exception& e) { + result.status = "error"; + result.message = absl::Substitute("Failed to create table: $0", e.what()); + } + + results_.push_back(result); + } + + return Status::OK(); + } + + bool NextRecord(FunctionContext*, RecordWriter* rw) { + if (idx_ >= static_cast(results_.size())) { + return false; + } + + const auto& result = results_[idx_]; + rw->Append(result.table_name); + rw->Append(result.status); + rw->Append(result.message); + + idx_++; + return idx_ < static_cast(results_.size()); + } + + private: + struct TableResult { + std::string table_name; + std::string status; + std::string message; + }; + + /** + * Generates a CREATE TABLE SQL statement for ClickHouse based on Pixie table schema. + * Follows the pattern from carnot_executable.cc: + * - Maps Pixie types to ClickHouse types + * - Adds hostname String column + * - Adds event_time DateTime64(3) column + * - Uses ENGINE = MergeTree() + * - Uses PARTITION BY toYYYYMM(event_time) + * - Uses ORDER BY (hostname, event_time) + */ + std::string GenerateCreateTableSQL(const std::string& table_name, + const px::table_store::schemapb::Relation& schema, + bool use_if_not_exists) { + std::vector column_defs; + + // Add columns from schema + for (const auto& col : schema.columns()) { + std::string column_name = col.column_name(); + if (column_name == "event_time" || column_name == "hostname") { + // event_time and hostname are added separately + continue; + } + std::string clickhouse_type = clickhouse_schema::PixieTypeToClickHouseType( + col.column_type(), column_name); + column_defs.push_back(absl::Substitute("$0 $1", column_name, clickhouse_type)); + } + + // Add hostname column + column_defs.push_back("hostname String"); + + // Add event_time column for partitioning (will be populated from time_ column) + column_defs.push_back("event_time DateTime64(3)"); + + // Build the CREATE TABLE statement + std::string columns_str = absl::StrJoin(column_defs, ",\n "); + + std::string if_not_exists_clause = use_if_not_exists ? "IF NOT EXISTS " : ""; + std::string create_sql = absl::Substitute(R"( + CREATE TABLE $0$1 ( + $2 + ) ENGINE = MergeTree() + PARTITION BY toYYYYMM(event_time) + ORDER BY (hostname, event_time) + )", if_not_exists_clause, table_name, columns_str); + + return create_sql; + } + + int idx_ = 0; + std::vector results_; + std::shared_ptr stub_; + std::function add_context_authentication_func_; + std::unique_ptr clickhouse_client_; + + // ClickHouse connection parameters + std::string host_; + int port_; + std::string username_; + std::string password_; + std::string database_; + bool use_if_not_exists_; +}; + } // namespace md } // namespace funcs } // namespace vizier diff --git a/src/vizier/services/adaptive_export/BUILD.bazel b/src/vizier/services/adaptive_export/BUILD.bazel new file mode 100644 index 00000000000..4dc4aa3a5f7 --- /dev/null +++ b/src/vizier/services/adaptive_export/BUILD.bazel @@ -0,0 +1,48 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") +load("//bazel:pl_build_system.bzl", "pl_go_image") + +go_library( + name = "adaptive_export_lib", + srcs = ["cmd/main.go"], + importpath = "px.dev/pixie/src/vizier/services/adaptive_export", + visibility = ["//visibility:private"], + deps = [ + "//src/api/go/pxapi", + "//src/vizier/services/adaptive_export/internal/config", + "//src/vizier/services/adaptive_export/internal/pixie", + "//src/vizier/services/adaptive_export/internal/pxl", + "//src/vizier/services/adaptive_export/internal/script", + "@com_github_sirupsen_logrus//:logrus", + ], +) + +go_binary( + name = "adaptive_export", + embed = [":adaptive_export_lib"], + visibility = ["//visibility:public"], +) + +pl_go_image( + name = "adaptive_export_image", + binary = ":adaptive_export", + visibility = [ + "//k8s:__subpackages__", + "//src/vizier:__subpackages__", + ], +) diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go new file mode 100644 index 00000000000..b283fe8083b --- /dev/null +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -0,0 +1,296 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package main + +import ( + "context" + "fmt" + "os" + "os/signal" + "syscall" + "time" + + log "github.com/sirupsen/logrus" + "px.dev/pixie/src/api/go/pxapi" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/config" + "px.dev/pixie/src/vizier/services/adaptive_export/internal/pixie" + "px.dev/pixie/src/vizier/services/adaptive_export/internal/pxl" + "px.dev/pixie/src/vizier/services/adaptive_export/internal/script" +) + +const ( + defaultRetries = 100 + defaultSleepTime = 15 * time.Second + schemaCreationInterval = 2 * time.Minute + setupTimeout = 30 * time.Second + scriptExecutionTimeout = 60 * time.Second +) + +const ( + // TODO(ddelnano): Clickhouse configuration should come from plugin config. + schemaCreationScript = ` +import px +px.display(px.CreateClickHouseSchemas( + host="hyperdx-hdx-oss-v2-clickhouse.click.svc.cluster.local", + port=9000, + username="otelcollector", + password="otelcollectorpass", + database="default" +)) +` + detectionScript = ` +import px + +df = px.DataFrame('kubescape_logs', clickhouse_dsn='otelcollector:otelcollectorpass@hyperdx-hdx-oss-v2-clickhouse.click.svc.cluster.local:9000/default', start_time='-%ds') +df.alert = df.message +df.namespace = px.pluck(df.RuntimeK8sDetails, "podNamespace") +df.podName = px.pluck(df.RuntimeK8sDetails, "podName") +df.time_ = px.int64_to_time(df.event_time * 1000000000) +df = df[['time_', 'alert', 'namespace', 'podName']] +px.display(df) +` +) + +func main() { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + log.Info("Starting the ClickHouse Adaptive Export service") + cfg, err := config.GetConfig() + if err != nil { + log.WithError(err).Fatal("failed to load configuration") + } + + clusterId := cfg.Pixie().ClusterID() + clusterName := cfg.Worker().ClusterName() + + // Setup Pixie Plugin API client + log.Infof("Setting up Pixie plugin API client for cluster-id %s", clusterId) + pluginClient, err := setupPixie(ctx, cfg.Pixie(), defaultRetries, defaultSleepTime) + if err != nil { + log.WithError(err).Fatal("setting up Pixie plugin client failed") + } + + // Setup Pixie pxapi client for executing PxL scripts + log.Info("Setting up Pixie pxapi client") + // Use parent context - client stores this and uses it for all subsequent operations + pxClient, err := pxapi.NewClient(ctx, pxapi.WithAPIKey(cfg.Pixie().APIKey()), pxapi.WithCloudAddr(cfg.Pixie().Host())) + if err != nil { + log.WithError(err).Fatal("failed to create pxapi client") + } + + // Start schema creation background task + go runSchemaCreationTask(ctx, pxClient, clusterId) + + // Start detection script that monitors for when to enable persistence + go runDetectionTask(ctx, pxClient, pluginClient, cfg, clusterId, clusterName) + + // Wait for signal to shutdown + sigCh := make(chan os.Signal, 1) + signal.Notify(sigCh, syscall.SIGINT, syscall.SIGTERM) + <-sigCh + + log.Info("Shutting down adaptive export service") + cancel() + time.Sleep(1 * time.Second) +} + +func runSchemaCreationTask(ctx context.Context, client *pxapi.Client, clusterID string) { + ticker := time.NewTicker(schemaCreationInterval) + defer ticker.Stop() + + // Run immediately on startup + log.Info("Running schema creation script") + execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) + if _, err := pxl.ExecuteScript(execCtx, client, clusterID, schemaCreationScript); err != nil { + log.WithError(err).Error("failed to execute schema creation script") + } else { + log.Info("Schema creation script completed successfully") + } + cancel() + + for { + select { + case <-ctx.Done(): + log.Info("Schema creation task shutting down") + return + case <-ticker.C: + log.Info("Running schema creation script") + execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) + if _, err := pxl.ExecuteScript(execCtx, client, clusterID, schemaCreationScript); err != nil { + log.WithError(err).Error("failed to execute schema creation script") + } else { + log.Info("Schema creation script completed successfully") + } + cancel() + } + } +} + +func runDetectionTask(ctx context.Context, pxClient *pxapi.Client, pluginClient *pixie.Client, cfg config.Config, clusterID string, clusterName string) { + detectionInterval := time.Duration(cfg.Worker().DetectionInterval()) * time.Second + detectionLookback := cfg.Worker().DetectionLookback() + + ticker := time.NewTicker(detectionInterval) + defer ticker.Stop() + + pluginEnabled := false + + for { + select { + case <-ctx.Done(): + log.Info("Detection task shutting down") + return + case <-ticker.C: + log.Info("Running detection script") + // Run detection script with lookback period + detectionPxl := fmt.Sprintf(detectionScript, detectionLookback) + execCtx, cancel := context.WithTimeout(ctx, scriptExecutionTimeout) + recordCount, err := pxl.ExecuteScript(execCtx, pxClient, clusterID, detectionPxl) + cancel() + + if err != nil { + log.WithError(err).Error("failed to execute detection script") + continue + } + + log.Debugf("Detection script returned %d records", recordCount) + + // If we have records and plugin is not enabled, enable it + if recordCount > 0 && !pluginEnabled { + log.Info("Detection script returned records - enabling forensic export") + pluginCtx, pluginCancel := context.WithTimeout(ctx, 2*time.Minute) + if err := enableClickHousePlugin(pluginCtx, pluginClient, cfg, clusterID, clusterName); err != nil { + log.WithError(err).Error("failed to enable forensic export") + } else { + pluginEnabled = true + log.Info("Forensic export enabled successfully") + } + pluginCancel() + } else if recordCount > 0 && pluginEnabled { + log.Info("Detection script returned records but forensic export already enabled, no action taken") + } + } + } +} + +func enableClickHousePlugin(ctx context.Context, client *pixie.Client, cfg config.Config, clusterID string, clusterName string) error { + log.Info("Checking the current ClickHouse plugin configuration") + plugin, err := client.GetClickHousePlugin() + if err != nil { + return fmt.Errorf("getting data retention plugins failed: %w", err) + } + + enablePlugin := true + if plugin.RetentionEnabled { + enablePlugin = false + config, err := client.GetClickHousePluginConfig() + if err != nil { + return fmt.Errorf("getting ClickHouse plugin config failed: %w", err) + } + if config.ExportUrl != cfg.ClickHouse().DSN() { + log.Info("ClickHouse plugin is configured with different DSN... Overwriting") + enablePlugin = true + } + } + + if enablePlugin { + log.Info("Enabling ClickHouse plugin") + err := client.EnableClickHousePlugin(&pixie.ClickHousePluginConfig{ + ExportUrl: cfg.ClickHouse().DSN(), + }, plugin.LatestVersion) + if err != nil { + return fmt.Errorf("failed to enable ClickHouse plugin: %w", err) + } + } + + log.Info("Setting up the data retention scripts") + + log.Info("Getting preset script from the Pixie plugin") + defsFromPixie, err := client.GetPresetScripts() + if err != nil { + return fmt.Errorf("failed to get preset scripts: %w", err) + } + + definitions := defsFromPixie + + log.Infof("Getting current scripts for cluster") + currentScripts, err := client.GetClusterScripts(clusterID, clusterName) + if err != nil { + return fmt.Errorf("failed to get data retention scripts: %w", err) + } + + actions := script.GetActions(definitions, currentScripts, script.ScriptConfig{ + ClusterName: clusterName, + ClusterId: clusterID, + CollectInterval: cfg.Worker().CollectInterval(), + }) + + var errs []error + + for _, s := range actions.ToDelete { + log.Infof("Deleting script %s", s.Name) + err := client.DeleteDataRetentionScript(s.ScriptId) + if err != nil { + errs = append(errs, err) + } + } + + for _, s := range actions.ToUpdate { + log.Infof("Updating script %s", s.Name) + err := client.UpdateDataRetentionScript(clusterID, s.ScriptId, s.Name, s.Description, s.FrequencyS, s.Script) + if err != nil { + errs = append(errs, err) + } + } + + for _, s := range actions.ToCreate { + log.Infof("Creating script %s", s.Name) + err := client.AddDataRetentionScript(clusterID, s.Name, s.Description, s.FrequencyS, s.Script) + if err != nil { + errs = append(errs, err) + } + } + + if len(errs) > 0 { + return fmt.Errorf("errors while setting up data retention scripts: %v", errs) + } + + log.Info("All done! The ClickHouse plugin is now configured.") + return nil +} + +func setupPixie(ctx context.Context, cfg config.Pixie, tries int, sleepTime time.Duration) (*pixie.Client, error) { + apiKey := cfg.APIKey() + host := cfg.Host() + log.Infof("setupPixie: API Key length=%d, Host=%s", len(apiKey), host) + + for tries > 0 { + // Use parent context - client stores this and uses it for all subsequent operations + client, err := pixie.NewClient(ctx, apiKey, host) + if err == nil { + return client, nil + } + tries -= 1 + log.WithError(err).Warning("error creating Pixie API client") + if tries > 0 { + time.Sleep(sleepTime) + } + } + return nil, fmt.Errorf("exceeded maximum number of retries") +} diff --git a/src/vizier/services/adaptive_export/internal/config/BUILD.bazel b/src/vizier/services/adaptive_export/internal/config/BUILD.bazel new file mode 100644 index 00000000000..4d19f27afab --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/config/BUILD.bazel @@ -0,0 +1,36 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "config", + srcs = [ + "config.go", + "definition.go", + ], + importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/config", + visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], + deps = [ + "//src/utils/shared/k8s", + "//src/vizier/services/adaptive_export/internal/script", + "@com_github_sirupsen_logrus//:logrus", + "@in_gopkg_yaml_v2//:yaml_v2", + "@io_k8s_apimachinery//pkg/apis/meta/v1:meta", + "@io_k8s_client_go//kubernetes", + "@io_k8s_client_go//rest", + ], +) diff --git a/src/vizier/services/adaptive_export/internal/config/config.go b/src/vizier/services/adaptive_export/internal/config/config.go new file mode 100644 index 00000000000..fc500359dfe --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/config/config.go @@ -0,0 +1,429 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "context" + "fmt" + "os" + "strconv" + "strings" + "sync" + + log "github.com/sirupsen/logrus" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + + "px.dev/pixie/src/utils/shared/k8s" +) + +const ( + envVerbose = "VERBOSE" + envClickHouseDSN = "CLICKHOUSE_DSN" + envPixieClusterID = "PIXIE_CLUSTER_ID" + envPixieEndpoint = "PIXIE_ENDPOINT" + envPixieAPIKey = "PIXIE_API_KEY" + envClusterName = "CLUSTER_NAME" + envCollectInterval = "COLLECT_INTERVAL_SEC" + envDetectionInterval = "DETECTION_INTERVAL_SEC" + envDetectionLookback = "DETECTION_LOOKBACK_SEC" + defPixieHostname = "work.withpixie.ai:443" + boolTrue = "true" + defCollectInterval = 30 + defDetectionInterval = 10 + defDetectionLookback = 15 +) + +var ( + integrationVersion = "0.0.0" + gitCommit = "" + buildDate = "" + once sync.Once + instance Config +) + +// findVizierNamespace looks for the namespace that the vizier is running in. +func findVizierNamespace(clientset *kubernetes.Clientset) (string, error) { + vzPods, err := clientset.CoreV1().Pods("").List(context.Background(), metav1.ListOptions{ + LabelSelector: "component=vizier", + }) + if err != nil { + return "", err + } + + if len(vzPods.Items) == 0 { + return "", fmt.Errorf("no vizier pods found") + } + + return vzPods.Items[0].Namespace, nil +} + +// getK8sConfig attempts to read configuration from Kubernetes secrets and configmaps. +// Returns (clusterID, apiKey, clusterName, host, error). +func getK8sConfig() (string, string, string, string, error) { + // Try in-cluster config first (when running in K8s) + config, err := rest.InClusterConfig() + if err != nil { + log.WithError(err).Debug("In-cluster config not available, trying kubeconfig...") + // Fall back to kubeconfig for local/adhoc testing + config = k8s.GetConfig() + if config == nil { + return "", "", "", "", fmt.Errorf("unable to get kubernetes config") + } + } else { + log.Debug("Using in-cluster Kubernetes config") + } + + clientset, err := kubernetes.NewForConfig(config) + if err != nil { + return "", "", "", "", fmt.Errorf("unable to create kubernetes clientset: %w", err) + } + + vzNs, err := findVizierNamespace(clientset) + if err != nil || vzNs == "" { + return "", "", "", "", fmt.Errorf("unable to find vizier namespace: %w", err) + } + + // Get cluster-id and cluster-name from pl-cluster-secrets + clusterSecrets := k8s.GetSecret(clientset, vzNs, "pl-cluster-secrets") + if clusterSecrets == nil { + return "", "", "", "", fmt.Errorf("unable to get pl-cluster-secrets") + } + + clusterID := "" + if cID, ok := clusterSecrets.Data["cluster-id"]; ok { + clusterID = string(cID) + } + + clusterName := "" + if cn, ok := clusterSecrets.Data["cluster-name"]; ok { + clusterName = string(cn) + } + + // Note: pl-deploy-secrets contains the deployment key (for registering vizier), + // not the user API key (for accessing cloud APIs). The user API key must be + // provided via PIXIE_API_KEY environment variable. + apiKey := "" + + // Get PL_CLOUD_ADDR from pl-cloud-config + cloudConfig, err := clientset.CoreV1().ConfigMaps(vzNs).Get(context.Background(), "pl-cloud-config", metav1.GetOptions{}) + host := "" + if err == nil { + if addr, ok := cloudConfig.Data["PL_CLOUD_ADDR"]; ok { + host = addr + } + } + + return clusterID, apiKey, clusterName, host, nil +} + +func GetConfig() (Config, error) { + var err error + once.Do(func() { + err = setUpConfig() + }) + return instance, err +} + +func setUpConfig() error { + log.SetLevel(log.InfoLevel) + + // Try to read configuration from environment variables first + clickhouseDSN := os.Getenv(envClickHouseDSN) + pixieClusterID := os.Getenv(envPixieClusterID) + pixieAPIKey := os.Getenv(envPixieAPIKey) + clusterName := os.Getenv(envClusterName) + pixieHost := getEnvWithDefault(envPixieEndpoint, defPixieHostname) + enableDebug := os.Getenv(envVerbose) + + if strings.EqualFold(enableDebug, boolTrue) { + log.SetLevel(log.DebugLevel) + } + + log.Debugf("Config from environment - ClickHouse DSN: %s", clickhouseDSN) + log.Debugf("Config from environment - Pixie Cluster ID: %s", pixieClusterID) + log.Debugf("Config from environment - Pixie API Key: %s", pixieAPIKey) + log.Debugf("Config from environment - Cluster Name: %s", clusterName) + log.Debugf("Config from environment - Pixie Host: %s", pixieHost) + + // If key values are not set via environment, try reading from Kubernetes + // Note: API key cannot be read from K8s (only deployment key is there), must be provided via env + if pixieClusterID == "" || clusterName == "" || pixieHost == defPixieHostname { + log.Info("Attempting to read Pixie configuration from Kubernetes resources...") + k8sClusterID, _, k8sClusterName, k8sHost, err := getK8sConfig() + if err != nil { + log.WithError(err).Warn("Failed to read configuration from Kubernetes, will use environment variables only") + } else { + // Use k8s values only if env vars are not set + if pixieClusterID == "" { + pixieClusterID = k8sClusterID + log.Debugf("Using cluster ID from Kubernetes: %s", pixieClusterID) + } + if clusterName == "" { + clusterName = k8sClusterName + log.Debugf("Using cluster name from Kubernetes: %s", clusterName) + } + if pixieHost == defPixieHostname && k8sHost != "" { + pixieHost = k8sHost + log.Debugf("Using host from Kubernetes: %s", pixieHost) + } + } + } + + log.Debugf("Final config - Pixie Cluster ID: %s", pixieClusterID) + log.Debugf("Final config - Pixie API Key: %s", pixieAPIKey) + log.Debugf("Final config - Cluster Name: %s", clusterName) + log.Debugf("Final config - Pixie Host: %s", pixieHost) + log.Debugf("Final config - ClickHouse DSN: %s", clickhouseDSN) + + collectInterval, err := getIntEnvWithDefault(envCollectInterval, defCollectInterval) + if err != nil { + return err + } + + detectionInterval, err := getIntEnvWithDefault(envDetectionInterval, defDetectionInterval) + if err != nil { + return err + } + + detectionLookback, err := getIntEnvWithDefault(envDetectionLookback, defDetectionLookback) + if err != nil { + return err + } + + instance = &config{ + settings: &settings{ + buildDate: buildDate, + commit: gitCommit, + version: integrationVersion, + }, + worker: &worker{ + clusterName: clusterName, + pixieClusterID: pixieClusterID, + collectInterval: collectInterval, + detectionInterval: detectionInterval, + detectionLookback: detectionLookback, + }, + clickhouse: &clickhouse{ + dsn: clickhouseDSN, + userAgent: "pixie-clickhouse/" + integrationVersion, + }, + pixie: &pixie{ + apiKey: pixieAPIKey, + clusterID: pixieClusterID, + host: pixieHost, + }, + } + return instance.validate() +} + +func getEnvWithDefault(key, defaultValue string) string { + value := os.Getenv(key) + if value == "" { + return defaultValue + } + return value +} + +func getIntEnvWithDefault(key string, defaultValue int64) (int64, error) { + value := os.Getenv(key) + if value == "" { + return defaultValue, nil + } + i, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return 0, fmt.Errorf("Environment variable %s is not an integer.", key) + } + return i, nil +} + +type Config interface { + Verbose() bool + Settings() Settings + ClickHouse() ClickHouse + Pixie() Pixie + Worker() Worker + validate() error +} + +type config struct { + verbose bool + worker Worker + clickhouse ClickHouse + pixie Pixie + settings Settings +} + +func (c *config) validate() error { + if err := c.Pixie().validate(); err != nil { + return fmt.Errorf("error validating pixie config: %w", err) + } + if err := c.Worker().validate(); err != nil { + return fmt.Errorf("error validating worker config: %w", err) + } + return c.ClickHouse().validate() +} + +func (c *config) Settings() Settings { + return c.settings +} + +func (c *config) Verbose() bool { + return c.verbose +} + +func (c *config) ClickHouse() ClickHouse { + return c.clickhouse +} + +func (c *config) Worker() Worker { + return c.worker +} + +func (c *config) Pixie() Pixie { + return c.pixie +} + +type Settings interface { + Version() string + Commit() string + BuildDate() string +} + +type settings struct { + buildDate string + commit string + version string +} + +func (s *settings) Version() string { + return s.version +} + +func (s *settings) Commit() string { + return s.commit +} + +func (s *settings) BuildDate() string { + return s.buildDate +} + +type ClickHouse interface { + DSN() string + UserAgent() string + validate() error +} + +type clickhouse struct { + dsn string + userAgent string +} + +func (c *clickhouse) validate() error { + if c.dsn == "" { + return fmt.Errorf("missing required env variable '%s'", envClickHouseDSN) + } + return nil +} + +func (c *clickhouse) DSN() string { + return c.dsn +} + +func (c *clickhouse) UserAgent() string { + return c.userAgent +} + +type Pixie interface { + APIKey() string + ClusterID() string + Host() string + validate() error +} + +type pixie struct { + apiKey string + clusterID string + host string +} + +func (p *pixie) validate() error { + if p.apiKey == "" { + return fmt.Errorf("missing required env variable '%s'", envPixieAPIKey) + } + if p.clusterID == "" { + return fmt.Errorf("missing required env variable '%s'", envPixieClusterID) + } + return nil +} + +func (p *pixie) APIKey() string { + return p.apiKey +} + +func (p *pixie) ClusterID() string { + return p.clusterID +} + +func (p *pixie) Host() string { + return p.host +} + +type Worker interface { + ClusterName() string + PixieClusterID() string + CollectInterval() int64 + DetectionInterval() int64 + DetectionLookback() int64 + validate() error +} + +type worker struct { + clusterName string + pixieClusterID string + collectInterval int64 + detectionInterval int64 + detectionLookback int64 +} + +func (a *worker) validate() error { + if a.clusterName == "" { + return fmt.Errorf("missing required env variable '%s'", envClusterName) + } + return nil +} + +func (a *worker) ClusterName() string { + return a.clusterName +} + +func (a *worker) PixieClusterID() string { + return a.pixieClusterID +} + +func (a *worker) CollectInterval() int64 { + return a.collectInterval +} + +func (a *worker) DetectionInterval() int64 { + return a.detectionInterval +} + +func (a *worker) DetectionLookback() int64 { + return a.detectionLookback +} diff --git a/src/vizier/services/adaptive_export/internal/config/definition.go b/src/vizier/services/adaptive_export/internal/config/definition.go new file mode 100644 index 00000000000..fd772022753 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/config/definition.go @@ -0,0 +1,66 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package config + +import ( + "io/ioutil" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v2" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/script" +) + +const scriptExtension = ".yaml" + +// ReadScriptDefinitions reads the script definition from the given directory path. +// Only .yaml files are read and subdirectories are not traversed. +func ReadScriptDefinitions(dir string) ([]*script.ScriptDefinition, error) { + if _, err := os.Stat(dir); os.IsNotExist(err) { + return nil, nil + } + files, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + var l []*script.ScriptDefinition + for _, file := range files { + if strings.HasSuffix(file.Name(), scriptExtension) { + description, err := readScriptDefinition(filepath.Join(dir, file.Name())) + if err != nil { + return nil, err + } + l = append(l, description) + } + } + return l, nil +} + +func readScriptDefinition(path string) (*script.ScriptDefinition, error) { + content, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + var definition script.ScriptDefinition + err = yaml.Unmarshal(content, &definition) + if err != nil { + return nil, err + } + return &definition, nil +} diff --git a/src/vizier/services/adaptive_export/internal/pixie/BUILD.bazel b/src/vizier/services/adaptive_export/internal/pixie/BUILD.bazel new file mode 100644 index 00000000000..29f239170a0 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/pixie/BUILD.bazel @@ -0,0 +1,34 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "pixie", + srcs = ["pixie.go"], + importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/pixie", + visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], + deps = [ + "//src/api/go/pxapi/utils", + "//src/api/proto/cloudpb:cloudapi_pl_go_proto", + "//src/api/proto/uuidpb:uuid_pl_go_proto", + "//src/vizier/services/adaptive_export/internal/script", + "@com_github_gogo_protobuf//types", + "@org_golang_google_grpc//:grpc", + "@org_golang_google_grpc//credentials", + "@org_golang_google_grpc//metadata", + ], +) diff --git a/src/vizier/services/adaptive_export/internal/pixie/pixie.go b/src/vizier/services/adaptive_export/internal/pixie/pixie.go new file mode 100644 index 00000000000..97e5bb8ae23 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/pixie/pixie.go @@ -0,0 +1,247 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package pixie + +import ( + "context" + "crypto/tls" + "fmt" + "strings" + + "github.com/gogo/protobuf/types" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/metadata" + "px.dev/pixie/src/api/go/pxapi/utils" + "px.dev/pixie/src/api/proto/cloudpb" + "px.dev/pixie/src/api/proto/uuidpb" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/script" +) + +const ( + clickhousePluginId = "clickhouse" + exportUrlConfig = "exportURL" +) + +type Client struct { + cloudAddr string + ctx context.Context + + grpcConn *grpc.ClientConn + pluginClient cloudpb.PluginServiceClient +} + +func NewClient(ctx context.Context, apiKey string, cloudAddr string) (*Client, error) { + if apiKey == "" { + fmt.Println("WARNING: API key is empty!") + } + + c := &Client{ + cloudAddr: cloudAddr, + ctx: metadata.AppendToOutgoingContext(ctx, "pixie-api-key", apiKey), + } + + if err := c.init(); err != nil { + return nil, err + } + + return c, nil +} + +func (c *Client) init() error { + isInternal := strings.ContainsAny(c.cloudAddr, "cluster.local") + + tlsConfig := &tls.Config{InsecureSkipVerify: isInternal} + creds := credentials.NewTLS(tlsConfig) + + conn, err := grpc.Dial(c.cloudAddr, grpc.WithTransportCredentials(creds)) + if err != nil { + return err + } + + c.grpcConn = conn + c.pluginClient = cloudpb.NewPluginServiceClient(conn) + return nil +} + +func (c *Client) GetClickHousePlugin() (*cloudpb.Plugin, error) { + req := &cloudpb.GetPluginsRequest{ + Kind: cloudpb.PK_RETENTION, + } + resp, err := c.pluginClient.GetPlugins(c.ctx, req) + if err != nil { + return nil, err + } + for _, plugin := range resp.Plugins { + if plugin.Id == clickhousePluginId { + return plugin, nil + } + } + return nil, fmt.Errorf("the %s plugin could not be found", clickhousePluginId) +} + +type ClickHousePluginConfig struct { + ExportUrl string +} + +func (c *Client) GetClickHousePluginConfig() (*ClickHousePluginConfig, error) { + req := &cloudpb.GetOrgRetentionPluginConfigRequest{ + PluginId: clickhousePluginId, + } + resp, err := c.pluginClient.GetOrgRetentionPluginConfig(c.ctx, req) + if err != nil { + return nil, err + } + exportUrl := resp.CustomExportUrl + if exportUrl == "" { + exportUrl, err = c.getDefaultClickHouseExportUrl() + if err != nil { + return nil, err + } + } + return &ClickHousePluginConfig{ + ExportUrl: exportUrl, + }, nil +} + +func (c *Client) getDefaultClickHouseExportUrl() (string, error) { + req := &cloudpb.GetRetentionPluginInfoRequest{ + PluginId: clickhousePluginId, + } + info, err := c.pluginClient.GetRetentionPluginInfo(c.ctx, req) + if err != nil { + return "", err + } + return info.DefaultExportURL, nil +} + +func (c *Client) EnableClickHousePlugin(config *ClickHousePluginConfig, version string) error { + req := &cloudpb.UpdateRetentionPluginConfigRequest{ + PluginId: clickhousePluginId, + Configs: map[string]string{ + exportUrlConfig: config.ExportUrl, + }, + Enabled: &types.BoolValue{Value: true}, + Version: &types.StringValue{Value: version}, + CustomExportUrl: &types.StringValue{Value: config.ExportUrl}, + InsecureTLS: &types.BoolValue{Value: false}, + DisablePresets: &types.BoolValue{Value: true}, + } + _, err := c.pluginClient.UpdateRetentionPluginConfig(c.ctx, req) + return err +} + +func (c *Client) GetPresetScripts() ([]*script.ScriptDefinition, error) { + resp, err := c.pluginClient.GetRetentionScripts(c.ctx, &cloudpb.GetRetentionScriptsRequest{}) + if err != nil { + return nil, err + } + var l []*script.ScriptDefinition + for _, s := range resp.Scripts { + if s.PluginId == clickhousePluginId && s.IsPreset { + sd, err := c.getScriptDefinition(s) + if err != nil { + return nil, err + } + l = append(l, sd) + } + } + return l, nil +} + +func (c *Client) GetClusterScripts(clusterId, clusterName string) ([]*script.Script, error) { + resp, err := c.pluginClient.GetRetentionScripts(c.ctx, &cloudpb.GetRetentionScriptsRequest{}) + if err != nil { + return nil, err + } + var l []*script.Script + for _, s := range resp.Scripts { + if s.PluginId == clickhousePluginId { + sd, err := c.getScriptDefinition(s) + if err != nil { + return nil, err + } + l = append(l, &script.Script{ + ScriptDefinition: *sd, + ScriptId: utils.ProtoToUUIDStr(s.ScriptID), + ClusterIds: getClusterIdsAsString(s.ClusterIDs), + }) + } + } + return l, nil +} + +func getClusterIdsAsString(clusterIDs []*uuidpb.UUID) string { + scriptClusterId := "" + for i, id := range clusterIDs { + if i > 0 { + scriptClusterId = scriptClusterId + "," + } + scriptClusterId = scriptClusterId + utils.ProtoToUUIDStr(id) + } + return scriptClusterId +} + +func (c *Client) getScriptDefinition(s *cloudpb.RetentionScript) (*script.ScriptDefinition, error) { + resp, err := c.pluginClient.GetRetentionScript(c.ctx, &cloudpb.GetRetentionScriptRequest{ID: s.ScriptID}) + if err != nil { + return nil, err + } + return &script.ScriptDefinition{ + Name: s.ScriptName, + Description: s.Description, + FrequencyS: s.FrequencyS, + Script: resp.Contents, + IsPreset: s.IsPreset, + }, nil +} + +func (c *Client) AddDataRetentionScript(clusterId string, scriptName string, description string, frequencyS int64, contents string) error { + req := &cloudpb.CreateRetentionScriptRequest{ + ScriptName: scriptName, + Description: description, + FrequencyS: frequencyS, + Contents: contents, + ClusterIDs: []*uuidpb.UUID{utils.ProtoFromUUIDStrOrNil(clusterId)}, + PluginId: clickhousePluginId, + } + _, err := c.pluginClient.CreateRetentionScript(c.ctx, req) + return err +} + +func (c *Client) UpdateDataRetentionScript(clusterId string, scriptId string, scriptName string, description string, frequencyS int64, contents string) error { + req := &cloudpb.UpdateRetentionScriptRequest{ + ID: utils.ProtoFromUUIDStrOrNil(scriptId), + ScriptName: &types.StringValue{Value: scriptName}, + Description: &types.StringValue{Value: description}, + Enabled: &types.BoolValue{Value: true}, + FrequencyS: &types.Int64Value{Value: frequencyS}, + Contents: &types.StringValue{Value: contents}, + ClusterIDs: []*uuidpb.UUID{utils.ProtoFromUUIDStrOrNil(clusterId)}, + } + _, err := c.pluginClient.UpdateRetentionScript(c.ctx, req) + return err +} + +func (c *Client) DeleteDataRetentionScript(scriptId string) error { + req := &cloudpb.DeleteRetentionScriptRequest{ + ID: utils.ProtoFromUUIDStrOrNil(scriptId), + } + _, err := c.pluginClient.DeleteRetentionScript(c.ctx, req) + return err +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/BUILD.bazel b/src/vizier/services/adaptive_export/internal/pxl/BUILD.bazel new file mode 100644 index 00000000000..80afa3f2875 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/pxl/BUILD.bazel @@ -0,0 +1,30 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "pxl", + srcs = ["pxl.go"], + importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/pxl", + visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], + deps = [ + "//src/api/go/pxapi", + "//src/api/go/pxapi/errdefs", + "//src/api/go/pxapi/types", + "@com_github_sirupsen_logrus//:logrus", + ], +) diff --git a/src/vizier/services/adaptive_export/internal/pxl/pxl.go b/src/vizier/services/adaptive_export/internal/pxl/pxl.go new file mode 100644 index 00000000000..e4e27a40b6b --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/pxl/pxl.go @@ -0,0 +1,80 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package pxl + +import ( + "context" + "fmt" + + log "github.com/sirupsen/logrus" + "px.dev/pixie/src/api/go/pxapi" + "px.dev/pixie/src/api/go/pxapi/errdefs" + "px.dev/pixie/src/api/go/pxapi/types" +) + +// recordCounter counts the number of records received +type recordCounter struct { + count int +} + +func (r *recordCounter) HandleInit(ctx context.Context, metadata types.TableMetadata) error { + return nil +} + +func (r *recordCounter) HandleRecord(ctx context.Context, record *types.Record) error { + r.count++ + return nil +} + +func (r *recordCounter) HandleDone(ctx context.Context) error { + return nil +} + +type recordCounterMux struct { + counter *recordCounter +} + +func (m *recordCounterMux) AcceptTable(ctx context.Context, metadata types.TableMetadata) (pxapi.TableRecordHandler, error) { + return m.counter, nil +} + +// ExecuteScript executes a PxL script and returns the number of records returned +func ExecuteScript(ctx context.Context, client *pxapi.Client, clusterID string, pxl string) (int, error) { + vz, err := client.NewVizierClient(ctx, clusterID) + if err != nil { + return 0, fmt.Errorf("failed to create vizier client: %w", err) + } + + counter := &recordCounter{} + tm := &recordCounterMux{counter: counter} + + resultSet, err := vz.ExecuteScript(ctx, pxl, tm) + if err != nil { + return 0, fmt.Errorf("failed to execute script: %w", err) + } + defer resultSet.Close() + + if err := resultSet.Stream(); err != nil { + if errdefs.IsCompilationError(err) { + return 0, fmt.Errorf("PxL compilation error: %w", err) + } + return 0, fmt.Errorf("error streaming results: %w", err) + } + + log.Debugf("Script execution time: %v, bytes received: %v", resultSet.Stats().ExecutionTime, resultSet.Stats().TotalBytes) + return counter.count, nil +} diff --git a/src/vizier/services/adaptive_export/internal/script/BUILD.bazel b/src/vizier/services/adaptive_export/internal/script/BUILD.bazel new file mode 100644 index 00000000000..28d764063a4 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/script/BUILD.bazel @@ -0,0 +1,24 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +load("@io_bazel_rules_go//go:def.bzl", "go_library") + +go_library( + name = "script", + srcs = ["script.go"], + importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/script", + visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], +) diff --git a/src/vizier/services/adaptive_export/internal/script/script.go b/src/vizier/services/adaptive_export/internal/script/script.go new file mode 100644 index 00000000000..23005ec8851 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/script/script.go @@ -0,0 +1,114 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package script + +import ( + "fmt" + "strings" +) + +const ( + scriptPrefix = "ch-" +) + +type ScriptConfig struct { + ClusterName string + ClusterId string + CollectInterval int64 +} + +type Script struct { + ScriptDefinition + ScriptId string + ClusterIds string +} + +type ScriptDefinition struct { + Name string `yaml:"name"` + Description string `yaml:"description"` + FrequencyS int64 `yaml:"frequencyS"` + Script string `yaml:"script"` + IsPreset bool `yaml:"-"` +} + +type ScriptActions struct { + ToDelete []*Script + ToUpdate []*Script + ToCreate []*Script +} + +func IsClickHouseScript(scriptName string) bool { + return strings.HasPrefix(scriptName, scriptPrefix) +} + +func IsScriptForCluster(scriptName, clusterName string) bool { + return IsClickHouseScript(scriptName) && strings.HasSuffix(scriptName, "-"+clusterName) +} + +func GetActions(scriptDefinitions []*ScriptDefinition, currentScripts []*Script, config ScriptConfig) ScriptActions { + definitions := make(map[string]ScriptDefinition) + for _, definition := range scriptDefinitions { + scriptName := getScriptName(definition.Name, config.ClusterName) + frequencyS := getInterval(definition, config) + if frequencyS > 0 { + definitions[scriptName] = ScriptDefinition{ + Name: scriptName, + Description: definition.Description, + FrequencyS: frequencyS, + Script: templateScript(definition, config), + } + } + } + actions := ScriptActions{} + for _, current := range currentScripts { + if definition, present := definitions[current.Name]; present { + if definition.Script != current.Script || definition.FrequencyS != current.FrequencyS || config.ClusterId != current.ClusterIds { + actions.ToUpdate = append(actions.ToUpdate, &Script{ + ScriptDefinition: definition, + ScriptId: current.ScriptId, + ClusterIds: config.ClusterId, + }) + } + delete(definitions, current.Name) + } else if IsClickHouseScript(current.Name) { + actions.ToDelete = append(actions.ToDelete, current) + } + } + for _, definition := range definitions { + actions.ToCreate = append(actions.ToCreate, &Script{ + ScriptDefinition: definition, + ClusterIds: config.ClusterId, + }) + } + return actions +} + +func getScriptName(scriptName string, clusterName string) string { + return fmt.Sprintf("%s%s-%s", scriptPrefix, scriptName, clusterName) +} + +func getInterval(definition *ScriptDefinition, config ScriptConfig) int64 { + if definition.FrequencyS == 0 { + return config.CollectInterval + } + return definition.FrequencyS +} + +func templateScript(definition *ScriptDefinition, config ScriptConfig) string { + // Return script as-is without any processing + return definition.Script +} diff --git a/src/vizier/services/agent/pem/tracepoint_manager_test.cc b/src/vizier/services/agent/pem/tracepoint_manager_test.cc index 9cd85aed2af..fd54f7badb2 100644 --- a/src/vizier/services/agent/pem/tracepoint_manager_test.cc +++ b/src/vizier/services/agent/pem/tracepoint_manager_test.cc @@ -115,8 +115,8 @@ TEST_F(TracepointManagerTest, CreateTracepoint) { tracepoint->set_name("test_tracepoint"); EXPECT_CALL(stirling_, - RegisterTracepoint(tracepoint_id, ::testing::Pointee(testing::proto::EqualsProto( - tracepoint->DebugString())))); + RegisterTracepoint(tracepoint_id, + ::testing::Pointee(testing::proto::EqualsProto(*tracepoint)))); EXPECT_OK(tracepoint_manager_->HandleMessage(std::move(msg))); EXPECT_CALL(stirling_, GetTracepointInfo(tracepoint_id)) @@ -152,8 +152,8 @@ TEST_F(TracepointManagerTest, CreateTracepointFailed) { tracepoint->set_name("test_tracepoint"); EXPECT_CALL(stirling_, - RegisterTracepoint(tracepoint_id, ::testing::Pointee(testing::proto::EqualsProto( - tracepoint->DebugString())))); + RegisterTracepoint(tracepoint_id, + ::testing::Pointee(testing::proto::EqualsProto(*tracepoint)))); EXPECT_OK(tracepoint_manager_->HandleMessage(std::move(msg))); EXPECT_CALL(stirling_, GetTracepointInfo(tracepoint_id)) @@ -185,8 +185,8 @@ TEST_F(TracepointManagerTest, CreateTracepointPreconditionFailed) { tracepoint->set_name("test_tracepoint"); EXPECT_CALL(stirling_, - RegisterTracepoint(tracepoint_id, ::testing::Pointee(testing::proto::EqualsProto( - tracepoint->DebugString())))); + RegisterTracepoint(tracepoint_id, + ::testing::Pointee(testing::proto::EqualsProto(*tracepoint)))); EXPECT_OK(tracepoint_manager_->HandleMessage(std::move(msg))); EXPECT_CALL(stirling_, GetTracepointInfo(tracepoint_id)) diff --git a/src/vizier/services/agent/shared/manager/chan_cache.h b/src/vizier/services/agent/shared/manager/chan_cache.h index 00106a2e2f4..6520a1c03a9 100644 --- a/src/vizier/services/agent/shared/manager/chan_cache.h +++ b/src/vizier/services/agent/shared/manager/chan_cache.h @@ -83,7 +83,7 @@ class ChanCache { }; // The cache of channels (grpc conns) made to other agents. - absl::flat_hash_map chan_cache_ GUARDED_BY(chan_cache_lock_); + absl::flat_hash_map chan_cache_ ABSL_GUARDED_BY(chan_cache_lock_); absl::base_internal::SpinLock chan_cache_lock_; // Connections that are alive for shorter than warm_up_period_ won't be cleared. std::chrono::nanoseconds warm_up_period_; diff --git a/src/vizier/services/agent/shared/manager/relation_info_manager.h b/src/vizier/services/agent/shared/manager/relation_info_manager.h index 10c05039328..f4cf1080e3d 100644 --- a/src/vizier/services/agent/shared/manager/relation_info_manager.h +++ b/src/vizier/services/agent/shared/manager/relation_info_manager.h @@ -65,7 +65,8 @@ class RelationInfoManager { private: mutable std::atomic has_updates_ = false; mutable absl::base_internal::SpinLock relation_info_map_lock_; - absl::btree_map relation_info_map_ GUARDED_BY(relation_info_map_lock_); + absl::btree_map relation_info_map_ + ABSL_GUARDED_BY(relation_info_map_lock_); }; } // namespace agent diff --git a/src/vizier/services/query_broker/controllers/mutation_executor.go b/src/vizier/services/query_broker/controllers/mutation_executor.go index f14ad3028de..813769362da 100644 --- a/src/vizier/services/query_broker/controllers/mutation_executor.go +++ b/src/vizier/services/query_broker/controllers/mutation_executor.go @@ -87,9 +87,27 @@ func (m *MutationExecutorImpl) Execute(ctx context.Context, req *vizierpb.Execut if err != nil { return nil, err } + var otelConfig *distributedpb.OTelEndpointConfig + if convertedReq.Configs != nil && convertedReq.Configs.OTelEndpointConfig != nil { + otelConfig = &distributedpb.OTelEndpointConfig{ + URL: convertedReq.Configs.OTelEndpointConfig.URL, + Headers: convertedReq.Configs.OTelEndpointConfig.Headers, + Insecure: convertedReq.Configs.OTelEndpointConfig.Insecure, + Timeout: convertedReq.Configs.OTelEndpointConfig.Timeout, + } + } + var pluginConfig *distributedpb.PluginConfig + if req.Configs != nil && req.Configs.PluginConfig != nil { + pluginConfig = &distributedpb.PluginConfig{ + StartTimeNs: req.Configs.PluginConfig.StartTimeNs, + EndTimeNs: req.Configs.PluginConfig.EndTimeNs, + } + } convertedReq.LogicalPlannerState = &distributedpb.LogicalPlannerState{ - DistributedState: m.distributedState, - PlanOptions: planOpts, + DistributedState: m.distributedState, + PlanOptions: planOpts, + OTelEndpointConfig: otelConfig, + PluginConfig: pluginConfig, } mutations, err := m.planner.CompileMutations(convertedReq) @@ -220,11 +238,11 @@ func (m *MutationExecutorImpl) Execute(ctx context.Context, req *vizierpb.Execut // MutationInfo returns the summarized mutation information. func (m *MutationExecutorImpl) MutationInfo(ctx context.Context) (*vizierpb.MutationInfo, error) { - req := &metadatapb.GetTracepointInfoRequest{ + tpReq := &metadatapb.GetTracepointInfoRequest{ IDs: make([]*uuidpb.UUID, 0), } for _, tp := range m.activeTracepoints { - req.IDs = append(req.IDs, utils.ProtoFromUUID(tp.ID)) + tpReq.IDs = append(tpReq.IDs, utils.ProtoFromUUID(tp.ID)) } aCtx, err := authcontext.FromContext(ctx) if err != nil { @@ -232,28 +250,28 @@ func (m *MutationExecutorImpl) MutationInfo(ctx context.Context) (*vizierpb.Muta } ctx = metadata.AppendToOutgoingContext(ctx, "authorization", fmt.Sprintf("bearer %s", aCtx.AuthToken)) - resp, err := m.mdtp.GetTracepointInfo(ctx, req) + tpResp, err := m.mdtp.GetTracepointInfo(ctx, tpReq) if err != nil { return nil, err } mutationInfo := &vizierpb.MutationInfo{ Status: &vizierpb.Status{Code: 0}, - States: make([]*vizierpb.MutationInfo_MutationState, len(resp.Tracepoints)), + States: make([]*vizierpb.MutationInfo_MutationState, len(tpResp.Tracepoints)), } - ready := true - for idx, tp := range resp.Tracepoints { + tpReady := true + for idx, tp := range tpResp.Tracepoints { mutationInfo.States[idx] = &vizierpb.MutationInfo_MutationState{ ID: utils.UUIDFromProtoOrNil(tp.ID).String(), State: convertLifeCycleStateToVizierLifeCycleState(tp.State), Name: tp.Name, } if tp.State != statuspb.RUNNING_STATE { - ready = false + tpReady = false } } - if !ready { + if !tpReady { mutationInfo.Status = &vizierpb.Status{ Code: int32(codes.Unavailable), Message: "probe installation in progress", diff --git a/src/vizier/services/query_broker/script_runner/script_runner.go b/src/vizier/services/query_broker/script_runner/script_runner.go index 48f78b9427b..fbe8afae032 100644 --- a/src/vizier/services/query_broker/script_runner/script_runner.go +++ b/src/vizier/services/query_broker/script_runner/script_runner.go @@ -22,6 +22,7 @@ import ( "context" "fmt" "io" + "strings" "sync" "time" @@ -262,13 +263,17 @@ func (r *runner) runScript(scriptPeriod time.Duration) { } } - // We set the time 1 second in the past to cover colletor latency and request latencies + // We set the time 1 second in the past to cover collector latency and request latencies // which can cause data overlaps or cause data to be missed. startTime := r.lastRun.Add(-time.Second) endTime := startTime.Add(scriptPeriod) r.lastRun = time.Now() + // TODO(ddelnano): This might not be the correct approach for handling mutations. + // This is done until the pxlog source can work with an indefinite ttl. + hasMutation := strings.Contains(r.cronScript.Script, "pxlog") execScriptClient, err := r.vzClient.ExecuteScript(ctx, &vizierpb.ExecuteScriptRequest{ QueryStr: r.cronScript.Script, + Mutation: hasMutation, Configs: &vizierpb.Configs{ OTelEndpointConfig: otelEndpoint, PluginConfig: &vizierpb.Configs_PluginConfig{ diff --git a/tools/chef/cookbooks/px_dev/recipes/linux.rb b/tools/chef/cookbooks/px_dev/recipes/linux.rb index c805c98fb20..aea415ac1e9 100644 --- a/tools/chef/cookbooks/px_dev/recipes/linux.rb +++ b/tools/chef/cookbooks/px_dev/recipes/linux.rb @@ -56,6 +56,16 @@ 'qemu-system-x86', 'qemu-user-static', 'qemu-utils', + + # Minikube dependencies for kvm + 'libnss3-tools', + 'libvirt-daemon-system', + 'libvirt-clients', + 'qemu-kvm', + 'virt-manager', + + # Pixie dependencies + 'mkcert', ] apt_package apt_pkg_list do