From 7356e28abb60278d82236786ceeb1fff8ef702e0 Mon Sep 17 00:00:00 2001 From: ikholopov Date: Thu, 27 Nov 2025 15:07:31 +0100 Subject: [PATCH] JiT context RPC service and protos Protos for JiT DB adapter and JiT compilation messages. --- packages/@dataform/core/BUILD | 14 +++++ protos/BUILD | 8 ++- protos/db_adapter.proto | 78 ++++++++++++++++++++++++ protos/execution.proto | 93 +++-------------------------- protos/jit.proto | 108 ++++++++++++++++++++++++++++++++++ 5 files changed, 215 insertions(+), 86 deletions(-) create mode 100644 protos/db_adapter.proto create mode 100644 protos/jit.proto diff --git a/packages/@dataform/core/BUILD b/packages/@dataform/core/BUILD index bad849792..15bfc8b56 100644 --- a/packages/@dataform/core/BUILD +++ b/packages/@dataform/core/BUILD @@ -67,6 +67,18 @@ add_license_header_to_file( to_file = "configs.proto", ) +add_license_header_to_file( + name = "db_adapter_proto_with_license", + from_file = "//protos:db_adapter.proto", + to_file = "db_adapter.proto", +) + +add_license_header_to_file( + name = "jit_proto_with_license", + from_file = "//protos:jit.proto", + to_file = "jit.proto", +) + add_license_header_to_file( name = "bundle_with_license", from_file = "bundle", @@ -95,6 +107,8 @@ pkg_npm_tar( ":bundle_with_license", ":configs_proto_with_license", ":core_proto_with_license", + ":db_adapter_proto_with_license", + ":jit_proto_with_license", ":package.json", "//:version.bzl", ], diff --git a/protos/BUILD b/protos/BUILD index 11c4e9331..2bd44d3ad 100644 --- a/protos/BUILD +++ b/protos/BUILD @@ -1,5 +1,4 @@ load("@rules_proto//proto:defs.bzl", "proto_library") -load("//testing:build_test.bzl", "build_test") load("//tools:ts_proto_library.bzl", "ts_proto_library") package(default_visibility = ["//visibility:public"]) @@ -9,11 +8,16 @@ proto_library( srcs = [ "configs.proto", "core.proto", + "db_adapter.proto", "evaluation.proto", "execution.proto", + "jit.proto", "profiles.proto", ], - deps = ["@com_google_protobuf//:struct_proto"], + deps = [ + "@com_google_protobuf//:empty_proto", + "@com_google_protobuf//:struct_proto", + ], ) ts_proto_library( diff --git a/protos/db_adapter.proto b/protos/db_adapter.proto new file mode 100644 index 000000000..14e4988ee --- /dev/null +++ b/protos/db_adapter.proto @@ -0,0 +1,78 @@ +syntax = "proto3"; + +package dataform; + +import "protos/core.proto"; + +// Common structures for Evaluation, execution and compilation. + +message Field { + enum Primitive { + UNKNOWN = 0; + INTEGER = 1; + FLOAT = 2; + NUMERIC = 5; + BOOLEAN = 3; + STRING = 4; + DATE = 6; + DATETIME = 7; + TIMESTAMP = 8; + TIME = 9; + BYTES = 10; + ANY = 11; + GEOGRAPHY = 12; + } + + enum Flag { + UNKNOWN_FLAG = 0; + REPEATED = 1; + } + + string name = 1; + + repeated Flag flags = 6; + + oneof type { + Primitive primitive = 7; + Fields struct = 3; + } + + string description = 5; + + reserved 2, 4; +} + +message Fields { + repeated Field fields = 1; +} + +message TableMetadata { + Target target = 1; + enum Type { + UNKNOWN = 0; + TABLE = 1; + VIEW = 2; + } + Type type = 6; + + repeated Field fields = 3; + string description = 5; + map labels = 7; + int64 last_updated_millis = 4; + + message BigQuery { + bool has_streaming_buffer = 1; + } + BigQuery bigquery = 8; + + reserved 2; +} + +message ExecutionMetadata { + message BigqueryMetadata { + string job_id = 1; + int64 total_bytes_processed = 2; + int64 total_bytes_billed = 3; + } + BigqueryMetadata bigquery = 1; +} diff --git a/protos/execution.proto b/protos/execution.proto index 17f883719..4a362d51d 100644 --- a/protos/execution.proto +++ b/protos/execution.proto @@ -1,12 +1,12 @@ -syntax="proto3"; +syntax = "proto3"; package dataform; import "protos/core.proto"; +import "protos/db_adapter.proto"; option go_package = "github.com/dataform-co/dataform/protos/dataform"; - message RunConfig { repeated string actions = 1; repeated string tags = 5; @@ -14,7 +14,7 @@ message RunConfig { bool include_dependents = 11; bool full_refresh = 2; int32 timeout_millis = 7; - + // For internal use only, will be removed at a later date. bool disable_set_metadata = 9; @@ -36,7 +36,6 @@ message ExecutionAction { string type = 4; string table_type = 6; - repeated Target dependency_targets = 11; ActionHermeticity hermeticity = 10; @@ -82,7 +81,6 @@ message RunResult { } message ActionResult { - Target target = 5; enum ExecutionStatus { @@ -103,24 +101,14 @@ message ActionResult { reserved 1, 6, 7; } - -message ExecutionMetadata { - message BigqueryMetadata { - string job_id = 1; - int64 total_bytes_processed = 2; - int64 total_bytes_billed = 3; - } - BigqueryMetadata bigquery = 1; -} - message TaskResult { enum ExecutionStatus { - UNKNOWN = 0; - RUNNING = 1; - SUCCESSFUL = 2; - FAILED = 3; - SKIPPED = 4; - CANCELLED = 5; + UNKNOWN = 0; + RUNNING = 1; + SUCCESSFUL = 2; + FAILED = 3; + SKIPPED = 4; + CANCELLED = 5; } ExecutionStatus status = 1; string error_message = 2; @@ -133,66 +121,3 @@ message TestResult { bool successful = 2; repeated string messages = 3; } - -message Field { - - enum Primitive { - UNKNOWN = 0; - INTEGER = 1; - FLOAT = 2; - NUMERIC = 5; - BOOLEAN = 3; - STRING = 4; - DATE = 6; - DATETIME = 7; - TIMESTAMP = 8; - TIME = 9; - BYTES = 10; - ANY = 11; - GEOGRAPHY = 12; - } - - enum Flag { - UNKNOWN_FLAG = 0; - REPEATED = 1; - } - - string name = 1; - - repeated Flag flags = 6; - - oneof type { - Primitive primitive = 7; - Fields struct = 3; - } - - string description = 5; - - reserved 2, 4; -} - -message Fields { - repeated Field fields = 1; -} - -message TableMetadata { - Target target = 1; - enum Type { - UNKNOWN = 0; - TABLE = 1; - VIEW = 2; - } - Type type = 6; - - repeated Field fields = 3; - string description = 5; - map labels = 7; - int64 last_updated_millis = 4; - - message BigQuery { - bool has_streaming_buffer = 1; - } - BigQuery bigquery = 8; - - reserved 2; -} diff --git a/protos/jit.proto b/protos/jit.proto new file mode 100644 index 000000000..4a15a3bf7 --- /dev/null +++ b/protos/jit.proto @@ -0,0 +1,108 @@ +syntax = "proto3"; + +package dataform; + +import "google/protobuf/empty.proto"; +import "google/protobuf/struct.proto"; +import "protos/core.proto"; +import "protos/db_adapter.proto"; + +// Database adapter, available in JiT compilation context. +service DbAdapter { + // Executes a query. + rpc Execute(ExecuteRequest) returns (ExecuteResponse) {} + + // Lists table in schema. + rpc ListTables(ListTablesRequest) returns (ListTablesResponse) {} + + // Gets table for target. + rpc GetTable(GetTableRequest) returns (TableMetadata) {} + + // Drops table of specified target. + rpc DeleteTable(DeleteTableRequest) returns (google.protobuf.Empty) {} +} + +// Request to execute synchronous SQL query. +message ExecuteRequest { + // SQL statement query. + string statement = 1; + // Max rows to return in the job. + int64 row_limit = 2; + // Max bytes to process. + int64 byte_limit = 3; + // Whether or not fetch result rows. + bool fetch_results = 4; + + BigQueryExecuteOptions big_query_options = 5; +} + +message BigQueryExecuteOptions { + // Priority - interactive if true or batch if false (default). + bool interactive = 1; + // See https://docs.cloud.google.com/bigquery/docs/reference/legacy-sql. + bool use_legacy_sql = 2; + // Labels to add to the job. + map labels = 3; + // Location to execute the job at. + string location = 4; + // Job prefix to add. + string job_prefix = 5; + // Is dry run job. + bool dry_run = 6; +} + +// Synchronous execution response result. +message ExecuteResponse { + // Job metadata. + ExecutionMetadata execution_metadata = 1; + // Rows. For BigQuery, see + // https://docs.cloud.google.com/bigquery/docs/reference/rest/v2/jobs/getQueryResults. + repeated google.protobuf.Struct rows = 2; + repeated string errors = 3; +} + +// Request to list tables in schema. +message ListTablesRequest { + string database = 1; + string schema = 2; +} + +// Tables metadata in schema. +message ListTablesResponse { + repeated TableMetadata tables = 1; +} + +// Request to get table metadata for target. +message GetTableRequest { + Target target = 1; +} + +// Request to drop the table for target. +message DeleteTableRequest { + Target target = 1; +} + +// JiT compilation result for table actions (including views). +// Fields match the Table message. +message JitTableResult { + // SQL Select query of the table. + string query = 1; + // Optional pre-operation statements. + repeated string pre_ops = 3; + // Optional post-operation statements. + repeated string post_ops = 4; +} + +// JiT compilation result for incremental table actions. +message JitIncrementalTableResult { + // Fields match the Table message. + JitTableResult regular = 1; + // Fields match incremental_ fields in the Table message. + JitTableResult incremental = 2; +} + +// JiT compilation result for operation actions. +message JitOperationResult { + // Sequence of SQL operations. + repeated string queries = 1; +}