From 3d50463844c6c49007db6cc031a3daf96ada2f0e Mon Sep 17 00:00:00 2001 From: Sreekanth Vadigi Date: Mon, 15 Jun 2026 15:17:10 +0000 Subject: [PATCH] Fix timezone-shifted TIMESTAMP in nested complex types (ES-1978662) TIMESTAMP fields inside nested complex types (STRUCT/ARRAY/MAP) are serialized by Arrow as epoch microseconds. ComplexDataTypeParser built the value via Timestamp.from(instant), which anchors an absolute instant that getString()/getObject() then re-render in the JVM default timezone, producing a spurious offset (e.g. a -5h shift) for nested timestamps while scalar TIMESTAMP retrieval was unaffected. Build the Timestamp from the UTC wall-clock instead, mirroring the scalar conversion path (ArrowToJavaObjectConverter.convertToTimestamp), so the JVM zone cancels out on render. This also fixes nested timestamps in ARRAY and MAP, which share the same parsing path. Adds unit tests (STRUCT and ARRAY) that force a non-UTC JVM zone and assert no shift. Signed-off-by: Sreekanth Vadigi --- NEXT_CHANGELOG.md | 1 + .../jdbc/api/impl/ComplexDataTypeParser.java | 5 +- .../jdbc/api/impl/ComplexDataTypeTest.java | 51 +++++++++++++++++++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/NEXT_CHANGELOG.md b/NEXT_CHANGELOG.md index a972de65e2..bc5d2a6322 100644 --- a/NEXT_CHANGELOG.md +++ b/NEXT_CHANGELOG.md @@ -10,6 +10,7 @@ - Fixed `setCatalog()` and `setSchema()` producing invalid SQL (e.g. `SET CATALOG ``name``) when the catalog or schema name was passed already wrapped in backticks. Backticks are now stripped before wrapping, and `getCatalog()`/`getSchema()` return the bare identifier name. - Fixed metadata SQL generation for catalog, schema, and table identifiers containing backticks. - Fixed SEA result truncation when direct results are disabled. Large, highly-compressible results that span multiple chunks were delivered inline via the old hybrid path and truncated to the first chunk. The SQL Execution path now uses an async (`0s`) wait timeout when direct results are disabled, so results are returned via external links and fetched in full. +- Fixed timezone-shifted TIMESTAMP values when retrieving nested complex types (STRUCT/ARRAY/MAP) with `EnableComplexDatatypeSupport=1`. --- *Note: When making changes, please add your change under the appropriate section diff --git a/src/main/java/com/databricks/jdbc/api/impl/ComplexDataTypeParser.java b/src/main/java/com/databricks/jdbc/api/impl/ComplexDataTypeParser.java index c5162e578b..82dca96d8f 100644 --- a/src/main/java/com/databricks/jdbc/api/impl/ComplexDataTypeParser.java +++ b/src/main/java/com/databricks/jdbc/api/impl/ComplexDataTypeParser.java @@ -20,6 +20,7 @@ import java.time.Instant; import java.time.LocalDate; import java.time.LocalDateTime; +import java.time.ZoneOffset; import java.util.ArrayList; import java.util.Base64; import java.util.Iterator; @@ -241,7 +242,9 @@ private Object convertPrimitive(String text, String type) { long seconds = Math.floorDiv(micros, 1_000_000L); long microsRemainder = Math.floorMod(micros, 1_000_000L); Instant instant = Instant.ofEpochSecond(seconds, microsRemainder * 1_000); - return Timestamp.from(instant); + // Build from the UTC wall-clock; Timestamp.from(instant) gets re-rendered in the JVM + // default timezone, shifting nested TIMESTAMP fields (ES-1978662). + return Timestamp.valueOf(LocalDateTime.ofInstant(instant, ZoneOffset.UTC)); } catch (NumberFormatException nfe) { LOGGER.error(e, "Failed to parse TIMESTAMP value '{}' as epoch microseconds", text); throw e; diff --git a/src/test/java/com/databricks/jdbc/api/impl/ComplexDataTypeTest.java b/src/test/java/com/databricks/jdbc/api/impl/ComplexDataTypeTest.java index bdfe9668cb..93de681ae4 100644 --- a/src/test/java/com/databricks/jdbc/api/impl/ComplexDataTypeTest.java +++ b/src/test/java/com/databricks/jdbc/api/impl/ComplexDataTypeTest.java @@ -4,10 +4,12 @@ import java.math.BigDecimal; import java.sql.*; +import java.time.Instant; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.TimeZone; import org.junit.jupiter.api.Test; public class ComplexDataTypeTest { @@ -643,4 +645,53 @@ public void testStructWithSQLTypes() throws SQLException { assertTrue(attrs[6] instanceof byte[]); assertArrayEquals("binaryData".getBytes(), (byte[]) attrs[6]); } + + /** + * ES-1978662: a nested TIMESTAMP serialized as epoch microseconds must not be timezone-shifted. + * The JVM default zone is forced to a fixed UTC-5 zone so the shift is caught on any host/CI. + */ + @Test + public void testStructTimestampFromEpochMicrosIsNotTimezoneShifted() throws SQLException { + TimeZone originalTimeZone = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("America/Bogota")); + + // Epoch micros for 2017-03-26 01:01:02.345 UTC. + long epochMicros = Instant.parse("2017-03-26T01:01:02.345Z").toEpochMilli() * 1_000L; + String json = "{\"ts_field\":" + epochMicros + "}"; + String metadata = "STRUCT"; + + DatabricksStruct struct = + new ComplexDataTypeParser().parseJsonStringToDbStruct(json, metadata); + Object value = struct.getAttributes()[0]; + + assertTrue(value instanceof Timestamp); + assertEquals(Timestamp.valueOf("2017-03-26 01:01:02.345"), value); + assertEquals("2017-03-26 01:01:02.345", value.toString()); + } finally { + TimeZone.setDefault(originalTimeZone); + } + } + + /** Same fix applies to TIMESTAMP elements inside an ARRAY. */ + @Test + public void testArrayTimestampFromEpochMicrosIsNotTimezoneShifted() throws SQLException { + TimeZone originalTimeZone = TimeZone.getDefault(); + try { + TimeZone.setDefault(TimeZone.getTimeZone("America/Bogota")); + + long epochMicros = Instant.parse("2017-03-26T01:01:02.345Z").toEpochMilli() * 1_000L; + String json = "[" + epochMicros + "]"; + String metadata = "ARRAY"; + + DatabricksArray array = new ComplexDataTypeParser().parseJsonStringToDbArray(json, metadata); + Object value = ((Object[]) array.getArray())[0]; + + assertTrue(value instanceof Timestamp); + assertEquals(Timestamp.valueOf("2017-03-26 01:01:02.345"), value); + assertEquals("2017-03-26 01:01:02.345", value.toString()); + } finally { + TimeZone.setDefault(originalTimeZone); + } + } }