Skip to content

fix(bigquery-jdbc): preserve civil digits for DATETIME/DATE #14910

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
keshavdandeva wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix(bigquery-jdbc): preserve civil digits for DATETIME/DATE #14910

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseArray.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,14 +172,15 @@ public String toString() {
if (array == null) {
return "null";
}
if (this.schema.getType().getStandardType() == StandardSQLTypeName.BYTES) {
StandardSQLTypeName type = this.schema.getType().getStandardType();
if (type == StandardSQLTypeName.BYTES) {
return Arrays.stream(array)
.map(
element ->
element == null ? "null" : Base64.getEncoder().encodeToString((byte[]) element))
.collect(Collectors.joining(", ", "[", "]"));
}
if (this.schema.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
if (type == StandardSQLTypeName.TIMESTAMP) {
return Arrays.stream(array)
.map(
element -> {
Expand All @@ -193,7 +194,7 @@ public String toString() {
})
.collect(Collectors.joining(", ", "[", "]"));
}
if (this.schema.getType().getStandardType() == StandardSQLTypeName.TIME) {
if (BigQueryTemporalUtility.isCivilTemporalType(type)) {
StringBuilder sb = new StringBuilder("[");
for (int i = 0; i < array.length; i++) {
if (i > 0) {
Expand All @@ -202,8 +203,7 @@ public String toString() {
sb.append(
array[i] == null
? "null"
: BigQueryTypeRegistry.convert(
getRawElement(i), StandardSQLTypeName.TIME, String.class, null));
: BigQueryTemporalUtility.formatCivilTemporalElement(type, getRawElement(i)));
}
sb.append("]");
return sb.toString();
Expand Down
47 changes: 15 additions & 32 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseResultSet.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,7 @@
import java.sql.Statement;
import java.sql.Time;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalDateTime;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.util.Calendar;
import java.util.List;
import javax.sql.rowset.serial.SerialBlob;
Expand Down Expand Up @@ -318,45 +312,34 @@ protected Object getObject(int columnIndex, ZoneId zone) throws SQLException {

abstract Object getObjectInternal(int columnIndex) throws SQLException;

private Object getTemporalValue(int columnIndex, StandardSQLTypeName bqType, ZoneId zone)
throws SQLException {
return BigQueryTemporalUtility.isCivilTemporalType(bqType)
? getObjectInternal(columnIndex)
: getObject(columnIndex, zone);
}

@Override
public <T> T getObject(int columnIndex, Class<T> type) throws SQLException {
LOG.finestTrace("getObject");
try {
if (isRawViewTarget(type)) {
StandardSQLTypeName bqType = getStandardSQLTypeName(columnIndex);
// java.sql types can't hold every value of these (sub-ms TIME, pre-1582 DATE/DATETIME).
if (bqType == StandardSQLTypeName.TIME
|| bqType == StandardSQLTypeName.DATE
|| bqType == StandardSQLTypeName.DATETIME) {
return BigQueryTypeRegistry.convert(getObjectInternal(columnIndex), bqType, type, null);
}
StandardSQLTypeName bqType = getStandardSQLTypeName(columnIndex);
if (type != null && BigQueryTemporalUtility.isCivilTemporalType(bqType)) {
return BigQueryTypeRegistry.convert(getObjectInternal(columnIndex), bqType, type, null);
}
Object value = getObject(columnIndex);
if (value == null) {
return null;
}
if (type != null && !type.isInstance(value)) {
StandardSQLTypeName bqType = getStandardSQLTypeName(columnIndex);
if (bqType == StandardSQLTypeName.TIMESTAMP) {
return BigQueryTypeRegistry.convert(value, bqType, type, null);
}
if (type != null && !type.isInstance(value) && bqType == StandardSQLTypeName.TIMESTAMP) {
return BigQueryTypeRegistry.convert(value, bqType, type, null);
}
return BigQueryTypeRegistry.convert(value, type);
} catch (RuntimeException e) {
throw createCoercionException(columnIndex, type, e);
}
}

private static boolean isRawViewTarget(Class<?> type) {
return type == String.class
|| type == LocalDate.class
|| type == LocalDateTime.class
|| type == LocalTime.class
|| type == Instant.class
|| type == OffsetDateTime.class
|| type == ZonedDateTime.class;
}

@Override
public <T> T getObject(String columnLabel, Class<T> type) throws SQLException {
return getObject(getColumnIndex(columnLabel), type);
Expand Down Expand Up @@ -592,7 +575,7 @@ public Date getDate(int columnIndex, Calendar cal) throws SQLException {
StandardSQLTypeName bqType = getStandardSQLTypeName(columnIndex);
ZoneId zone = getEffectiveZone(bqType, cal);
try {
Object value = getObject(columnIndex, zone);
Object value = getTemporalValue(columnIndex, bqType, zone);
return BigQueryTypeRegistry.convert(value, bqType, Date.class, zone);
} catch (BigQueryJdbcException e) {
throw createCoercionException(columnIndex, Date.class, e);
Expand All @@ -608,7 +591,7 @@ public Time getTime(int columnIndex, Calendar cal) throws SQLException {
}
ZoneId zone = getEffectiveZone(bqType, cal);
try {
Object value = getObject(columnIndex, zone);
Object value = getTemporalValue(columnIndex, bqType, zone);
return BigQueryTypeRegistry.convert(value, bqType, Time.class, zone);
} catch (BigQueryJdbcException e) {
throw createCoercionException(columnIndex, Time.class, e);
Expand All @@ -624,7 +607,7 @@ public Timestamp getTimestamp(int columnIndex, Calendar cal) throws SQLException
}
ZoneId zone = getEffectiveZone(bqType, cal);
try {
Object value = getObject(columnIndex, zone);
Object value = getTemporalValue(columnIndex, bqType, zone);
return BigQueryTypeRegistry.convert(value, bqType, Timestamp.class, zone);
} catch (BigQueryJdbcException e) {
throw createCoercionException(columnIndex, Timestamp.class, e);
Expand Down
12 changes: 5 additions & 7 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryBaseStruct.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,23 +117,21 @@ public String toString() {

private static void appendJsonValue(StringBuilder sb, Field field, Object value, Object rawValue)
throws SQLException {
StandardSQLTypeName type = field.getType().getStandardType();
if (value == null) {
sb.append("null");
} else if (field.getType().getStandardType() == StandardSQLTypeName.JSON) {
} else if (type == StandardSQLTypeName.JSON) {
sb.append(value);
} else if (value instanceof String || value instanceof Text) {
String stringValue = value.toString().replace("\"", "\\\"");
sb.append("\"").append(stringValue).append("\"");
} else if (value instanceof Timestamp
&& field.getType().getStandardType() == StandardSQLTypeName.TIMESTAMP) {
} else if (value instanceof Timestamp && type == StandardSQLTypeName.TIMESTAMP) {
sb.append("\"")
.append(BigQueryTemporalUtility.formatTimestampToUtcString((Timestamp) value))
.append("\"");
} else if (value instanceof Time
&& field.getType().getStandardType() == StandardSQLTypeName.TIME) {
} else if (BigQueryTemporalUtility.isCivilTemporalType(type)) {
sb.append("\"")
.append(
BigQueryTypeRegistry.convert(rawValue, StandardSQLTypeName.TIME, String.class, null))
.append(BigQueryTemporalUtility.formatCivilTemporalElement(type, rawValue))
.append("\"");
} else if (value instanceof Timestamp || value instanceof Date || value instanceof Time) {
sb.append("\"").append(value).append("\"");
Expand Down
22 changes: 22 additions & 0 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTemporalUtility.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,6 +392,28 @@ static String formatTimestampToUtcString(Timestamp ts) {
return UTC_TIMESTAMP_TO_STRING_FORMATTER.format(ts.toInstant());
}

static boolean isCivilTemporalType(StandardSQLTypeName type) {
return type == StandardSQLTypeName.DATE
|| type == StandardSQLTypeName.TIME
|| type == StandardSQLTypeName.DATETIME;
}

/**
* Formats a raw civil temporal value ({@code DATE}, {@code TIME}, or {@code DATETIME}) as a
* string for complex types ({@code ARRAY} and {@code STRUCT}) without boxing through {@code
* java.sql} types.
*/
static String formatCivilTemporalElement(StandardSQLTypeName type, Object rawValue)
throws BigQueryJdbcException {
if (type == StandardSQLTypeName.DATETIME) {
LocalDateTime ldt =
BigQueryTypeRegistry.convert(
rawValue, StandardSQLTypeName.DATETIME, LocalDateTime.class, null);
return ldt == null ? null : UTC_TIMESTAMP_TO_STRING_FORMATTER.format(ldt);
}
return BigQueryTypeRegistry.convert(rawValue, type, String.class, null);
}

/**
* Formats a timestamp value (which may be a {@link Long} epoch microsecond, an ISO-8601 string, a
* {@link java.sql.Timestamp}, or an epoch decimal string) into a standard UTC JDBC timestamp
Expand Down
83 changes: 40 additions & 43 deletions java-bigquery-jdbc/src/main/java/com/google/cloud/bigquery/jdbc/BigQueryTypeRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -298,57 +298,54 @@ static TypeDescriptor<?> createDatetimeDescriptor() {
StandardSQLTypeName.DATETIME,
Arrays.asList(Timestamp.class, LocalDateTime.class),
(val, targetClass, zone) -> {
// Modern fast-path: Bypass intermediate object creation for JSR-310 targets
if (targetClass == LocalDateTime.class
|| targetClass == String.class
|| targetClass == LocalDate.class
|| targetClass == LocalTime.class
|| isInstantView(targetClass)) {
LocalDateTime localDateTime;
if (val instanceof LocalDateTime) {
localDateTime = (LocalDateTime) val;
} else if (val instanceof Timestamp) {
localDateTime = ((Timestamp) val).toLocalDateTime();
} else if (val instanceof java.util.Date) {
localDateTime = new Timestamp(((java.util.Date) val).getTime()).toLocalDateTime();
} else if (val instanceof String) {
String isoString =
BigQueryTemporalUtility.truncateFractionalSeconds(
((String) val).replace(' ', 'T'), 9);
localDateTime = LocalDateTime.parse(isoString);
} else {
throw new BigQueryJdbcException("Cannot convert to DATETIME: " + val);
}
if (targetClass == LocalDateTime.class) {
return localDateTime;
}
if (targetClass == String.class) {
return TIMESTAMP_FORMATTER.format(localDateTime);
}
if (targetClass == LocalDate.class) {
return localDateTime.toLocalDate();
}
if (targetClass == LocalTime.class) {
return localDateTime.toLocalTime();
}
return anchorCivil(localDateTime, targetClass, zone);
if (val instanceof Timestamp && targetClass == Timestamp.class && zone == null) {
return val;
}
if (val instanceof java.util.Date
&& !(val instanceof Timestamp)
&& targetClass == Timestamp.class
&& zone == null) {
return new Timestamp(((java.util.Date) val).getTime());
}

// Legacy path: Box values into java.sql.Timestamp
Timestamp ts;
if (val instanceof Timestamp) {
ts = (Timestamp) val;
LocalDateTime localDateTime;
if (val instanceof LocalDateTime) {
localDateTime = (LocalDateTime) val;
} else if (val instanceof Timestamp) {
localDateTime = ((Timestamp) val).toLocalDateTime();
} else if (val instanceof java.util.Date) {
ts = new Timestamp(((java.util.Date) val).getTime());
} else if (val instanceof LocalDateTime) {
ts = BigQueryTemporalUtility.boxDateTime((LocalDateTime) val, zone);
localDateTime = new Timestamp(((java.util.Date) val).getTime()).toLocalDateTime();
} else if (val instanceof String) {
ts = BigQueryTemporalUtility.boxDateTime((String) val, zone);
String isoString =
BigQueryTemporalUtility.truncateFractionalSeconds(
((String) val).replace(' ', 'T'), 9);
localDateTime = LocalDateTime.parse(isoString);
} else {
throw new BigQueryJdbcException("Cannot convert to DATETIME: " + val);
}

return ts;
if (targetClass == LocalDateTime.class) {
return localDateTime;
}
if (targetClass == String.class) {
return TIMESTAMP_FORMATTER.format(localDateTime);
}
if (targetClass == LocalDate.class) {
return localDateTime.toLocalDate();
}
if (targetClass == LocalTime.class) {
return localDateTime.toLocalTime();
}
if (targetClass == Date.class) {
return BigQueryTemporalUtility.boxDate(localDateTime.toLocalDate(), zone);
}
if (targetClass == Time.class) {
return BigQueryTemporalUtility.boxTime(localDateTime.toLocalTime(), zone);
}
if (isInstantView(targetClass)) {
return anchorCivil(localDateTime, targetClass, zone);
}
return BigQueryTemporalUtility.boxDateTime(localDateTime, zone);
});
}

Expand Down
27 changes: 18 additions & 9 deletions ...igquery-jdbc/src/test/java/com/google/cloud/bigquery/jdbc/BigQueryArrowResultSetTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -700,16 +700,20 @@ public void testGetStringOnTimestampAndDatetimeAcrossJvmZones(String jvmZone)
TimeZone originalDefault = TimeZone.getDefault();
try {
TimeZone.setDefault(TimeZone.getTimeZone(ZoneId.of(jvmZone)));
Instant ref = Instant.parse("2025-01-01T12:34:56.123Z");
Instant ref = Instant.parse("2025-03-09T02:34:56.123Z");
BigQueryArrowResultSet rs = createTemporalArrowResultSet(ref);
assertThat(rs.next()).isTrue();

assertThat(rs.getString("ts")).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(rs.getObject("ts", String.class)).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(rs.getString("dt")).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(rs.getString("ts")).isEqualTo("2025-03-09 02:34:56.123000");
assertThat(rs.getObject("ts", String.class)).isEqualTo("2025-03-09 02:34:56.123000");
assertThat(rs.getString("dt")).isEqualTo("2025-03-09 02:34:56.123000");
assertThat(rs.getDate("dt").toLocalDate()).isEqualTo(LocalDate.of(2025, 3, 9));
assertThat(rs.getObject("dt", Date.class).toLocalDate()).isEqualTo(LocalDate.of(2025, 3, 9));
assertThat(rs.getTime("dt").toLocalTime()).isEqualTo(LocalTime.of(2, 34, 56));
assertThat(rs.getObject("dt", Time.class).toLocalTime()).isEqualTo(LocalTime.of(2, 34, 56));

long tsMicros = ref.getEpochSecond() * 1_000_000L + ref.getNano() / 1_000L;
LocalDateTime dtLocal = LocalDateTime.of(2025, 1, 1, 12, 34, 56, 123_000_000);
LocalDateTime dtLocal = LocalDateTime.of(2025, 3, 9, 2, 34, 56, 123_000_000);

Field tsArraySchema =
Field.newBuilder("ts_array", StandardSQLTypeName.TIMESTAMP)
Expand All @@ -718,10 +722,10 @@ public void testGetStringOnTimestampAndDatetimeAcrossJvmZones(String jvmZone)
JsonStringArrayList<Long> tsList = new JsonStringArrayList<>();
tsList.add(tsMicros);
BigQueryArrowArray tsArrowArray = new BigQueryArrowArray(tsArraySchema, tsList);
assertThat(tsArrowArray.toString()).isEqualTo("[2025-01-01 12:34:56.123]");
assertThat(tsArrowArray.toString()).isEqualTo("[2025-03-09 02:34:56.123]");
try (ResultSet nestedTsRs = tsArrowArray.getResultSet()) {
assertThat(nestedTsRs.next()).isTrue();
assertThat(nestedTsRs.getString(2)).isEqualTo("2025-01-01 12:34:56.123000");
assertThat(nestedTsRs.getString(2)).isEqualTo("2025-03-09 02:34:56.123000");
}

Field dtArraySchema =
Expand All @@ -731,7 +735,12 @@ public void testGetStringOnTimestampAndDatetimeAcrossJvmZones(String jvmZone)
JsonStringArrayList<LocalDateTime> dtList = new JsonStringArrayList<>();
dtList.add(dtLocal);
BigQueryArrowArray dtArrowArray = new BigQueryArrowArray(dtArraySchema, dtList);
assertThat(dtArrowArray.toString()).isEqualTo("[2025-01-01 12:34:56.123]");
assertThat(dtArrowArray.toString()).isEqualTo("[2025-03-09 02:34:56.123]");
try (ResultSet nestedDtRs = dtArrowArray.getResultSet()) {
assertThat(nestedDtRs.next()).isTrue();
assertThat(nestedDtRs.getString(2)).isEqualTo("2025-03-09 02:34:56.123000");
assertThat(nestedDtRs.getTime(2).toLocalTime()).isEqualTo(LocalTime.of(2, 34, 56));
}

FieldList structSchema =
FieldList.of(
Expand All @@ -742,7 +751,7 @@ public void testGetStringOnTimestampAndDatetimeAcrossJvmZones(String jvmZone)
structValues.put("dt", dtLocal);
BigQueryArrowStruct arrowStruct = new BigQueryArrowStruct(structSchema, structValues);
assertThat(arrowStruct.toString())
.isEqualTo("{\"ts\":\"2025-01-01 12:34:56.123\",\"dt\":\"2025-01-01 12:34:56.123\"}");
.isEqualTo("{\"ts\":\"2025-03-09 02:34:56.123\",\"dt\":\"2025-03-09 02:34:56.123\"}");
} finally {
TimeZone.setDefault(originalDefault);
}
Expand Down
Loading
Loading