From a85fc7fb935970cdfdd5b62b6deb3f93e3279974 Mon Sep 17 00:00:00 2001 From: zipg Date: Tue, 22 Sep 2026 21:50:38 +0800 Subject: [PATCH] fix(jdbc): render bit columns as bit strings --- .../main/java/app/dbx/jdbc/DbxJdbcPlugin.java | 171 +++++++++++++++++- .../java/app/dbx/jdbc/DbxJdbcPluginTest.java | 147 +++++++++++++++ 2 files changed, 309 insertions(+), 9 deletions(-) diff --git a/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java b/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java index d1fc093ce9..4eb064383b 100644 --- a/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java +++ b/plugins/jdbc/src/main/java/app/dbx/jdbc/DbxJdbcPlugin.java @@ -82,6 +82,24 @@ public final class DbxJdbcPlugin { "SYSTEM TABLE", "SYSTEM VIEW" }; + /** + * MySQL/PostgreSQL 家族（含金仓、瀚高、优炫、海量这类 PG 衍生库）里 BIT 是位字段/位串：数据库自带 + * 工具和 DBX 的内置驱动都按 `0`/`1`/`10101010` 展示。JDBC 驱动把这些列暴露成裸字节或强制转布尔， + * 走通用分支就成了 `0x00`/`true`。这类连接由 readBitStringColumnValue 按位串读取。 + * SQL Server 这类把 BIT 当布尔类型的库不在名单内，保持驱动返回的布尔值。 + */ + private static final String[] BIT_STRING_JDBC_URL_PREFIXES = new String[] { + "jdbc:mysql:", + "jdbc:mariadb:", + "jdbc:starrocks:", + "jdbc:doris:", + "jdbc:postgresql:", + "jdbc:kingbase", + "jdbc:highgo:", + "jdbc:uxdb:", + "jdbc:vastbase:" + }; + private static final JdbcDriverQuirks DEFAULT_QUIRKS = new JdbcDriverQuirks( false, false, @@ -853,6 +871,16 @@ public final class DbxJdbcPlugin { return driverClass != null && driverClass.equalsIgnoreCase("org.postgresql.Driver"); } + private static boolean usesBitStringColumns(JsonNode connection) { + String url = jdbcUrl(connection); + for (String prefix : BIT_STRING_JDBC_URL_PREFIXES) { + if (urlMatchesPrefix(url, prefix)) { + return true; + } + } + return false; + } + private static boolean isPrestoOrTrinoConnection(JsonNode connection) { String url = jdbcUrl(connection); if (urlMatchesPrefix(url, "jdbc:presto:") || urlMatchesPrefix(url, "jdbc:trino:")) { @@ -993,6 +1021,7 @@ public final class DbxJdbcPlugin { JdbcDriverQuirks quirks = driverQuirks(connection); boolean preserveOracleDateTime = isOracleUrl(jdbcUrl(connection)); ZoneId timestampZone = tdengineTimestampZone(connection, conn); + boolean bitStringColumns = usesBitStringColumns(connection); try (Statement statement = conn.createStatement()) { applyStatementOptions(statement, maxRows, fetchSize, timeoutSecs, quirks); String trimmedSql = trimStatementSql(sql); @@ -1022,7 +1051,7 @@ public final class DbxJdbcPlugin { } ArrayNode row = MAPPER.createArrayNode(); for (int i = 1; i <= columnCount; i++) { - row.add(MAPPER.valueToTree(readValue(rs, meta, i, preserveOracleDateTime, timestampZone))); + row.add(MAPPER.valueToTree(readValue(rs, meta, i, preserveOracleDateTime, timestampZone, bitStringColumns))); } rows.add(row); } @@ -1116,6 +1145,7 @@ public final class DbxJdbcPlugin { private final boolean restoreAutoCommit; private final boolean preserveOracleDateTime; private final ZoneId timestampZone; + private final boolean bitStringColumns; private int rowsReturned; private ArrayNode pendingRow; @@ -1130,7 +1160,8 @@ public final class DbxJdbcPlugin { Connection connection, boolean restoreAutoCommit, boolean preserveOracleDateTime, - ZoneId timestampZone + ZoneId timestampZone, + boolean bitStringColumns ) { this.id = id; this.statement = statement; @@ -1143,6 +1174,7 @@ public final class DbxJdbcPlugin { this.restoreAutoCommit = restoreAutoCommit; this.preserveOracleDateTime = preserveOracleDateTime; this.timestampZone = timestampZone; + this.bitStringColumns = bitStringColumns; } } @@ -1162,6 +1194,7 @@ public final class DbxJdbcPlugin { JdbcDriverQuirks quirks = driverQuirks(connection); boolean preserveOracleDateTime = isOracleUrl(jdbcUrl(connection)); ZoneId timestampZone = tdengineTimestampZone(connection, conn); + boolean bitStringColumns = usesBitStringColumns(connection); boolean restoreAutoCommit = beginPagedQueryTransaction(connection, conn); Statement statement; try { @@ -1209,7 +1242,8 @@ public final class DbxJdbcPlugin { conn, restoreAutoCommit, preserveOracleDateTime, - timestampZone + timestampZone, + bitStringColumns ); QUERY_SESSIONS.put(sessionId, session); try { @@ -1282,7 +1316,7 @@ public final class DbxJdbcPlugin { closeQuerySession(session.id); return queryPageResult(session, rows, false, false); } - row = readRow(session.resultSet, session.meta, session.preserveOracleDateTime, session.timestampZone); + row = readRow(session.resultSet, session.meta, session.preserveOracleDateTime, session.timestampZone, session.bitStringColumns); } rows.add(row); session.rowsReturned++; @@ -1300,7 +1334,7 @@ public final class DbxJdbcPlugin { return queryPageResult(session, rows, false, false); } - session.pendingRow = readRow(session.resultSet, session.meta, session.preserveOracleDateTime, session.timestampZone); + session.pendingRow = readRow(session.resultSet, session.meta, session.preserveOracleDateTime, session.timestampZone, session.bitStringColumns); return queryPageResult(session, rows, false, true); } @@ -1354,11 +1388,12 @@ public final class DbxJdbcPlugin { ResultSet rs, ResultSetMetaData meta, boolean preserveOracleDateTime, - ZoneId timestampZone + ZoneId timestampZone, + boolean bitStringColumns ) throws SQLException { ArrayNode row = MAPPER.createArrayNode(); for (int i = 1; i <= meta.getColumnCount(); i++) { - row.add(MAPPER.valueToTree(readValue(rs, meta, i, preserveOracleDateTime, timestampZone))); + row.add(MAPPER.valueToTree(readValue(rs, meta, i, preserveOracleDateTime, timestampZone, bitStringColumns))); } return row; } @@ -4258,7 +4293,7 @@ public final class DbxJdbcPlugin { int index, boolean preserveOracleDateTime ) throws SQLException { - return readValue(rs, meta, index, preserveOracleDateTime, null); + return readValue(rs, meta, index, preserveOracleDateTime, null, false); } private static Object readValue( @@ -4266,7 +4301,8 @@ public final class DbxJdbcPlugin { ResultSetMetaData meta, int index, boolean preserveOracleDateTime, - ZoneId timestampZone + ZoneId timestampZone, + boolean bitStringColumns ) throws SQLException { int columnType = meta.getColumnType(index); @@ -4295,6 +4331,13 @@ public final class DbxJdbcPlugin { return bytes == null ? null : binaryToHex(bytes); } + if (bitStringColumns && isBitStringColumn(meta, index, columnType)) { + Object bitValue = readBitStringColumnValue(rs, meta, index); + if (bitValue != BIT_COLUMN_UNSUPPORTED) { + return bitValue; + } + } + Object value = rs.getObject(index); if (value == null) { return null; @@ -4454,6 +4497,116 @@ public final class DbxJdbcPlugin { return "\"" + identifier.replace("\"", "\"\"") + "\""; } + /** + * readBitStringColumnValue 的哨兵：驱动读不出位串（例如 mssql-jdbc 不允许把 BIT 读成 byte[]）， + * 调用方据此回落到原来的取值路径。SQL NULL 用 null 表示，两者必须分开。 + */ + private static final Object BIT_COLUMN_UNSUPPORTED = new Object(); + + private static boolean isBitStringColumn(ResultSetMetaData meta, int index, int columnType) { + String typeName = columnTypeName(meta, index); + String normalized = typeName.trim().toLowerCase(Locale.ROOT); + if (normalized.equals("bool") || normalized.equals("boolean")) { + // 金仓/PostgreSQL 的布尔列走 Types.BIT 上报，它们继续保持布尔值。 + return false; + } + if (normalized.equals("bit") || normalized.equals("varbit") || normalized.startsWith("bit ") + || normalized.startsWith("bit(") || normalized.startsWith("bit varying")) { + return true; + } + return columnType == Types.BIT; + } + + private static String columnTypeName(ResultSetMetaData meta, int index) { + try { + String typeName = meta.getColumnTypeName(index); + return typeName == null ? "" : typeName; + } catch (SQLException | RuntimeException error) { + return ""; + } + } + + private static Object readBitStringColumnValue(ResultSet rs, ResultSetMetaData meta, int index) throws SQLException { + byte[] payload; + try { + payload = rs.getBytes(index); + } catch (SQLException | AbstractMethodError | UnsupportedOperationException error) { + return BIT_COLUMN_UNSUPPORTED; + } + if (payload == null) { + // wasNull() 区分 SQL NULL 与「驱动不支持按字节读取」。 + return rs.wasNull() ? null : BIT_COLUMN_UNSUPPORTED; + } + int precision = bitColumnPrecision(meta, index); + if (payload.length == 1 && precision <= 1 && (payload[0] == 't' || payload[0] == 'f')) { + // GaussDB/金仓的布尔位串按 't'/'f' 返回，保持布尔语义；位宽大于 1 的列是位字段而不是布尔。 + return payload[0] == 't'; + } + if (payload.length == 1 && precision == 1 && payload[0] != 0x00 && payload[0] != 0x01 + && payload[0] != '0' && payload[0] != '1') { + // Connector/J 默认把 tinyint(1) 上报成 Types.BIT 且位宽为 1，载荷超出 0/1 说明是数值列： + // 截成单个比特会静默丢数据，按有符号字节十进制展示（`'0'`/`'1'` 文本是驱动的位串形式，除外）。 + return payload[0]; + } + String bitString = bitStringFromPayload(payload, precision); + // 位宽不可信时 bitStringFromPayload 返回 null（不是 SQL NULL），交给通用分支保持 `0x..` 展示。 + return bitString == null ? BIT_COLUMN_UNSUPPORTED : bitString; + } + + private static int bitColumnPrecision(ResultSetMetaData meta, int index) { + try { + return meta.getPrecision(index); + } catch (SQLException | RuntimeException error) { + return 0; + } + } + + /** + * 把 BIT 列的字节载荷还原成位串： + * + * - `0x30`/`0x31`（`'0'`/`'1'` 文本）是 PG 家族驱动的文本形式，位串长度等于声明位宽； + * - 其余载荷是裸位字段（MySQL 的 `bit(n)`、驱动不返回文本的 PG 列），按声明位宽展开； + * - 声明位宽超出载荷容量时返回 null，交给调用方保持原有的 `0x..` 展示。 + */ + private static String bitStringFromPayload(byte[] payload, int precision) { + if (payload.length == 0) { + return ""; + } + if (isAsciiBitStringPayload(payload) && (precision <= 0 || payload.length == precision)) { + return new String(payload, StandardCharsets.US_ASCII); + } + int width = precision > 0 ? precision : significantBitWidth(payload); + if (width > payload.length * 8) { + return null; + } + StringBuilder bits = new StringBuilder(width); + for (int bit = width - 1; bit >= 0; bit--) { + int value = payload[payload.length - 1 - bit / 8]; + bits.append(((value >> bit % 8) & 1) == 1 ? '1' : '0'); + } + return bits.toString(); + } + + private static boolean isAsciiBitStringPayload(byte[] payload) { + for (byte value : payload) { + if (value != '0' && value != '1') { + return false; + } + } + return true; + } + + /** 没有声明位宽时，用能表达该数值的最短位宽（`0x00` -> 1 位，`0xaa` -> 8 位）。 */ + private static int significantBitWidth(byte[] payload) { + for (int index = 0; index < payload.length; index++) { + int value = payload[index] & 0xff; + if (value != 0) { + return (payload.length - index - 1) * 8 + (32 - Integer.numberOfLeadingZeros(value)); + } + } + return 1; + } + private static String binaryToHex(byte[] bytes) { StringBuilder out = new StringBuilder(2 + bytes.length * 2); out.append("0x"); diff --git a/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java b/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java index 6c7e4b611f..b47451d51e 100644 --- a/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java +++ b/plugins/jdbc/src/test/java/app/dbx/jdbc/DbxJdbcPluginTest.java @@ -11,6 +11,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Method; import java.lang.reflect.Proxy; import java.math.BigDecimal; +import java.nio.charset.StandardCharsets; import java.sql.Connection; import java.sql.Clob; import java.sql.DatabaseMetaData; @@ -760,6 +761,108 @@ final class DbxJdbcPluginTest { assertEquals("0x0102", method.invoke(null, objectResultSet(new byte[] { 1, 2 }), columnMeta(Types.BIT), 1, false)); } + @Test + void readValueRendersBitFieldColumnsAsBitStrings() throws Exception { + Method method = bitStringReadValue(); + + // MySQL Connector/J 对 bit(n) 直接返回裸位字段，DBX 之前会渲染成 `0x..`。 + assertEquals("0", method.invoke(null, bytesResultSet(new byte[] { 0x00 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + assertEquals("1", method.invoke(null, bytesResultSet(new byte[] { 0x01 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + assertEquals("10101010", method.invoke(null, bytesResultSet(new byte[] { (byte) 0xaa }), columnMeta(Types.BIT, "BIT", 8), 1, false, null, true)); + assertEquals("0000000100000010", method.invoke(null, bytesResultSet(new byte[] { 0x01, 0x02 }), columnMeta(Types.BIT, "BIT", 16), 1, false, null, true)); + assertEquals(null, method.invoke(null, bytesResultSet(null), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + } + + @Test + void readValueRendersTextBitPayloadsAsBitStrings() throws Exception { + Method method = bitStringReadValue(); + + // 金仓/PostgreSQL 驱动把 bit(n) 当 `0`/`1` 文本返回，getBytes 给出 ASCII 位串。 + assertEquals("0", method.invoke(null, bytesResultSet(new byte[] { '0' }), columnMeta(Types.BIT, "bit", 1), 1, false, null, true)); + assertEquals("1", method.invoke(null, bytesResultSet(new byte[] { '1' }), columnMeta(Types.BIT, "bit", 1), 1, false, null, true)); + assertEquals("10101010", method.invoke(null, bytesResultSet("10101010".getBytes(StandardCharsets.US_ASCII)), columnMeta(Types.BIT, "bit", 8), 1, false, null, true)); + assertEquals("101", method.invoke(null, bytesResultSet("101".getBytes(StandardCharsets.US_ASCII)), columnMeta(Types.OTHER, "varbit", 3), 1, false, null, true)); + // 布尔位串按 `t`/`f` 返回时保持布尔语义。 + assertEquals(true, method.invoke(null, bytesResultSet(new byte[] { 't' }), columnMeta(Types.BIT, "bit", 1), 1, false, null, true)); + assertEquals(false, method.invoke(null, bytesResultSet(new byte[] { 'f' }), columnMeta(Types.BIT, "bit", 1), 1, false, null, true)); + // 位宽超出载荷容量（不可信的元数据）时保持原来的 `0x..` 展示。 + assertEquals("0x0102", method.invoke(null, bytesResultSet(new byte[] { 0x01, 0x02 }), columnMeta(Types.BIT, "BIT", 24), 1, false, null, true)); + // 没有声明位宽时按数值的最短位宽展开。 + assertEquals("100000010", method.invoke(null, bytesResultSet(new byte[] { 0x01, 0x02 }), columnMeta(Types.BIT, "BIT", 0), 1, false, null, true)); + } + + @Test + void readValueRendersTinyInt1BitColumnsAsNumbers() throws Exception { + Method method = bitStringReadValue(); + + // Connector/J 默认把 tinyint(1) 上报成 Types.BIT（位宽 1），超出 0/1 的载荷是数值列， + // 截成单个比特会静默丢数据（2 显示成 "0"、3 显示成 "1"），必须按有符号字节十进制展示。 + assertEquals(Byte.valueOf((byte) 2), method.invoke(null, bytesResultSet(new byte[] { 2 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + assertEquals(Byte.valueOf((byte) 127), method.invoke(null, bytesResultSet(new byte[] { 127 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + assertEquals(Byte.valueOf((byte) -1), method.invoke(null, bytesResultSet(new byte[] { (byte) 0xff }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + // 真正的 bit(1) 载荷仍是 0/1，按位串展示。 + assertEquals("0", method.invoke(null, bytesResultSet(new byte[] { 0x00 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + assertEquals("1", method.invoke(null, bytesResultSet(new byte[] { 0x01 }), columnMeta(Types.BIT, "BIT", 1), 1, false, null, true)); + } + + @Test + void readValueDoesNotTreatMultiBitPayloadsAsBooleanText() throws Exception { + Method method = bitStringReadValue(); + + // bit(8) 的裸载荷恰好等于 't'/'f' 的字节值时是位字段而不是布尔文本，按位串展开。 + assertEquals("01110100", method.invoke(null, bytesResultSet(new byte[] { 't' }), columnMeta(Types.BIT, "BIT", 8), 1, false, null, true)); + assertEquals("01100110", method.invoke(null, bytesResultSet(new byte[] { 'f' }), columnMeta(Types.BIT, "BIT", 8), 1, false, null, true)); + } + + @Test + void readValueKeepsDriverBooleanWhenBitColumnCannotBeReadAsBytes() throws Exception { + Method method = bitStringReadValue(); + // mssql-jdbc 拒绝把 BIT 读成 byte[]，此时必须保持驱动返回的布尔值。 + ResultSet rs = (ResultSet) Proxy.newProxyInstance( + DbxJdbcPluginTest.class.getClassLoader(), + new Class[] { ResultSet.class }, + (proxy, invokedMethod, args) -> switch (invokedMethod.getName()) { + case "getBytes" -> throw new SQLException("The conversion from bit to byte[] is not supported."); + case "getObject" -> Boolean.TRUE; + default -> defaultValue(invokedMethod.getReturnType()); + } + ); + + assertEquals(true, method.invoke(null, rs, columnMeta(Types.BIT, "bit", 1), 1, false, null, true)); + } + + @Test + void readValueKeepsBooleanSourceColumnBoolean() throws Exception { + Method method = bitStringReadValue(); + // 金仓的布尔列同样上报 Types.BIT，不能被当成位字段。 + ResultSet rs = (ResultSet) Proxy.newProxyInstance( + DbxJdbcPluginTest.class.getClassLoader(), + new Class[] { ResultSet.class }, + (proxy, invokedMethod, args) -> switch (invokedMethod.getName()) { + case "getBytes" -> new byte[] { 't' }; + case "getObject" -> Boolean.TRUE; + default -> defaultValue(invokedMethod.getReturnType()); + } + ); + + assertEquals(true, method.invoke(null, rs, columnMeta(Types.BIT, "bool", 1), 1, false, null, true)); + } + + @Test + void bitStringColumnsMatchOnlyBitFieldDialects() throws Exception { + Method method = DbxJdbcPlugin.class.getDeclaredMethod("usesBitStringColumns", JsonNode.class); + method.setAccessible(true); + + assertTrue((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:mysql://db:3306/demo\" }"))); + assertTrue((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:mariadb://db:3306/demo\" }"))); + assertTrue((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:kingbase8://db:54321/demo\" }"))); + assertTrue((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:postgresql://db:5432/demo\" }"))); + assertTrue((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:vastbase://db:5432/demo\" }"))); + assertFalse((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:sqlserver://db:1433;databaseName=demo\" }"))); + assertFalse((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:oracle:thin:@db:1521/XE\" }"))); + assertFalse((Boolean) method.invoke(null, MAPPER.readTree("{ \"connection_string\": \"jdbc:h2:mem:dbx\" }"))); + } + @Test void readValueUsesJdbcBooleanAccessForBooleanColumns() throws Exception { Method method = DbxJdbcPlugin.class.getDeclaredMethod( @@ -3209,6 +3312,23 @@ final class DbxJdbcPluginTest { return columnMeta(columnType, ""); } + private static ResultSetMetaData columnMeta(int columnType, String typeName, int precision) { + return (ResultSetMetaData) Proxy.newProxyInstance( + DbxJdbcPluginTest.class.getClassLoader(), + new Class[] { ResultSetMetaData.class }, + (proxy, method, args) -> { + return switch (method.getName()) { + case "getColumnCount" -> 1; + case "getColumnLabel", "getColumnName" -> "FLAG"; + case "getColumnType" -> columnType; + case "getColumnTypeName" -> typeName; + case "getPrecision" -> precision; + default -> defaultValue(method.getReturnType()); + }; + } + ); + } + private static ResultSetMetaData columnMeta(int columnType, String typeName) { return (ResultSetMetaData) Proxy.newProxyInstance( DbxJdbcPluginTest.class.getClassLoader(), @@ -3244,6 +3364,33 @@ final class DbxJdbcPluginTest { ); } + private static Method bitStringReadValue() throws Exception { + Method method = DbxJdbcPlugin.class.getDeclaredMethod( + "readValue", + ResultSet.class, + ResultSetMetaData.class, + int.class, + boolean.class, + ZoneId.class, + boolean.class + ); + method.setAccessible(true); + return method; + } + + private static ResultSet bytesResultSet(byte[] value) { + return (ResultSet) Proxy.newProxyInstance( + DbxJdbcPluginTest.class.getClassLoader(), + new Class[] { ResultSet.class }, + (proxy, method, args) -> switch (method.getName()) { + // 真实驱动会同时通过 getBytes/getObject 暴露二进制载荷。 + case "getBytes", "getObject" -> value; + case "wasNull" -> value == null; + default -> defaultValue(method.getReturnType()); + } + ); + } + private static ResultSet objectResultSet(Object value) { return (ResultSet) Proxy.newProxyInstance( DbxJdbcPluginTest.class.getClassLoader(),