fix(oracle): load CLOB values lazily and safely

This commit is contained in:
zipg authored and GitHub committed 2026-08-14 17:46:34 +08:00
1 parent 1cd962d9c7
commit 518a8b593c
6 files changed
+495 -62

No files matched your search

+185 -55
View File
@@ -261,8 +261,11 @@ type queryOptions struct {
MaxRows int `json:"maxRows"` MaxRows int `json:"maxRows"`
FetchSize int `json:"fetchSize"` FetchSize int `json:"fetchSize"`
TimeoutSecs int `json:"timeoutSecs"` TimeoutSecs int `json:"timeoutSecs"`
DeferLOBs bool `json:"deferLobs"`
} }
const largeValueBytesColumnPrefix = "__DBX_LARGE_VALUE_BYTES_"
type queryResult struct { type queryResult struct {
Columns []string `json:"columns"` Columns []string `json:"columns"`
ColumnTypes []string `json:"column_types"` ColumnTypes []string `json:"column_types"`
@@ -3354,7 +3357,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes
HasMore: false, HasMore: false,
}, err }, err
} }
rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, opts.TimeoutSecs) rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, opts.DeferLOBs)
if err != nil { if err != nil {
return queryPageResult{}, err return queryPageResult{}, err
} }
@@ -3420,7 +3423,7 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul
if !isQuerySQL(sqlText) { if !isQuerySQL(sqlText) {
return queryPageResult{}, errors.New("table read requires a SELECT query") return queryPageResult{}, errors.New("table read requires a SELECT query")
} }
rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, opts.TimeoutSecs) rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, opts.DeferLOBs)
if err != nil { if err != nil {
return queryPageResult{}, err return queryPageResult{}, err
} }
@@ -3555,7 +3558,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
maxRows = defaultMaxRows maxRows = defaultMaxRows
} }
if isQuerySQL(sqlText) { if isQuerySQL(sqlText) {
result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs) result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs, opts.DeferLOBs)
result.ExecutionTimeMS = time.Since(start).Milliseconds() result.ExecutionTimeMS = time.Since(start).Milliseconds()
return result, err return result, err
} }
@@ -3598,11 +3601,11 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) {
return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil
} }
func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int) (queryResult, error) { func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int, deferLOBs bool) (queryResult, error) {
return executeOracleSelectWithXMLTypeRetry( return executeOracleSelectWithXMLTypeRetry(
sqlText, sqlText,
func(query string) (queryResult, error) { func(query string) (queryResult, error) {
return s.executeSelectOnce(query, maxRows, timeoutSecs) return s.executeSelectOnce(query, maxRows, timeoutSecs, deferLOBs)
}, },
s.rewriteXMLTypeSelectSQL, s.rewriteXMLTypeSelectSQL,
) )
@@ -3633,8 +3636,8 @@ func shouldRetryOracleXMLTypeRewrite(err error) bool {
strings.Contains(message, "TTC error: received code ") strings.Contains(message, "TTC error: received code ")
} }
func (s *server) executeSelectOnce(sqlText string, maxRows int, timeoutSecs int) (queryResult, error) { func (s *server) executeSelectOnce(sqlText string, maxRows int, timeoutSecs int, deferLOBs bool) (queryResult, error) {
rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, timeoutSecs) rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, timeoutSecs, deferLOBs)
if err != nil { if err != nil {
return queryResult{}, err return queryResult{}, err
} }
@@ -3693,15 +3696,22 @@ func columnTypeNames(rows *sql.Rows) []string {
return result return result
} }
func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string, timeoutSecs int) (*sql.Rows, error) { func (s *server) queryRowsWithOracleValueRewriteIfNeeded(sqlText string, timeoutSecs int, deferLOBs bool) (*sql.Rows, error) {
if deferLOBs {
rewritten, err := rewriteOracleSelectSQL(sqlText, s.loadOracleColumnMeta, true)
if err == nil && rewritten != sqlText {
return s.queryRowsWithTimeout(rewritten, nil, timeoutSecs)
}
}
rows, err := s.queryRowsWithTimeout(sqlText, nil, timeoutSecs) rows, err := s.queryRowsWithTimeout(sqlText, nil, timeoutSecs)
if err != nil { if err != nil {
return nil, err return nil, err
} }
if !rowsContainOracleXMLType(rows) { typeNames := columnTypeNames(rows)
if !oracleColumnTypeNamesContainXMLType(typeNames) {
return rows, nil return rows, nil
} }
rewritten, err := s.rewriteXMLTypeSelectSQL(sqlText) rewritten, err := rewriteOracleSelectSQL(sqlText, s.loadOracleColumnMeta, false)
if err != nil { if err != nil {
s.closeRows(rows) s.closeRows(rows)
return nil, err return nil, err
@@ -3709,24 +3719,12 @@ func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string, timeoutSecs
if rewritten == sqlText { if rewritten == sqlText {
return rows, nil return rows, nil
} }
// Only pay the ALL_TAB_COLUMNS rewrite cost when the result metadata shows // XMLTYPE keeps its metadata-triggered fallback so ordinary non-preview
// XMLTYPE. Ordinary Oracle queries should not run dictionary probes first. // queries do not run dictionary probes.
s.closeRows(rows) s.closeRows(rows)
return s.queryRowsWithTimeout(rewritten, nil, timeoutSecs) return s.queryRowsWithTimeout(rewritten, nil, timeoutSecs)
} }
func rowsContainOracleXMLType(rows *sql.Rows) bool {
types, err := rows.ColumnTypes()
if err != nil {
return false
}
typeNames := make([]string, 0, len(types))
for _, columnType := range types {
typeNames = append(typeNames, columnType.DatabaseTypeName())
}
return oracleColumnTypeNamesContainXMLType(typeNames)
}
func oracleColumnTypeNamesContainXMLType(typeNames []string) bool { func oracleColumnTypeNamesContainXMLType(typeNames []string) bool {
for _, typeName := range typeNames { for _, typeName := range typeNames {
if isOracleXMLType(typeName) { if isOracleXMLType(typeName) {
@@ -3741,22 +3739,84 @@ func (s *server) rewriteXMLTypeSelectSQL(sqlText string) (string, error) {
} }
func rewriteOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader) (string, error) { func rewriteOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader) (string, error) {
rewritten, _, err := rewriteOracleXMLTypeSelectSQLDepth(sqlText, loadColumns, 0) return rewriteOracleSelectSQL(sqlText, loadColumns, false)
}
func rewriteOracleSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool) (string, error) {
rewritten, _, err := rewriteOracleSelectSQLDepth(sqlText, loadColumns, deferLOBs, 0)
return rewritten, err return rewritten, err
} }
func rewriteOracleXMLTypeSelectSQLDepth(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) { func rewriteOracleSelectSQLDepth(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool, depth int) (string, bool, error) {
if depth > 8 { if depth > 8 {
return sqlText, false, nil return sqlText, false, nil
} }
if rewritten, changed, handled, err := rewriteDirectOracleXMLTypeSelectSQL(sqlText, loadColumns); handled || err != nil { if rewritten, changed, handled, err := rewriteDirectOracleSelectSQL(sqlText, loadColumns, deferLOBs); handled || err != nil {
return rewritten, changed, err return rewritten, changed, err
} }
rewritten, changed, err := rewriteNestedOracleSelects(sqlText, loadColumns, depth) if deferLOBs {
return rewriteOracleFullPassthroughInnerSelect(sqlText, loadColumns, depth)
}
rewritten, changed, err := rewriteNestedOracleSelects(sqlText, loadColumns, deferLOBs, depth)
return rewritten, changed, err return rewritten, changed, err
} }
func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) { func rewriteOracleFullPassthroughInnerSelect(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) {
innerStart, innerEnd, ok := oracleFullPassthroughInnerSelectRange(sqlText)
if !ok {
return sqlText, false, nil
}
inner := sqlText[innerStart:innerEnd]
rewritten, changed, err := rewriteOracleSelectSQLDepth(inner, loadColumns, true, depth+1)
if err != nil || !changed {
return sqlText, false, err
}
return sqlText[:innerStart] + rewritten + sqlText[innerEnd:], true, nil
}
func oracleFullPassthroughInnerSelectRange(sqlText string) (int, int, bool) {
selectStart := leadingSQLSelectListStart(sqlText)
if selectStart < 0 {
return 0, 0, false
}
fromIdx := findTopLevelSQLKeyword(sqlText, selectStart, "from")
if fromIdx < 0 {
return 0, 0, false
}
prefix, selectList := splitOracleSelectListModifier(sqlText[selectStart:fromIdx])
if strings.TrimSpace(prefix) != "" {
return 0, 0, false
}
items := splitTopLevelSQLList(selectList)
if len(items) != 1 {
return 0, 0, false
}
if _, ok := parseOracleStarSelectItem(items[0]); !ok {
return 0, 0, false
}
open := skipSQLWhitespace(sqlText, fromIdx+len("from"))
if open >= len(sqlText) || sqlText[open] != '(' {
return 0, 0, false
}
close := findMatchingSQLParen(sqlText, open)
if close < 0 || !startsWithSQLKeyword(trimLeadingSQLComments(sqlText[open+1:close]), "select") {
return 0, 0, false
}
pos := skipSQLWhitespace(sqlText, close+1)
if pos < len(sqlText) && sqlText[pos] != ';' && !nextKeywordIsOracleClause(sqlText[pos:]) {
_, afterAlias, aliasOK := readOracleIdentifierToken(sqlText, pos)
if !aliasOK {
return 0, 0, false
}
pos = skipSQLWhitespace(sqlText, afterAlias)
}
if pos < len(sqlText) && sqlText[pos] != ';' && !nextKeywordIsOracleClause(sqlText[pos:]) {
return 0, 0, false
}
return open + 1, close, true
}
func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool, depth int) (string, bool, error) {
var builder strings.Builder var builder strings.Builder
changed := false changed := false
last := 0 last := 0
@@ -3781,7 +3841,7 @@ func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoad
} }
inner := sqlText[pos+1 : end] inner := sqlText[pos+1 : end]
if startsWithSQLKeyword(trimLeadingSQLComments(inner), "select") { if startsWithSQLKeyword(trimLeadingSQLComments(inner), "select") {
rewrittenInner, innerChanged, err := rewriteOracleXMLTypeSelectSQLDepth(inner, loadColumns, depth+1) rewrittenInner, innerChanged, err := rewriteOracleSelectSQLDepth(inner, loadColumns, deferLOBs, depth+1)
if err != nil { if err != nil {
return "", false, err return "", false, err
} }
@@ -3802,7 +3862,7 @@ func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoad
return builder.String(), true, nil return builder.String(), true, nil
} }
func rewriteDirectOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader) (string, bool, bool, error) { func rewriteDirectOracleSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool) (string, bool, bool, error) {
selectStart := leadingSQLSelectListStart(sqlText) selectStart := leadingSQLSelectListStart(sqlText)
if selectStart < 0 { if selectStart < 0 {
return sqlText, false, false, nil return sqlText, false, false, nil
@@ -3811,23 +3871,30 @@ func rewriteDirectOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColum
if fromIdx < 0 { if fromIdx < 0 {
return sqlText, false, false, nil return sqlText, false, false, nil
} }
if deferLOBs && oracleSQLHasTopLevelSetOperator(sqlText, fromIdx+len("from")) {
return sqlText, false, false, nil
}
selectListPrefix, selectList := splitOracleSelectListModifier(sqlText[selectStart:fromIdx]) selectListPrefix, selectList := splitOracleSelectListModifier(sqlText[selectStart:fromIdx])
deferSelectLOBs := deferLOBs && !startsWithSQLKeyword(strings.TrimSpace(selectListPrefix), "distinct")
tableRef, ok := parseSingleOracleTableRef(sqlText[fromIdx+len("from"):]) tableRef, ok := parseSingleOracleTableRef(sqlText[fromIdx+len("from"):])
if !ok { if !ok {
return sqlText, false, false, nil return sqlText, false, false, nil
} }
items := splitTopLevelSQLList(selectList) items := splitTopLevelSQLList(selectList)
if len(items) == 0 || !oracleSelectListMayReferenceXMLType(items) { if len(items) == 0 || !oracleSelectListMayReferenceTableColumns(items) {
return sqlText, false, true, nil return sqlText, false, true, nil
} }
columns, err := loadColumns(tableRef.Schema, tableRef.Table) columns, err := loadColumns(tableRef.Schema, tableRef.Table)
if err != nil { if err != nil {
return "", false, true, err return "", false, true, err
} }
if !oracleColumnsHaveXMLType(columns) { if deferSelectLOBs && oracleColumnsConflictWithLargeValueMarkers(columns) {
deferSelectLOBs = false
}
if !oracleColumnsNeedValueRewrite(columns, deferSelectLOBs) {
return sqlText, false, true, nil return sqlText, false, true, nil
} }
rewrittenItems, changed := rewriteOracleSelectItemsForXMLType(items, columns, tableRef) rewrittenItems, changed := rewriteOracleSelectItems(items, columns, tableRef, deferSelectLOBs)
if !changed { if !changed {
return sqlText, false, true, nil return sqlText, false, true, nil
} }
@@ -3922,7 +3989,16 @@ func splitOracleSelectListModifier(selectList string) (string, string) {
return selectList[:prefixLen], selectList[prefixLen:] return selectList[:prefixLen], selectList[prefixLen:]
} }
func oracleSelectListMayReferenceXMLType(items []string) bool { func oracleSQLHasTopLevelSetOperator(sqlText string, start int) bool {
for _, keyword := range []string{"union", "minus", "intersect"} {
if findTopLevelSQLKeyword(sqlText, start, keyword) >= 0 {
return true
}
}
return false
}
func oracleSelectListMayReferenceTableColumns(items []string) bool {
for _, item := range items { for _, item := range items {
if _, ok := parseOracleStarSelectItem(item); ok { if _, ok := parseOracleStarSelectItem(item); ok {
return true return true
@@ -3934,49 +4010,89 @@ func oracleSelectListMayReferenceXMLType(items []string) bool {
return false return false
} }
func rewriteOracleSelectItemsForXMLType(items []string, columns []oracleColumnMeta, tableRef oracleTableRef) ([]string, bool) { func rewriteOracleSelectItems(items []string, columns []oracleColumnMeta, tableRef oracleTableRef, deferLOBs bool) ([]string, bool) {
xmlColumns := map[string]oracleColumnMeta{} columnsByName := map[string]oracleColumnMeta{}
for _, column := range columns { for _, column := range columns {
if isOracleXMLType(column.DataType) { columnsByName[oracleIdentifierKey(column.Name)] = column
xmlColumns[oracleIdentifierKey(column.Name)] = column
}
} }
rewritten := make([]string, 0, len(items)) rewritten := make([]string, 0, len(items))
changed := false changed := false
sourceIndex := 0
for _, item := range items { for _, item := range items {
if qualifier, ok := parseOracleStarSelectItem(item); ok && oracleQualifierMatchesTable(qualifier, tableRef) { if qualifier, ok := parseOracleStarSelectItem(item); ok && oracleQualifierMatchesTable(qualifier, tableRef) {
for _, column := range columns { for _, column := range columns {
rewritten = append(rewritten, oracleSelectExpressionForColumn(column, tableRef, xmlColumns)) columnRef := oracleColumnRef(tableRef.AliasText, column.Name)
outputAlias := quoteIdentifier(column.Name)
if isOracleXMLType(column.DataType) {
rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias))
} else if deferLOBs {
if expressions, ok := oracleDeferredLOBExpressions(columnRef, outputAlias, sourceIndex, column.DataType); ok {
rewritten = append(rewritten, expressions...)
} else {
rewritten = append(rewritten, columnRef)
}
} else {
rewritten = append(rewritten, columnRef)
}
sourceIndex++
} }
changed = true changed = true
continue continue
} }
qualifier, column, alias, ok := parseOracleColumnSelectItem(item) qualifier, column, alias, ok := parseOracleColumnSelectItem(item)
if ok && oracleQualifierMatchesTable(qualifier, tableRef) { if ok && oracleQualifierMatchesTable(qualifier, tableRef) {
if meta, isXML := xmlColumns[oracleIdentifierKey(column.Name)]; isXML { if meta, exists := columnsByName[oracleIdentifierKey(column.Name)]; exists {
outputAlias := alias outputAlias := alias
if outputAlias == "" { if outputAlias == "" {
outputAlias = quoteIdentifier(meta.Name) outputAlias = quoteIdentifier(meta.Name)
} }
rewritten = append(rewritten, oracleXMLSerializeExpression(oracleColumnRef(qualifier, meta.Name), outputAlias)) columnRef := oracleColumnRef(qualifier, meta.Name)
changed = true if isOracleXMLType(meta.DataType) {
continue rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias))
changed = true
sourceIndex++
continue
}
if deferLOBs {
if expressions, isLOB := oracleDeferredLOBExpressions(columnRef, outputAlias, sourceIndex, meta.DataType); isLOB {
rewritten = append(rewritten, expressions...)
changed = true
sourceIndex++
continue
}
}
} }
} }
rewritten = append(rewritten, item) rewritten = append(rewritten, item)
sourceIndex++
} }
return rewritten, changed return rewritten, changed
} }
func oracleSelectExpressionForColumn(column oracleColumnMeta, tableRef oracleTableRef, xmlColumns map[string]oracleColumnMeta) string { func oracleDeferredLOBExpressions(columnRef, outputAlias string, sourceIndex int, dataType string) ([]string, bool) {
qualifier := "" kind, placeholder, ok := oracleDeferredLOBKind(dataType)
if tableRef.AliasText != "" { if !ok {
qualifier = tableRef.AliasText return nil, false
} }
if _, isXML := xmlColumns[oracleIdentifierKey(column.Name)]; isXML { valueExpression := fmt.Sprintf("CASE WHEN %s IS NULL THEN NULL ELSE '%s' END AS %s", columnRef, placeholder, outputAlias)
return oracleXMLSerializeExpression(oracleColumnRef(qualifier, column.Name), quoteIdentifier(column.Name)) markerAlias := fmt.Sprintf("%s%s_%d", largeValueBytesColumnPrefix, kind, sourceIndex)
markerExpression := fmt.Sprintf("CASE WHEN %s IS NULL THEN NULL ELSE 'D:1' END AS %s", columnRef, quoteIdentifier(markerAlias))
return []string{valueExpression, markerExpression}, true
}
func oracleDeferredLOBKind(dataType string) (kind, placeholder string, ok bool) {
switch strings.ToUpper(strings.TrimSpace(dataType)) {
case "CLOB":
return "C", "<CLOB>", true
case "NCLOB":
return "N", "<NCLOB>", true
case "BLOB":
return "L", "<BLOB>", true
case "BFILE":
return "F", "<BFILE>", true
default:
return "", "", false
} }
return oracleColumnRef(qualifier, column.Name)
} }
func oracleXMLSerializeExpression(columnRef, alias string) string { func oracleXMLSerializeExpression(columnRef, alias string) string {
@@ -4057,15 +4173,29 @@ func oracleQualifierMatchesTable(qualifier string, tableRef oracleTableRef) bool
return key == oracleIdentifierKey(tableRef.Table) return key == oracleIdentifierKey(tableRef.Table)
} }
func oracleColumnsHaveXMLType(columns []oracleColumnMeta) bool { func oracleColumnsNeedValueRewrite(columns []oracleColumnMeta, deferLOBs bool) bool {
for _, column := range columns { for _, column := range columns {
if isOracleXMLType(column.DataType) { if isOracleXMLType(column.DataType) || (deferLOBs && isOracleDeferredLOBType(column.DataType)) {
return true return true
} }
} }
return false return false
} }
func oracleColumnsConflictWithLargeValueMarkers(columns []oracleColumnMeta) bool {
for _, column := range columns {
if strings.HasPrefix(strings.ToUpper(column.Name), largeValueBytesColumnPrefix) {
return true
}
}
return false
}
func isOracleDeferredLOBType(dataType string) bool {
_, _, ok := oracleDeferredLOBKind(dataType)
return ok
}
func isOracleXMLType(dataType string) bool { func isOracleXMLType(dataType string) bool {
normalized := strings.ToUpper(strings.TrimSpace(dataType)) normalized := strings.ToUpper(strings.TrimSpace(dataType))
return normalized == "XMLTYPE" || normalized == "SYS.XMLTYPE" return normalized == "XMLTYPE" || normalized == "SYS.XMLTYPE"
@@ -4224,7 +4354,7 @@ func oracleIdentifierKey(value string) string {
func oracleIdentifierIsClause(value string) bool { func oracleIdentifierIsClause(value string) bool {
switch oracleIdentifierKey(value) { switch oracleIdentifierKey(value) {
case "WHERE", "GROUP", "ORDER", "HAVING", "CONNECT", "START", "MODEL", "FETCH", "OFFSET", "UNION", "MINUS", "INTERSECT": case "WHERE", "GROUP", "ORDER", "HAVING", "CONNECT", "START", "MODEL", "FETCH", "OFFSET", "FOR", "AS", "UNION", "MINUS", "INTERSECT":
return true return true
default: default:
return false return false
+117
View File
@@ -2046,6 +2046,123 @@ func TestRewriteOracleXMLTypeSkipsJoins(t *testing.T) {
} }
} }
func TestRewriteOracleLOBSelectStarAsDeferredValues(t *testing.T) {
sqlText, err := rewriteOracleSelectSQL(
`SELECT t.* FROM TEST_LOBS t ORDER BY t.ID DESC`,
fakeOracleColumnLoader([]oracleColumnMeta{
{Name: "ID", DataType: "NUMBER"},
{Name: "PAYLOAD", DataType: "CLOB"},
{Name: "NATIONAL_TEXT", DataType: "NCLOB"},
{Name: "BINARY_DATA", DataType: "BLOB"},
{Name: "FILE_DATA", DataType: "BFILE"},
}),
true,
)
if err != nil {
t.Fatal(err)
}
want := `SELECT t."ID", CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE '<CLOB>' END AS "PAYLOAD", CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_C_1", CASE WHEN t."NATIONAL_TEXT" IS NULL THEN NULL ELSE '<NCLOB>' END AS "NATIONAL_TEXT", CASE WHEN t."NATIONAL_TEXT" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_N_2", CASE WHEN t."BINARY_DATA" IS NULL THEN NULL ELSE '<BLOB>' END AS "BINARY_DATA", CASE WHEN t."BINARY_DATA" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_L_3", CASE WHEN t."FILE_DATA" IS NULL THEN NULL ELSE '<BFILE>' END AS "FILE_DATA", CASE WHEN t."FILE_DATA" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_F_4" FROM TEST_LOBS t ORDER BY t.ID DESC`
if sqlText != want {
t.Fatalf("rewriteOracleSelectSQL() = %s, want %s", sqlText, want)
}
}
func TestRewriteOracleLOBExplicitColumnUsesVisibleResultIndex(t *testing.T) {
sqlText, err := rewriteOracleSelectSQL(
`SELECT t.ID, t.PAYLOAD AS body, LENGTH(t.PAYLOAD) AS payload_length FROM TEST_LOBS t`,
fakeOracleColumnLoader([]oracleColumnMeta{
{Name: "ID", DataType: "NUMBER"},
{Name: "PAYLOAD", DataType: "CLOB"},
}),
true,
)
if err != nil {
t.Fatal(err)
}
want := `SELECT t.ID, CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE '<CLOB>' END AS body, CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_C_1", LENGTH(t.PAYLOAD) AS payload_length FROM TEST_LOBS t`
if sqlText != want {
t.Fatalf("rewriteOracleSelectSQL() = %s, want %s", sqlText, want)
}
}
func TestRewriteOracleLOBNestedRownumQuery(t *testing.T) {
sqlText, err := rewriteOracleSelectSQL(
`SELECT * FROM (SELECT ID, PAYLOAD FROM TEST_LOBS ORDER BY ID DESC) WHERE ROWNUM <= 10`,
fakeOracleColumnLoader([]oracleColumnMeta{
{Name: "ID", DataType: "NUMBER"},
{Name: "PAYLOAD", DataType: "CLOB"},
}),
true,
)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(sqlText, `CASE WHEN "PAYLOAD" IS NULL THEN NULL ELSE '<CLOB>' END AS "PAYLOAD"`) ||
!strings.Contains(sqlText, `"__DBX_LARGE_VALUE_BYTES_C_1"`) {
t.Fatalf("expected nested CLOB column to be deferred, got: %s", sqlText)
}
}
func TestRewriteOracleLOBSkipsDerivedProjectionThatDropsMarkers(t *testing.T) {
called := false
input := `SELECT PAYLOAD FROM (SELECT PAYLOAD FROM TEST_LOBS)`
sqlText, err := rewriteOracleSelectSQL(
input,
func(schema, table string) ([]oracleColumnMeta, error) {
called = true
return []oracleColumnMeta{{Name: "PAYLOAD", DataType: "CLOB"}}, nil
},
true,
)
if err != nil {
t.Fatal(err)
}
if called {
t.Fatal("derived projection should not load table metadata")
}
if sqlText != input {
t.Fatalf("derived projection should remain unchanged, got: %s", sqlText)
}
}
func TestRewriteOracleLOBPreservesUnsafeQueries(t *testing.T) {
columns := []oracleColumnMeta{{Name: "ID", DataType: "NUMBER"}, {Name: "PAYLOAD", DataType: "CLOB"}}
tests := []string{
`SELECT DISTINCT PAYLOAD FROM TEST_LOBS`,
`SELECT l.PAYLOAD FROM TEST_LOBS l JOIN OTHER_TABLE o ON o.ID = l.ID`,
`SELECT PAYLOAD FROM TEST_LOBS UNION ALL SELECT PAYLOAD FROM OTHER_TABLE`,
}
for _, input := range tests {
sqlText, err := rewriteOracleSelectSQL(input, fakeOracleColumnLoader(columns), true)
if err != nil {
t.Fatal(err)
}
if sqlText != input {
t.Fatalf("unsafe query should not be rewritten: %s", sqlText)
}
}
}
func TestRewriteOracleLOBRequiresDeferredModeAndSafeMarkerNames(t *testing.T) {
input := `SELECT * FROM TEST_LOBS`
tests := []struct {
deferLOBs bool
columns []oracleColumnMeta
}{
{deferLOBs: false, columns: []oracleColumnMeta{{Name: "ID", DataType: "NUMBER"}, {Name: "PAYLOAD", DataType: "CLOB"}}},
{deferLOBs: true, columns: []oracleColumnMeta{{Name: "PAYLOAD", DataType: "CLOB"}, {Name: "__DBX_LARGE_VALUE_BYTES_C_0", DataType: "VARCHAR2"}}},
}
for _, test := range tests {
sqlText, err := rewriteOracleSelectSQL(input, fakeOracleColumnLoader(test.columns), test.deferLOBs)
if err != nil {
t.Fatal(err)
}
if sqlText != input {
t.Fatalf("query should remain unchanged, got: %s", sqlText)
}
}
}
func TestOracleColumnTypeNamesContainXMLType(t *testing.T) { func TestOracleColumnTypeNamesContainXMLType(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -4202,7 +4202,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[])
} }
} }
if (requestsByColumn.size === 0) return resolved; if (requestsByColumn.size === 0) return resolved;
if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres" && resolvedDatabaseType.value !== "oracle") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) {
throw new Error(t("grid.largeValueNeedsStableKey")); throw new Error(t("grid.largeValueNeedsStableKey"));
} }
const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex); const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex);
@@ -672,6 +672,119 @@ describe("queryStore hidden primary key editing", () => {
expect(tab.result?.hidden_column_indexes).toBeUndefined(); expect(tab.result?.hidden_column_indexes).toBeUndefined();
}); });
it("enables deferred Oracle LOBs only when a base-table query has a stable key", async () => {
getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 });
getColumns.mockResolvedValue([
{ name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: true, extra: null },
{ name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
]);
lookupLocalCompletionTables.mockReturnValue([{ name: "DOCUMENTS", type: "table", schema: "APP" }]);
analyzeEditableQueryEditability.mockImplementation(async (sql: string) => ({
editable: true,
analysis: {
schema: "APP",
tableName: "DOCUMENTS",
selectStar: false,
columns: [{ sourceName: "PAYLOAD", resultName: "PAYLOAD", expression: "PAYLOAD" }, ...(sql.includes("__DBX_PK_0") ? [{ sourceName: "ID", resultName: "__DBX_PK_0", expression: '"ID"' }] : [])],
},
}));
executeMulti.mockResolvedValue([{ columns: ["PAYLOAD", "__DBX_PK_0"], rows: [["<CLOB>", 1]], affected_rows: 0, execution_time_ms: 1 }]);
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("oracle-1", "ORCL", "Query");
await store.executeTabSql(tabId, "SELECT PAYLOAD FROM APP.DOCUMENTS");
expect(executeMulti).toHaveBeenCalledWith("oracle-1", "ORCL", 'SELECT PAYLOAD, "ID" AS "__DBX_PK_0" FROM APP.DOCUMENTS', undefined, expect.any(String), expect.objectContaining({ tableDataPreview: true, timeoutSecs: 30 }));
});
it("keeps deferred Oracle LOBs disabled for views", async () => {
getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 });
getColumns.mockResolvedValue([
{ name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: false, extra: null },
{ name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
]);
listIndexes.mockResolvedValue([]);
lookupLocalCompletionTables.mockReturnValue([{ name: "DOCUMENT_VIEW", type: "view", schema: "APP" }]);
analyzeEditableQueryEditability.mockResolvedValue({
editable: true,
analysis: {
schema: "APP",
tableName: "DOCUMENT_VIEW",
selectStar: true,
columns: [],
},
});
executeMulti.mockResolvedValue([{ columns: ["ID", "PAYLOAD"], rows: [[1, "value"]], affected_rows: 0, execution_time_ms: 1 }]);
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("oracle-1", "ORCL", "Query");
await store.executeTabSql(tabId, "SELECT * FROM APP.DOCUMENT_VIEW");
expect(executeMulti).toHaveBeenCalledOnce();
expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview");
});
it("keeps deferred Oracle LOBs disabled when a keyless source has no safe ROWID path", async () => {
getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 });
getColumns.mockResolvedValue([
{ name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: false, extra: null },
{ name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
]);
listIndexes.mockResolvedValue([]);
lookupLocalCompletionTables.mockReturnValue([]);
analyzeEditableQueryEditability.mockResolvedValue({
editable: true,
analysis: {
schema: undefined,
tableName: "DOCUMENTS",
selectStar: true,
columns: [],
},
});
executeMulti.mockResolvedValue([{ columns: ["ID", "PAYLOAD"], rows: [[1, "value"]], affected_rows: 0, execution_time_ms: 1 }]);
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("oracle-1", "ORCL", "Query");
await store.executeTabSql(tabId, "SELECT * FROM DOCUMENTS");
expect(executeMulti).toHaveBeenCalledOnce();
expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview");
});
it("does not request deferred Oracle LOB handling for ordinary non-LOB queries", async () => {
getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 });
getColumns.mockResolvedValue([
{ name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: true, extra: null },
{ name: "NAME", data_type: "VARCHAR2(100)", is_nullable: true, column_default: null, is_primary_key: false, extra: null },
]);
lookupLocalCompletionTables.mockReturnValue([{ name: "CUSTOMERS", type: "table", schema: "APP" }]);
analyzeEditableQueryEditability.mockImplementation(async (sql: string) => ({
editable: true,
analysis: {
schema: "APP",
tableName: "CUSTOMERS",
selectStar: false,
columns: [{ sourceName: "NAME", resultName: "NAME", expression: "NAME" }, ...(sql.includes("__DBX_PK_0") ? [{ sourceName: "ID", resultName: "__DBX_PK_0", expression: '"ID"' }] : [])],
},
}));
executeMulti.mockResolvedValue([{ columns: ["NAME", "__DBX_PK_0"], rows: [["Alice", 1]], affected_rows: 0, execution_time_ms: 1 }]);
const { useQueryStore } = await import("@/stores/queryStore");
const store = useQueryStore();
const tabId = store.createTab("oracle-1", "ORCL", "Query");
await store.executeTabSql(tabId, "SELECT NAME FROM APP.CUSTOMERS");
expect(executeMulti).toHaveBeenCalledOnce();
expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview");
});
it.each([ it.each([
["only a foreign owner is cached", [{ name: "CUSTOMERS", type: "table", schema: "APP" }]], ["only a foreign owner is cached", [{ name: "CUSTOMERS", type: "table", schema: "APP" }]],
[ [
+22 -5
View File
@@ -80,6 +80,7 @@ import type { SqlExecutionTargetContext } from "@/lib/database/sqlExecutionTarge
import type { MultiDbExecutionTarget, MultiDbResultRunExecution } from "@/types/sqlExecution"; import type { MultiDbExecutionTarget, MultiDbResultRunExecution } from "@/types/sqlExecution";
const ORACLE_LIKE_METADATA_TYPES = new Set<string>(["oracle", "dameng", "oceanbase-oracle"]); const ORACLE_LIKE_METADATA_TYPES = new Set<string>(["oracle", "dameng", "oceanbase-oracle"]);
const ORACLE_DEFERRED_LOB_TYPES = new Set<string>(["CLOB", "NCLOB", "BLOB", "BFILE"]);
const UPPERCASE_FOLDED_METADATA_TYPES = new Set<string>([...ORACLE_LIKE_METADATA_TYPES, "saphana"]); const UPPERCASE_FOLDED_METADATA_TYPES = new Set<string>([...ORACLE_LIKE_METADATA_TYPES, "saphana"]);
const HIDDEN_QUERY_KEY_DATABASE_TYPES = new Set<DatabaseType>(["mysql", "postgres", "sqlserver", "oracle"]); const HIDDEN_QUERY_KEY_DATABASE_TYPES = new Set<DatabaseType>(["mysql", "postgres", "sqlserver", "oracle"]);
const QUERY_RESULT_EXPORT_UNSUPPORTED_ERROR = "Streaming export is unsupported for this query. Simplify it or use a supported driver."; const QUERY_RESULT_EXPORT_UNSUPPORTED_ERROR = "Streaming export is unsupported for this query. Simplify it or use a supported driver.";
@@ -530,6 +531,17 @@ function projectsAllColumnsForSource(analysis: EditableQueryInfo, sourceKey: str
return analysis.selectStar || analysis.columns.some((column) => column.star && (!column.sourceKey || column.sourceKey === sourceKey)); return analysis.selectStar || analysis.columns.some((column) => column.star && (!column.sourceKey || column.sourceKey === sourceKey));
} }
function oracleQueryProjectsDeferredLob(analysis: EditableQueryInfo, sourceKey: string, columns: readonly { name: string; data_type: string }[]): boolean {
const deferredColumns = new Set(columns.filter((column) => ORACLE_DEFERRED_LOB_TYPES.has(column.data_type.trim().toUpperCase())).map((column) => column.name.toLowerCase()));
if (deferredColumns.size === 0) return false;
if (projectsAllColumnsForSource(analysis, sourceKey)) return true;
return analysis.columns.some((column) => column.sourceName && column.sourceKey === sourceKey && deferredColumns.has(column.sourceName.toLowerCase()));
}
function oracleColumnsAllowDeferredLobMarkers(columns: readonly { name: string }[]): boolean {
return !columns.some((column) => column.name.toUpperCase().startsWith("__DBX_LARGE_VALUE_BYTES_"));
}
function cloneAnalysisForSource(analysis: EditableQueryInfo, source: EditableQuerySource): EditableQueryInfo { function cloneAnalysisForSource(analysis: EditableQueryInfo, source: EditableQuerySource): EditableQueryInfo {
return { return {
...analysis, ...analysis,
@@ -3395,6 +3407,7 @@ export const useQueryStore = defineStore("query", () => {
sql: string; sql: string;
metadataSql: string; metadataSql: string;
hiddenPrimaryKeys: HiddenPrimaryKeyProjection[]; hiddenPrimaryKeys: HiddenPrimaryKeyProjection[];
oracleLobPreview: boolean;
} }
function applyQueryMetadataPatch(tab: QueryTab, patch: QueryMetadataPatch) { function applyQueryMetadataPatch(tab: QueryTab, patch: QueryMetadataPatch) {
@@ -3523,9 +3536,10 @@ export const useQueryStore = defineStore("query", () => {
return indexes.find((index) => !index.filter && index.columns.length > 0 && index.is_primary); return indexes.find((index) => !index.filter && index.columns.length > 0 && index.is_primary);
} }
function buildHiddenPrimaryKeyPreparation(sql: string, databaseType: DatabaseType, loaded: LoadedEditableSource, primaryKeys: string[], declaredPrimaryKeys: string[], traceId: string, elapsed: () => string): EditableQueryExecutionPreparation { function buildHiddenPrimaryKeyPreparation(tab: QueryTab, sql: string, databaseType: DatabaseType, loaded: LoadedEditableSource, primaryKeys: string[], declaredPrimaryKeys: string[], traceId: string, elapsed: () => string): EditableQueryExecutionPreparation {
const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [] };
const metadataAnalysis = expandStarProjectionColumnsForSource(bindColumnsForSource(databaseType, loaded.analysis, loaded.source, loaded.tableMeta.columns), loaded.source, loaded.tableMeta.columns); const metadataAnalysis = expandStarProjectionColumnsForSource(bindColumnsForSource(databaseType, loaded.analysis, loaded.source, loaded.tableMeta.columns), loaded.source, loaded.tableMeta.columns);
const oracleLobPreview = databaseType === "oracle" && primaryKeys.length > 0 && oracleRowIdIsSafeForQuery(tab, loaded) && oracleColumnsAllowDeferredLobMarkers(loaded.tableMeta.columns) && oracleQueryProjectsDeferredLob(metadataAnalysis, loaded.source.key, loaded.tableMeta.columns);
const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [], oracleLobPreview };
const missingPrimaryKeys = declaredPrimaryKeys.length === 0 ? primaryKeys : missingPrimaryKeysForSource(primaryKeys, metadataAnalysis, loaded.source.key); const missingPrimaryKeys = declaredPrimaryKeys.length === 0 ? primaryKeys : missingPrimaryKeysForSource(primaryKeys, metadataAnalysis, loaded.source.key);
if (missingPrimaryKeys.length === 0) return unchanged; if (missingPrimaryKeys.length === 0) return unchanged;
const primaryKeySet = new Set(primaryKeys); const primaryKeySet = new Set(primaryKeys);
@@ -3546,11 +3560,11 @@ export const useQueryStore = defineStore("query", () => {
keyCount: rewritten.projections.length, keyCount: rewritten.projections.length,
elapsed: elapsed(), elapsed: elapsed(),
}); });
return { sql: rewritten.sql, metadataSql: rewritten.sql, hiddenPrimaryKeys: rewritten.projections }; return { sql: rewritten.sql, metadataSql: rewritten.sql, hiddenPrimaryKeys: rewritten.projections, oracleLobPreview };
} }
async function prepareEditableQueryExecution(tab: QueryTab, sql: string, conn: ConnectionConfig | undefined, databaseType: DatabaseType | undefined, executionDatabase: string, traceId: string, elapsed: () => string): Promise<EditableQueryExecutionPreparation> { async function prepareEditableQueryExecution(tab: QueryTab, sql: string, conn: ConnectionConfig | undefined, databaseType: DatabaseType | undefined, executionDatabase: string, traceId: string, elapsed: () => string): Promise<EditableQueryExecutionPreparation> {
const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [] }; const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [], oracleLobPreview: false };
if (!databaseType || !HIDDEN_QUERY_KEY_DATABASE_TYPES.has(databaseType) || !tab.connectionId) return unchanged; if (!databaseType || !HIDDEN_QUERY_KEY_DATABASE_TYPES.has(databaseType) || !tab.connectionId) return unchanged;
try { try {
@@ -3591,7 +3605,7 @@ export const useQueryStore = defineStore("query", () => {
// ROWID from a view can fail with ORA-01445. // ROWID from a view can fail with ORA-01445.
if (databaseType === "oracle" && declaredPrimaryKeys.length === 0 && !oracleRowIdIsSafeForQuery(tab, loaded)) return unchanged; if (databaseType === "oracle" && declaredPrimaryKeys.length === 0 && !oracleRowIdIsSafeForQuery(tab, loaded)) return unchanged;
const primaryKeys = editablePrimaryKeys(databaseType, loaded.tableMeta.columns, loaded.tableMeta.tableType); const primaryKeys = editablePrimaryKeys(databaseType, loaded.tableMeta.columns, loaded.tableMeta.tableType);
return buildHiddenPrimaryKeyPreparation(sql, databaseType, loaded, primaryKeys, declaredPrimaryKeys, traceId, elapsed); return buildHiddenPrimaryKeyPreparation(tab, sql, databaseType, loaded, primaryKeys, declaredPrimaryKeys, traceId, elapsed);
} catch (error) { } catch (error) {
// Metadata enrichment is optional. Query execution must retain its prior // Metadata enrichment is optional. Query execution must retain its prior
// behavior when metadata is unavailable or the SQL cannot be rewritten. // behavior when metadata is unavailable or the SQL cannot be rewritten.
@@ -3958,6 +3972,7 @@ export const useQueryStore = defineStore("query", () => {
let resultSortedSql = options?.resultSortedSql; let resultSortedSql = options?.resultSortedSql;
let queryMetadataSql = queryBaseSql; let queryMetadataSql = queryBaseSql;
let hiddenPrimaryKeys: HiddenPrimaryKeyProjection[] = []; let hiddenPrimaryKeys: HiddenPrimaryKeyProjection[] = [];
let useOracleLobPreview = false;
let pageSql: string | undefined; let pageSql: string | undefined;
let pageLimit: number | undefined; let pageLimit: number | undefined;
let pageOffset: number | undefined; let pageOffset: number | undefined;
@@ -4528,6 +4543,7 @@ export const useQueryStore = defineStore("query", () => {
// does not turn an otherwise editable result into a complex read-only one. // does not turn an otherwise editable result into a complex read-only one.
queryMetadataSql = options?.resultSortedSql && !options?.querySort ? queryBaseSql : prepared.metadataSql; queryMetadataSql = options?.resultSortedSql && !options?.querySort ? queryBaseSql : prepared.metadataSql;
hiddenPrimaryKeys = prepared.hiddenPrimaryKeys; hiddenPrimaryKeys = prepared.hiddenPrimaryKeys;
useOracleLobPreview = prepared.oracleLobPreview;
if (options?.querySort) { if (options?.querySort) {
const sorted = await api.buildSortedQuerySql({ const sorted = await api.buildSortedQuerySql({
originalSql: sqlToExecute, originalSql: sqlToExecute,
@@ -4627,6 +4643,7 @@ export const useQueryStore = defineStore("query", () => {
tableDataPreview: useTableDataPreview, tableDataPreview: useTableDataPreview,
} }
: {}), : {}),
...(useOracleLobPreview ? { tableDataPreview: true } : {}),
timeoutSecs: queryTimeoutSecs, timeoutSecs: queryTimeoutSecs,
catalog: executionCatalog, catalog: executionCatalog,
continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch, continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch,
+57 -1
View File
@@ -325,6 +325,7 @@ enum ServerLargeValuePreviewKind {
Text, Text,
Binary, Binary,
Vector, Vector,
Deferred,
} }
#[derive(Clone, Copy)] #[derive(Clone, Copy)]
@@ -350,6 +351,10 @@ fn server_large_value_alias(
"J" => (ServerLargeValuePreviewKind::Text, Some("json")), "J" => (ServerLargeValuePreviewKind::Text, Some("json")),
"K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")), "K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")),
"S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")), "S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")),
"C" => (ServerLargeValuePreviewKind::Deferred, Some("clob")),
"N" => (ServerLargeValuePreviewKind::Deferred, Some("nclob")),
"L" => (ServerLargeValuePreviewKind::Deferred, Some("blob")),
"F" => (ServerLargeValuePreviewKind::Deferred, Some("bfile")),
_ => return None, _ => return None,
}; };
Some((source_index, Some(preview_kind), source_type)) Some((source_index, Some(preview_kind), source_type))
@@ -361,6 +366,7 @@ fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeVa
"T" => ServerLargeValuePreviewKind::Text, "T" => ServerLargeValuePreviewKind::Text,
"B" => ServerLargeValuePreviewKind::Binary, "B" => ServerLargeValuePreviewKind::Binary,
"V" => ServerLargeValuePreviewKind::Vector, "V" => ServerLargeValuePreviewKind::Vector,
"D" => ServerLargeValuePreviewKind::Deferred,
_ => return None, _ => return None,
}; };
Some((kind, preview_size.parse::<usize>().ok()?.max(1))) Some((kind, preview_size.parse::<usize>().ok()?.max(1)))
@@ -374,6 +380,9 @@ fn truncate_server_large_value_preview(
let serde_json::Value::String(text) = value else { let serde_json::Value::String(text) = value else {
return false; return false;
}; };
if matches!(kind, ServerLargeValuePreviewKind::Deferred) {
return true;
}
if matches!(kind, ServerLargeValuePreviewKind::Vector) { if matches!(kind, ServerLargeValuePreviewKind::Vector) {
let truncated = text.chars().count() > preview_size; let truncated = text.chars().count() > preview_size;
let vector_text = if truncated { let vector_text = if truncated {
@@ -397,7 +406,7 @@ fn truncate_server_large_value_preview(
.strip_prefix("0x") .strip_prefix("0x")
.filter(|hex| hex.len() > preview_size.saturating_mul(2)) .filter(|hex| hex.len() > preview_size.saturating_mul(2))
.map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))), .map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))),
ServerLargeValuePreviewKind::Vector => unreachable!(), ServerLargeValuePreviewKind::Vector | ServerLargeValuePreviewKind::Deferred => unreachable!(),
}; };
let Some(truncate_at) = truncate_at else { let Some(truncate_at) = truncate_at else {
return false; return false;
@@ -976,6 +985,7 @@ pub fn agent_execute_query_params(
let mut params = serde_json::json!({ let mut params = serde_json::json!({
"sql": sql, "sql": sql,
"maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)), "maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)),
"deferLobs": options.table_data_preview,
}); });
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) { if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
params["database"] = serde_json::json!(database); params["database"] = serde_json::json!(database);
@@ -1005,6 +1015,7 @@ pub fn agent_execute_query_page_params(
"sql": sql, "sql": sql,
"pageSize": agent_protocol_row_count(options.page_size.unwrap_or(MAX_ROWS)), "pageSize": agent_protocol_row_count(options.page_size.unwrap_or(MAX_ROWS)),
"maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)), "maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)),
"deferLobs": options.table_data_preview,
}); });
if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) { if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) {
params["database"] = serde_json::json!(database); params["database"] = serde_json::json!(database);
@@ -6921,6 +6932,7 @@ for line in sys.stdin:
fetch_size: Some(250), fetch_size: Some(250),
row_offset: Some(100), row_offset: Some(100),
timeout_secs: Some(600), timeout_secs: Some(600),
table_data_preview: true,
..Default::default() ..Default::default()
}, },
); );
@@ -6932,6 +6944,7 @@ for line in sys.stdin:
assert_eq!(params["fetchSize"], 250); assert_eq!(params["fetchSize"], 250);
assert_eq!(params["rowOffset"], 100); assert_eq!(params["rowOffset"], 100);
assert_eq!(params["timeoutSecs"], 600); assert_eq!(params["timeoutSecs"], 600);
assert_eq!(params["deferLobs"], true);
} }
#[test] #[test]
@@ -7101,6 +7114,7 @@ for line in sys.stdin:
fetch_size: Some(250), fetch_size: Some(250),
row_offset: Some(100), row_offset: Some(100),
timeout_secs: Some(600), timeout_secs: Some(600),
table_data_preview: true,
..Default::default() ..Default::default()
}, },
); );
@@ -7113,6 +7127,7 @@ for line in sys.stdin:
assert_eq!(params["rowOffset"], 100); assert_eq!(params["rowOffset"], 100);
assert_eq!(params["timeoutSecs"], 600); assert_eq!(params["timeoutSecs"], 600);
assert_eq!(params["maxRows"], MAX_ROWS); assert_eq!(params["maxRows"], MAX_ROWS);
assert_eq!(params["deferLobs"], true);
} }
#[test] #[test]
@@ -7355,6 +7370,47 @@ for line in sys.stdin:
assert_eq!(cells.len(), 1); assert_eq!(cells.len(), 1);
} }
#[test]
fn extracts_deferred_oracle_clob_markers_without_changing_placeholder() {
let mut result = db::QueryResult {
columns: vec![
"id".to_string(),
"payload".to_string(),
format!("{}C_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX),
],
column_types: vec!["number".to_string(), "varchar2".to_string(), "varchar2".to_string()],
column_sortables: vec![true; 3],
spatial_columns: Vec::new(),
spatial_values: Vec::new(),
rows: vec![
vec![serde_json::json!(1), serde_json::json!("<CLOB>"), serde_json::json!("D:1")],
vec![serde_json::json!(2), serde_json::Value::Null, serde_json::Value::Null],
],
affected_rows: 0,
execution_time_ms: 0,
truncated: false,
session_id: None,
has_more: false,
elasticsearch_raw_body: None,
messages: Vec::new(),
};
let cells = extract_server_large_value_markers(&mut result);
assert_eq!(result.columns, vec!["id", "payload"]);
assert_eq!(result.column_types, vec!["number", "clob"]);
assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("<CLOB>")]);
assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::Value::Null]);
assert_eq!(
cells,
vec![db::LargeValueCell {
row_index: 0,
column_index: 1,
original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES,
}]
);
}
#[test] #[test]
fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() { fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() {
let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX); let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);