diff --git a/agents/drivers/oracle-go/main.go b/agents/drivers/oracle-go/main.go index df314acc7d..b133c54350 100644 --- a/agents/drivers/oracle-go/main.go +++ b/agents/drivers/oracle-go/main.go @@ -261,8 +261,11 @@ type queryOptions struct { MaxRows int `json:"maxRows"` FetchSize int `json:"fetchSize"` TimeoutSecs int `json:"timeoutSecs"` + DeferLOBs bool `json:"deferLobs"` } +const largeValueBytesColumnPrefix = "__DBX_LARGE_VALUE_BYTES_" + type queryResult struct { Columns []string `json:"columns"` ColumnTypes []string `json:"column_types"` @@ -3354,7 +3357,7 @@ func (s *server) executeQueryPage(opts queryOptions, pageSize int) (queryPageRes HasMore: false, }, err } - rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, opts.TimeoutSecs) + rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, opts.DeferLOBs) if err != nil { return queryPageResult{}, err } @@ -3420,7 +3423,7 @@ func (s *server) startTableRead(opts queryOptions, pageSize int) (queryPageResul if !isQuerySQL(sqlText) { return queryPageResult{}, errors.New("table read requires a SELECT query") } - rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, opts.TimeoutSecs) + rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, opts.TimeoutSecs, opts.DeferLOBs) if err != nil { return queryPageResult{}, err } @@ -3555,7 +3558,7 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) { maxRows = defaultMaxRows } if isQuerySQL(sqlText) { - result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs) + result, err := s.executeSelect(sqlText, maxRows, opts.TimeoutSecs, opts.DeferLOBs) result.ExecutionTimeMS = time.Since(start).Milliseconds() return result, err } @@ -3598,11 +3601,11 @@ func (s *server) executeQuery(opts queryOptions) (queryResult, error) { return queryResult{Columns: []string{}, ColumnTypes: []string{}, Rows: [][]any{}, AffectedRows: affected, ExecutionTimeMS: time.Since(start).Milliseconds()}, nil } -func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int) (queryResult, error) { +func (s *server) executeSelect(sqlText string, maxRows int, timeoutSecs int, deferLOBs bool) (queryResult, error) { return executeOracleSelectWithXMLTypeRetry( sqlText, func(query string) (queryResult, error) { - return s.executeSelectOnce(query, maxRows, timeoutSecs) + return s.executeSelectOnce(query, maxRows, timeoutSecs, deferLOBs) }, s.rewriteXMLTypeSelectSQL, ) @@ -3633,8 +3636,8 @@ func shouldRetryOracleXMLTypeRewrite(err error) bool { strings.Contains(message, "TTC error: received code ") } -func (s *server) executeSelectOnce(sqlText string, maxRows int, timeoutSecs int) (queryResult, error) { - rows, err := s.queryRowsWithXMLTypeRewriteIfNeeded(sqlText, timeoutSecs) +func (s *server) executeSelectOnce(sqlText string, maxRows int, timeoutSecs int, deferLOBs bool) (queryResult, error) { + rows, err := s.queryRowsWithOracleValueRewriteIfNeeded(sqlText, timeoutSecs, deferLOBs) if err != nil { return queryResult{}, err } @@ -3693,15 +3696,22 @@ func columnTypeNames(rows *sql.Rows) []string { return result } -func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string, timeoutSecs int) (*sql.Rows, error) { +func (s *server) queryRowsWithOracleValueRewriteIfNeeded(sqlText string, timeoutSecs int, deferLOBs bool) (*sql.Rows, error) { + if deferLOBs { + rewritten, err := rewriteOracleSelectSQL(sqlText, s.loadOracleColumnMeta, true) + if err == nil && rewritten != sqlText { + return s.queryRowsWithTimeout(rewritten, nil, timeoutSecs) + } + } rows, err := s.queryRowsWithTimeout(sqlText, nil, timeoutSecs) if err != nil { return nil, err } - if !rowsContainOracleXMLType(rows) { + typeNames := columnTypeNames(rows) + if !oracleColumnTypeNamesContainXMLType(typeNames) { return rows, nil } - rewritten, err := s.rewriteXMLTypeSelectSQL(sqlText) + rewritten, err := rewriteOracleSelectSQL(sqlText, s.loadOracleColumnMeta, false) if err != nil { s.closeRows(rows) return nil, err @@ -3709,24 +3719,12 @@ func (s *server) queryRowsWithXMLTypeRewriteIfNeeded(sqlText string, timeoutSecs if rewritten == sqlText { return rows, nil } - // Only pay the ALL_TAB_COLUMNS rewrite cost when the result metadata shows - // XMLTYPE. Ordinary Oracle queries should not run dictionary probes first. + // XMLTYPE keeps its metadata-triggered fallback so ordinary non-preview + // queries do not run dictionary probes. s.closeRows(rows) return s.queryRowsWithTimeout(rewritten, nil, timeoutSecs) } -func rowsContainOracleXMLType(rows *sql.Rows) bool { - types, err := rows.ColumnTypes() - if err != nil { - return false - } - typeNames := make([]string, 0, len(types)) - for _, columnType := range types { - typeNames = append(typeNames, columnType.DatabaseTypeName()) - } - return oracleColumnTypeNamesContainXMLType(typeNames) -} - func oracleColumnTypeNamesContainXMLType(typeNames []string) bool { for _, typeName := range typeNames { if isOracleXMLType(typeName) { @@ -3741,22 +3739,84 @@ func (s *server) rewriteXMLTypeSelectSQL(sqlText string) (string, error) { } func rewriteOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader) (string, error) { - rewritten, _, err := rewriteOracleXMLTypeSelectSQLDepth(sqlText, loadColumns, 0) + return rewriteOracleSelectSQL(sqlText, loadColumns, false) +} + +func rewriteOracleSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool) (string, error) { + rewritten, _, err := rewriteOracleSelectSQLDepth(sqlText, loadColumns, deferLOBs, 0) return rewritten, err } -func rewriteOracleXMLTypeSelectSQLDepth(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) { +func rewriteOracleSelectSQLDepth(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool, depth int) (string, bool, error) { if depth > 8 { return sqlText, false, nil } - if rewritten, changed, handled, err := rewriteDirectOracleXMLTypeSelectSQL(sqlText, loadColumns); handled || err != nil { + if rewritten, changed, handled, err := rewriteDirectOracleSelectSQL(sqlText, loadColumns, deferLOBs); handled || err != nil { return rewritten, changed, err } - rewritten, changed, err := rewriteNestedOracleSelects(sqlText, loadColumns, depth) + if deferLOBs { + return rewriteOracleFullPassthroughInnerSelect(sqlText, loadColumns, depth) + } + rewritten, changed, err := rewriteNestedOracleSelects(sqlText, loadColumns, deferLOBs, depth) return rewritten, changed, err } -func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) { +func rewriteOracleFullPassthroughInnerSelect(sqlText string, loadColumns oracleColumnMetaLoader, depth int) (string, bool, error) { + innerStart, innerEnd, ok := oracleFullPassthroughInnerSelectRange(sqlText) + if !ok { + return sqlText, false, nil + } + inner := sqlText[innerStart:innerEnd] + rewritten, changed, err := rewriteOracleSelectSQLDepth(inner, loadColumns, true, depth+1) + if err != nil || !changed { + return sqlText, false, err + } + return sqlText[:innerStart] + rewritten + sqlText[innerEnd:], true, nil +} + +func oracleFullPassthroughInnerSelectRange(sqlText string) (int, int, bool) { + selectStart := leadingSQLSelectListStart(sqlText) + if selectStart < 0 { + return 0, 0, false + } + fromIdx := findTopLevelSQLKeyword(sqlText, selectStart, "from") + if fromIdx < 0 { + return 0, 0, false + } + prefix, selectList := splitOracleSelectListModifier(sqlText[selectStart:fromIdx]) + if strings.TrimSpace(prefix) != "" { + return 0, 0, false + } + items := splitTopLevelSQLList(selectList) + if len(items) != 1 { + return 0, 0, false + } + if _, ok := parseOracleStarSelectItem(items[0]); !ok { + return 0, 0, false + } + open := skipSQLWhitespace(sqlText, fromIdx+len("from")) + if open >= len(sqlText) || sqlText[open] != '(' { + return 0, 0, false + } + close := findMatchingSQLParen(sqlText, open) + if close < 0 || !startsWithSQLKeyword(trimLeadingSQLComments(sqlText[open+1:close]), "select") { + return 0, 0, false + } + pos := skipSQLWhitespace(sqlText, close+1) + if pos < len(sqlText) && sqlText[pos] != ';' && !nextKeywordIsOracleClause(sqlText[pos:]) { + _, afterAlias, aliasOK := readOracleIdentifierToken(sqlText, pos) + if !aliasOK { + return 0, 0, false + } + pos = skipSQLWhitespace(sqlText, afterAlias) + } + if pos < len(sqlText) && sqlText[pos] != ';' && !nextKeywordIsOracleClause(sqlText[pos:]) { + return 0, 0, false + } + return open + 1, close, true +} + +func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool, depth int) (string, bool, error) { var builder strings.Builder changed := false last := 0 @@ -3781,7 +3841,7 @@ func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoad } inner := sqlText[pos+1 : end] if startsWithSQLKeyword(trimLeadingSQLComments(inner), "select") { - rewrittenInner, innerChanged, err := rewriteOracleXMLTypeSelectSQLDepth(inner, loadColumns, depth+1) + rewrittenInner, innerChanged, err := rewriteOracleSelectSQLDepth(inner, loadColumns, deferLOBs, depth+1) if err != nil { return "", false, err } @@ -3802,7 +3862,7 @@ func rewriteNestedOracleSelects(sqlText string, loadColumns oracleColumnMetaLoad return builder.String(), true, nil } -func rewriteDirectOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader) (string, bool, bool, error) { +func rewriteDirectOracleSelectSQL(sqlText string, loadColumns oracleColumnMetaLoader, deferLOBs bool) (string, bool, bool, error) { selectStart := leadingSQLSelectListStart(sqlText) if selectStart < 0 { return sqlText, false, false, nil @@ -3811,23 +3871,30 @@ func rewriteDirectOracleXMLTypeSelectSQL(sqlText string, loadColumns oracleColum if fromIdx < 0 { return sqlText, false, false, nil } + if deferLOBs && oracleSQLHasTopLevelSetOperator(sqlText, fromIdx+len("from")) { + return sqlText, false, false, nil + } selectListPrefix, selectList := splitOracleSelectListModifier(sqlText[selectStart:fromIdx]) + deferSelectLOBs := deferLOBs && !startsWithSQLKeyword(strings.TrimSpace(selectListPrefix), "distinct") tableRef, ok := parseSingleOracleTableRef(sqlText[fromIdx+len("from"):]) if !ok { return sqlText, false, false, nil } items := splitTopLevelSQLList(selectList) - if len(items) == 0 || !oracleSelectListMayReferenceXMLType(items) { + if len(items) == 0 || !oracleSelectListMayReferenceTableColumns(items) { return sqlText, false, true, nil } columns, err := loadColumns(tableRef.Schema, tableRef.Table) if err != nil { return "", false, true, err } - if !oracleColumnsHaveXMLType(columns) { + if deferSelectLOBs && oracleColumnsConflictWithLargeValueMarkers(columns) { + deferSelectLOBs = false + } + if !oracleColumnsNeedValueRewrite(columns, deferSelectLOBs) { return sqlText, false, true, nil } - rewrittenItems, changed := rewriteOracleSelectItemsForXMLType(items, columns, tableRef) + rewrittenItems, changed := rewriteOracleSelectItems(items, columns, tableRef, deferSelectLOBs) if !changed { return sqlText, false, true, nil } @@ -3922,7 +3989,16 @@ func splitOracleSelectListModifier(selectList string) (string, string) { return selectList[:prefixLen], selectList[prefixLen:] } -func oracleSelectListMayReferenceXMLType(items []string) bool { +func oracleSQLHasTopLevelSetOperator(sqlText string, start int) bool { + for _, keyword := range []string{"union", "minus", "intersect"} { + if findTopLevelSQLKeyword(sqlText, start, keyword) >= 0 { + return true + } + } + return false +} + +func oracleSelectListMayReferenceTableColumns(items []string) bool { for _, item := range items { if _, ok := parseOracleStarSelectItem(item); ok { return true @@ -3934,49 +4010,89 @@ func oracleSelectListMayReferenceXMLType(items []string) bool { return false } -func rewriteOracleSelectItemsForXMLType(items []string, columns []oracleColumnMeta, tableRef oracleTableRef) ([]string, bool) { - xmlColumns := map[string]oracleColumnMeta{} +func rewriteOracleSelectItems(items []string, columns []oracleColumnMeta, tableRef oracleTableRef, deferLOBs bool) ([]string, bool) { + columnsByName := map[string]oracleColumnMeta{} for _, column := range columns { - if isOracleXMLType(column.DataType) { - xmlColumns[oracleIdentifierKey(column.Name)] = column - } + columnsByName[oracleIdentifierKey(column.Name)] = column } rewritten := make([]string, 0, len(items)) changed := false + sourceIndex := 0 for _, item := range items { if qualifier, ok := parseOracleStarSelectItem(item); ok && oracleQualifierMatchesTable(qualifier, tableRef) { for _, column := range columns { - rewritten = append(rewritten, oracleSelectExpressionForColumn(column, tableRef, xmlColumns)) + columnRef := oracleColumnRef(tableRef.AliasText, column.Name) + outputAlias := quoteIdentifier(column.Name) + if isOracleXMLType(column.DataType) { + rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias)) + } else if deferLOBs { + if expressions, ok := oracleDeferredLOBExpressions(columnRef, outputAlias, sourceIndex, column.DataType); ok { + rewritten = append(rewritten, expressions...) + } else { + rewritten = append(rewritten, columnRef) + } + } else { + rewritten = append(rewritten, columnRef) + } + sourceIndex++ } changed = true continue } qualifier, column, alias, ok := parseOracleColumnSelectItem(item) if ok && oracleQualifierMatchesTable(qualifier, tableRef) { - if meta, isXML := xmlColumns[oracleIdentifierKey(column.Name)]; isXML { + if meta, exists := columnsByName[oracleIdentifierKey(column.Name)]; exists { outputAlias := alias if outputAlias == "" { outputAlias = quoteIdentifier(meta.Name) } - rewritten = append(rewritten, oracleXMLSerializeExpression(oracleColumnRef(qualifier, meta.Name), outputAlias)) - changed = true - continue + columnRef := oracleColumnRef(qualifier, meta.Name) + if isOracleXMLType(meta.DataType) { + rewritten = append(rewritten, oracleXMLSerializeExpression(columnRef, outputAlias)) + changed = true + sourceIndex++ + continue + } + if deferLOBs { + if expressions, isLOB := oracleDeferredLOBExpressions(columnRef, outputAlias, sourceIndex, meta.DataType); isLOB { + rewritten = append(rewritten, expressions...) + changed = true + sourceIndex++ + continue + } + } } } rewritten = append(rewritten, item) + sourceIndex++ } return rewritten, changed } -func oracleSelectExpressionForColumn(column oracleColumnMeta, tableRef oracleTableRef, xmlColumns map[string]oracleColumnMeta) string { - qualifier := "" - if tableRef.AliasText != "" { - qualifier = tableRef.AliasText +func oracleDeferredLOBExpressions(columnRef, outputAlias string, sourceIndex int, dataType string) ([]string, bool) { + kind, placeholder, ok := oracleDeferredLOBKind(dataType) + if !ok { + return nil, false } - if _, isXML := xmlColumns[oracleIdentifierKey(column.Name)]; isXML { - return oracleXMLSerializeExpression(oracleColumnRef(qualifier, column.Name), quoteIdentifier(column.Name)) + valueExpression := fmt.Sprintf("CASE WHEN %s IS NULL THEN NULL ELSE '%s' END AS %s", columnRef, placeholder, outputAlias) + markerAlias := fmt.Sprintf("%s%s_%d", largeValueBytesColumnPrefix, kind, sourceIndex) + markerExpression := fmt.Sprintf("CASE WHEN %s IS NULL THEN NULL ELSE 'D:1' END AS %s", columnRef, quoteIdentifier(markerAlias)) + return []string{valueExpression, markerExpression}, true +} + +func oracleDeferredLOBKind(dataType string) (kind, placeholder string, ok bool) { + switch strings.ToUpper(strings.TrimSpace(dataType)) { + case "CLOB": + return "C", "", true + case "NCLOB": + return "N", "", true + case "BLOB": + return "L", "", true + case "BFILE": + return "F", "", true + default: + return "", "", false } - return oracleColumnRef(qualifier, column.Name) } func oracleXMLSerializeExpression(columnRef, alias string) string { @@ -4057,15 +4173,29 @@ func oracleQualifierMatchesTable(qualifier string, tableRef oracleTableRef) bool return key == oracleIdentifierKey(tableRef.Table) } -func oracleColumnsHaveXMLType(columns []oracleColumnMeta) bool { +func oracleColumnsNeedValueRewrite(columns []oracleColumnMeta, deferLOBs bool) bool { for _, column := range columns { - if isOracleXMLType(column.DataType) { + if isOracleXMLType(column.DataType) || (deferLOBs && isOracleDeferredLOBType(column.DataType)) { return true } } return false } +func oracleColumnsConflictWithLargeValueMarkers(columns []oracleColumnMeta) bool { + for _, column := range columns { + if strings.HasPrefix(strings.ToUpper(column.Name), largeValueBytesColumnPrefix) { + return true + } + } + return false +} + +func isOracleDeferredLOBType(dataType string) bool { + _, _, ok := oracleDeferredLOBKind(dataType) + return ok +} + func isOracleXMLType(dataType string) bool { normalized := strings.ToUpper(strings.TrimSpace(dataType)) return normalized == "XMLTYPE" || normalized == "SYS.XMLTYPE" @@ -4224,7 +4354,7 @@ func oracleIdentifierKey(value string) string { func oracleIdentifierIsClause(value string) bool { switch oracleIdentifierKey(value) { - case "WHERE", "GROUP", "ORDER", "HAVING", "CONNECT", "START", "MODEL", "FETCH", "OFFSET", "UNION", "MINUS", "INTERSECT": + case "WHERE", "GROUP", "ORDER", "HAVING", "CONNECT", "START", "MODEL", "FETCH", "OFFSET", "FOR", "AS", "UNION", "MINUS", "INTERSECT": return true default: return false diff --git a/agents/drivers/oracle-go/main_test.go b/agents/drivers/oracle-go/main_test.go index 998dd5b6ca..ab9d8c3560 100644 --- a/agents/drivers/oracle-go/main_test.go +++ b/agents/drivers/oracle-go/main_test.go @@ -2046,6 +2046,123 @@ func TestRewriteOracleXMLTypeSkipsJoins(t *testing.T) { } } +func TestRewriteOracleLOBSelectStarAsDeferredValues(t *testing.T) { + sqlText, err := rewriteOracleSelectSQL( + `SELECT t.* FROM TEST_LOBS t ORDER BY t.ID DESC`, + fakeOracleColumnLoader([]oracleColumnMeta{ + {Name: "ID", DataType: "NUMBER"}, + {Name: "PAYLOAD", DataType: "CLOB"}, + {Name: "NATIONAL_TEXT", DataType: "NCLOB"}, + {Name: "BINARY_DATA", DataType: "BLOB"}, + {Name: "FILE_DATA", DataType: "BFILE"}, + }), + true, + ) + if err != nil { + t.Fatal(err) + } + want := `SELECT t."ID", CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE '' END AS "PAYLOAD", CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_C_1", CASE WHEN t."NATIONAL_TEXT" IS NULL THEN NULL ELSE '' END AS "NATIONAL_TEXT", CASE WHEN t."NATIONAL_TEXT" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_N_2", CASE WHEN t."BINARY_DATA" IS NULL THEN NULL ELSE '' END AS "BINARY_DATA", CASE WHEN t."BINARY_DATA" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_L_3", CASE WHEN t."FILE_DATA" IS NULL THEN NULL ELSE '' END AS "FILE_DATA", CASE WHEN t."FILE_DATA" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_F_4" FROM TEST_LOBS t ORDER BY t.ID DESC` + if sqlText != want { + t.Fatalf("rewriteOracleSelectSQL() = %s, want %s", sqlText, want) + } +} + +func TestRewriteOracleLOBExplicitColumnUsesVisibleResultIndex(t *testing.T) { + sqlText, err := rewriteOracleSelectSQL( + `SELECT t.ID, t.PAYLOAD AS body, LENGTH(t.PAYLOAD) AS payload_length FROM TEST_LOBS t`, + fakeOracleColumnLoader([]oracleColumnMeta{ + {Name: "ID", DataType: "NUMBER"}, + {Name: "PAYLOAD", DataType: "CLOB"}, + }), + true, + ) + if err != nil { + t.Fatal(err) + } + want := `SELECT t.ID, CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE '' END AS body, CASE WHEN t."PAYLOAD" IS NULL THEN NULL ELSE 'D:1' END AS "__DBX_LARGE_VALUE_BYTES_C_1", LENGTH(t.PAYLOAD) AS payload_length FROM TEST_LOBS t` + if sqlText != want { + t.Fatalf("rewriteOracleSelectSQL() = %s, want %s", sqlText, want) + } +} + +func TestRewriteOracleLOBNestedRownumQuery(t *testing.T) { + sqlText, err := rewriteOracleSelectSQL( + `SELECT * FROM (SELECT ID, PAYLOAD FROM TEST_LOBS ORDER BY ID DESC) WHERE ROWNUM <= 10`, + fakeOracleColumnLoader([]oracleColumnMeta{ + {Name: "ID", DataType: "NUMBER"}, + {Name: "PAYLOAD", DataType: "CLOB"}, + }), + true, + ) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(sqlText, `CASE WHEN "PAYLOAD" IS NULL THEN NULL ELSE '' END AS "PAYLOAD"`) || + !strings.Contains(sqlText, `"__DBX_LARGE_VALUE_BYTES_C_1"`) { + t.Fatalf("expected nested CLOB column to be deferred, got: %s", sqlText) + } +} + +func TestRewriteOracleLOBSkipsDerivedProjectionThatDropsMarkers(t *testing.T) { + called := false + input := `SELECT PAYLOAD FROM (SELECT PAYLOAD FROM TEST_LOBS)` + sqlText, err := rewriteOracleSelectSQL( + input, + func(schema, table string) ([]oracleColumnMeta, error) { + called = true + return []oracleColumnMeta{{Name: "PAYLOAD", DataType: "CLOB"}}, nil + }, + true, + ) + if err != nil { + t.Fatal(err) + } + if called { + t.Fatal("derived projection should not load table metadata") + } + if sqlText != input { + t.Fatalf("derived projection should remain unchanged, got: %s", sqlText) + } +} + +func TestRewriteOracleLOBPreservesUnsafeQueries(t *testing.T) { + columns := []oracleColumnMeta{{Name: "ID", DataType: "NUMBER"}, {Name: "PAYLOAD", DataType: "CLOB"}} + tests := []string{ + `SELECT DISTINCT PAYLOAD FROM TEST_LOBS`, + `SELECT l.PAYLOAD FROM TEST_LOBS l JOIN OTHER_TABLE o ON o.ID = l.ID`, + `SELECT PAYLOAD FROM TEST_LOBS UNION ALL SELECT PAYLOAD FROM OTHER_TABLE`, + } + for _, input := range tests { + sqlText, err := rewriteOracleSelectSQL(input, fakeOracleColumnLoader(columns), true) + if err != nil { + t.Fatal(err) + } + if sqlText != input { + t.Fatalf("unsafe query should not be rewritten: %s", sqlText) + } + } +} + +func TestRewriteOracleLOBRequiresDeferredModeAndSafeMarkerNames(t *testing.T) { + input := `SELECT * FROM TEST_LOBS` + tests := []struct { + deferLOBs bool + columns []oracleColumnMeta + }{ + {deferLOBs: false, columns: []oracleColumnMeta{{Name: "ID", DataType: "NUMBER"}, {Name: "PAYLOAD", DataType: "CLOB"}}}, + {deferLOBs: true, columns: []oracleColumnMeta{{Name: "PAYLOAD", DataType: "CLOB"}, {Name: "__DBX_LARGE_VALUE_BYTES_C_0", DataType: "VARCHAR2"}}}, + } + for _, test := range tests { + sqlText, err := rewriteOracleSelectSQL(input, fakeOracleColumnLoader(test.columns), test.deferLOBs) + if err != nil { + t.Fatal(err) + } + if sqlText != input { + t.Fatalf("query should remain unchanged, got: %s", sqlText) + } + } +} + func TestOracleColumnTypeNamesContainXMLType(t *testing.T) { tests := []struct { name string diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index ccdcbb3d5b..cf94ab7c1e 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -4202,7 +4202,7 @@ async function resolveLargeValueCells(rowIds: number[], columnIndexes: number[]) } } if (requestsByColumn.size === 0) return resolved; - if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { + if ((resolvedDatabaseType.value !== "mysql" && resolvedDatabaseType.value !== "postgres" && resolvedDatabaseType.value !== "oracle") || !props.connectionId || !props.tableMeta?.tableName || props.tableMeta.primaryKeys.length === 0) { throw new Error(t("grid.largeValueNeedsStableKey")); } const primaryKeyIndexes = props.tableMeta.primaryKeys.map(largeValueSourceColumnIndex); diff --git a/apps/desktop/src/stores/__tests__/queryStore.hiddenPrimaryKey.spec.ts b/apps/desktop/src/stores/__tests__/queryStore.hiddenPrimaryKey.spec.ts index f577658341..6deb52a078 100644 --- a/apps/desktop/src/stores/__tests__/queryStore.hiddenPrimaryKey.spec.ts +++ b/apps/desktop/src/stores/__tests__/queryStore.hiddenPrimaryKey.spec.ts @@ -672,6 +672,119 @@ describe("queryStore hidden primary key editing", () => { expect(tab.result?.hidden_column_indexes).toBeUndefined(); }); + it("enables deferred Oracle LOBs only when a base-table query has a stable key", async () => { + getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 }); + getColumns.mockResolvedValue([ + { name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: true, extra: null }, + { name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null }, + ]); + lookupLocalCompletionTables.mockReturnValue([{ name: "DOCUMENTS", type: "table", schema: "APP" }]); + analyzeEditableQueryEditability.mockImplementation(async (sql: string) => ({ + editable: true, + analysis: { + schema: "APP", + tableName: "DOCUMENTS", + selectStar: false, + columns: [{ sourceName: "PAYLOAD", resultName: "PAYLOAD", expression: "PAYLOAD" }, ...(sql.includes("__DBX_PK_0") ? [{ sourceName: "ID", resultName: "__DBX_PK_0", expression: '"ID"' }] : [])], + }, + })); + executeMulti.mockResolvedValue([{ columns: ["PAYLOAD", "__DBX_PK_0"], rows: [["", 1]], affected_rows: 0, execution_time_ms: 1 }]); + + const { useQueryStore } = await import("@/stores/queryStore"); + const store = useQueryStore(); + const tabId = store.createTab("oracle-1", "ORCL", "Query"); + + await store.executeTabSql(tabId, "SELECT PAYLOAD FROM APP.DOCUMENTS"); + + expect(executeMulti).toHaveBeenCalledWith("oracle-1", "ORCL", 'SELECT PAYLOAD, "ID" AS "__DBX_PK_0" FROM APP.DOCUMENTS', undefined, expect.any(String), expect.objectContaining({ tableDataPreview: true, timeoutSecs: 30 })); + }); + + it("keeps deferred Oracle LOBs disabled for views", async () => { + getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 }); + getColumns.mockResolvedValue([ + { name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: false, extra: null }, + { name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null }, + ]); + listIndexes.mockResolvedValue([]); + lookupLocalCompletionTables.mockReturnValue([{ name: "DOCUMENT_VIEW", type: "view", schema: "APP" }]); + analyzeEditableQueryEditability.mockResolvedValue({ + editable: true, + analysis: { + schema: "APP", + tableName: "DOCUMENT_VIEW", + selectStar: true, + columns: [], + }, + }); + executeMulti.mockResolvedValue([{ columns: ["ID", "PAYLOAD"], rows: [[1, "value"]], affected_rows: 0, execution_time_ms: 1 }]); + + const { useQueryStore } = await import("@/stores/queryStore"); + const store = useQueryStore(); + const tabId = store.createTab("oracle-1", "ORCL", "Query"); + + await store.executeTabSql(tabId, "SELECT * FROM APP.DOCUMENT_VIEW"); + + expect(executeMulti).toHaveBeenCalledOnce(); + expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview"); + }); + + it("keeps deferred Oracle LOBs disabled when a keyless source has no safe ROWID path", async () => { + getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 }); + getColumns.mockResolvedValue([ + { name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: false, extra: null }, + { name: "PAYLOAD", data_type: "CLOB", is_nullable: true, column_default: null, is_primary_key: false, extra: null }, + ]); + listIndexes.mockResolvedValue([]); + lookupLocalCompletionTables.mockReturnValue([]); + analyzeEditableQueryEditability.mockResolvedValue({ + editable: true, + analysis: { + schema: undefined, + tableName: "DOCUMENTS", + selectStar: true, + columns: [], + }, + }); + executeMulti.mockResolvedValue([{ columns: ["ID", "PAYLOAD"], rows: [[1, "value"]], affected_rows: 0, execution_time_ms: 1 }]); + + const { useQueryStore } = await import("@/stores/queryStore"); + const store = useQueryStore(); + const tabId = store.createTab("oracle-1", "ORCL", "Query"); + + await store.executeTabSql(tabId, "SELECT * FROM DOCUMENTS"); + + expect(executeMulti).toHaveBeenCalledOnce(); + expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview"); + }); + + it("does not request deferred Oracle LOB handling for ordinary non-LOB queries", async () => { + getConnectionConfig.mockReturnValue({ id: "oracle-1", name: "Oracle", db_type: "oracle", database: "ORCL", query_timeout_secs: 30 }); + getColumns.mockResolvedValue([ + { name: "ID", data_type: "NUMBER", is_nullable: false, column_default: null, is_primary_key: true, extra: null }, + { name: "NAME", data_type: "VARCHAR2(100)", is_nullable: true, column_default: null, is_primary_key: false, extra: null }, + ]); + lookupLocalCompletionTables.mockReturnValue([{ name: "CUSTOMERS", type: "table", schema: "APP" }]); + analyzeEditableQueryEditability.mockImplementation(async (sql: string) => ({ + editable: true, + analysis: { + schema: "APP", + tableName: "CUSTOMERS", + selectStar: false, + columns: [{ sourceName: "NAME", resultName: "NAME", expression: "NAME" }, ...(sql.includes("__DBX_PK_0") ? [{ sourceName: "ID", resultName: "__DBX_PK_0", expression: '"ID"' }] : [])], + }, + })); + executeMulti.mockResolvedValue([{ columns: ["NAME", "__DBX_PK_0"], rows: [["Alice", 1]], affected_rows: 0, execution_time_ms: 1 }]); + + const { useQueryStore } = await import("@/stores/queryStore"); + const store = useQueryStore(); + const tabId = store.createTab("oracle-1", "ORCL", "Query"); + + await store.executeTabSql(tabId, "SELECT NAME FROM APP.CUSTOMERS"); + + expect(executeMulti).toHaveBeenCalledOnce(); + expect(executeMulti.mock.calls[0]?.[5]).not.toHaveProperty("tableDataPreview"); + }); + it.each([ ["only a foreign owner is cached", [{ name: "CUSTOMERS", type: "table", schema: "APP" }]], [ diff --git a/apps/desktop/src/stores/queryStore.ts b/apps/desktop/src/stores/queryStore.ts index 3916e67167..ffb2bdf9a1 100644 --- a/apps/desktop/src/stores/queryStore.ts +++ b/apps/desktop/src/stores/queryStore.ts @@ -80,6 +80,7 @@ import type { SqlExecutionTargetContext } from "@/lib/database/sqlExecutionTarge import type { MultiDbExecutionTarget, MultiDbResultRunExecution } from "@/types/sqlExecution"; const ORACLE_LIKE_METADATA_TYPES = new Set(["oracle", "dameng", "oceanbase-oracle"]); +const ORACLE_DEFERRED_LOB_TYPES = new Set(["CLOB", "NCLOB", "BLOB", "BFILE"]); const UPPERCASE_FOLDED_METADATA_TYPES = new Set([...ORACLE_LIKE_METADATA_TYPES, "saphana"]); const HIDDEN_QUERY_KEY_DATABASE_TYPES = new Set(["mysql", "postgres", "sqlserver", "oracle"]); const QUERY_RESULT_EXPORT_UNSUPPORTED_ERROR = "Streaming export is unsupported for this query. Simplify it or use a supported driver."; @@ -530,6 +531,17 @@ function projectsAllColumnsForSource(analysis: EditableQueryInfo, sourceKey: str return analysis.selectStar || analysis.columns.some((column) => column.star && (!column.sourceKey || column.sourceKey === sourceKey)); } +function oracleQueryProjectsDeferredLob(analysis: EditableQueryInfo, sourceKey: string, columns: readonly { name: string; data_type: string }[]): boolean { + const deferredColumns = new Set(columns.filter((column) => ORACLE_DEFERRED_LOB_TYPES.has(column.data_type.trim().toUpperCase())).map((column) => column.name.toLowerCase())); + if (deferredColumns.size === 0) return false; + if (projectsAllColumnsForSource(analysis, sourceKey)) return true; + return analysis.columns.some((column) => column.sourceName && column.sourceKey === sourceKey && deferredColumns.has(column.sourceName.toLowerCase())); +} + +function oracleColumnsAllowDeferredLobMarkers(columns: readonly { name: string }[]): boolean { + return !columns.some((column) => column.name.toUpperCase().startsWith("__DBX_LARGE_VALUE_BYTES_")); +} + function cloneAnalysisForSource(analysis: EditableQueryInfo, source: EditableQuerySource): EditableQueryInfo { return { ...analysis, @@ -3395,6 +3407,7 @@ export const useQueryStore = defineStore("query", () => { sql: string; metadataSql: string; hiddenPrimaryKeys: HiddenPrimaryKeyProjection[]; + oracleLobPreview: boolean; } function applyQueryMetadataPatch(tab: QueryTab, patch: QueryMetadataPatch) { @@ -3523,9 +3536,10 @@ export const useQueryStore = defineStore("query", () => { return indexes.find((index) => !index.filter && index.columns.length > 0 && index.is_primary); } - function buildHiddenPrimaryKeyPreparation(sql: string, databaseType: DatabaseType, loaded: LoadedEditableSource, primaryKeys: string[], declaredPrimaryKeys: string[], traceId: string, elapsed: () => string): EditableQueryExecutionPreparation { - const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [] }; + function buildHiddenPrimaryKeyPreparation(tab: QueryTab, sql: string, databaseType: DatabaseType, loaded: LoadedEditableSource, primaryKeys: string[], declaredPrimaryKeys: string[], traceId: string, elapsed: () => string): EditableQueryExecutionPreparation { const metadataAnalysis = expandStarProjectionColumnsForSource(bindColumnsForSource(databaseType, loaded.analysis, loaded.source, loaded.tableMeta.columns), loaded.source, loaded.tableMeta.columns); + const oracleLobPreview = databaseType === "oracle" && primaryKeys.length > 0 && oracleRowIdIsSafeForQuery(tab, loaded) && oracleColumnsAllowDeferredLobMarkers(loaded.tableMeta.columns) && oracleQueryProjectsDeferredLob(metadataAnalysis, loaded.source.key, loaded.tableMeta.columns); + const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [], oracleLobPreview }; const missingPrimaryKeys = declaredPrimaryKeys.length === 0 ? primaryKeys : missingPrimaryKeysForSource(primaryKeys, metadataAnalysis, loaded.source.key); if (missingPrimaryKeys.length === 0) return unchanged; const primaryKeySet = new Set(primaryKeys); @@ -3546,11 +3560,11 @@ export const useQueryStore = defineStore("query", () => { keyCount: rewritten.projections.length, elapsed: elapsed(), }); - return { sql: rewritten.sql, metadataSql: rewritten.sql, hiddenPrimaryKeys: rewritten.projections }; + return { sql: rewritten.sql, metadataSql: rewritten.sql, hiddenPrimaryKeys: rewritten.projections, oracleLobPreview }; } async function prepareEditableQueryExecution(tab: QueryTab, sql: string, conn: ConnectionConfig | undefined, databaseType: DatabaseType | undefined, executionDatabase: string, traceId: string, elapsed: () => string): Promise { - const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [] }; + const unchanged = { sql, metadataSql: sql, hiddenPrimaryKeys: [], oracleLobPreview: false }; if (!databaseType || !HIDDEN_QUERY_KEY_DATABASE_TYPES.has(databaseType) || !tab.connectionId) return unchanged; try { @@ -3591,7 +3605,7 @@ export const useQueryStore = defineStore("query", () => { // ROWID from a view can fail with ORA-01445. if (databaseType === "oracle" && declaredPrimaryKeys.length === 0 && !oracleRowIdIsSafeForQuery(tab, loaded)) return unchanged; const primaryKeys = editablePrimaryKeys(databaseType, loaded.tableMeta.columns, loaded.tableMeta.tableType); - return buildHiddenPrimaryKeyPreparation(sql, databaseType, loaded, primaryKeys, declaredPrimaryKeys, traceId, elapsed); + return buildHiddenPrimaryKeyPreparation(tab, sql, databaseType, loaded, primaryKeys, declaredPrimaryKeys, traceId, elapsed); } catch (error) { // Metadata enrichment is optional. Query execution must retain its prior // behavior when metadata is unavailable or the SQL cannot be rewritten. @@ -3958,6 +3972,7 @@ export const useQueryStore = defineStore("query", () => { let resultSortedSql = options?.resultSortedSql; let queryMetadataSql = queryBaseSql; let hiddenPrimaryKeys: HiddenPrimaryKeyProjection[] = []; + let useOracleLobPreview = false; let pageSql: string | undefined; let pageLimit: number | undefined; let pageOffset: number | undefined; @@ -4528,6 +4543,7 @@ export const useQueryStore = defineStore("query", () => { // does not turn an otherwise editable result into a complex read-only one. queryMetadataSql = options?.resultSortedSql && !options?.querySort ? queryBaseSql : prepared.metadataSql; hiddenPrimaryKeys = prepared.hiddenPrimaryKeys; + useOracleLobPreview = prepared.oracleLobPreview; if (options?.querySort) { const sorted = await api.buildSortedQuerySql({ originalSql: sqlToExecute, @@ -4627,6 +4643,7 @@ export const useQueryStore = defineStore("query", () => { tableDataPreview: useTableDataPreview, } : {}), + ...(useOracleLobPreview ? { tableDataPreview: true } : {}), timeoutSecs: queryTimeoutSecs, catalog: executionCatalog, continueOnError: settingsStore.editorSettings.continueOnErrorOnBatch, diff --git a/crates/dbx-core/src/query.rs b/crates/dbx-core/src/query.rs index c03c5fd5bd..d1593740ad 100644 --- a/crates/dbx-core/src/query.rs +++ b/crates/dbx-core/src/query.rs @@ -325,6 +325,7 @@ enum ServerLargeValuePreviewKind { Text, Binary, Vector, + Deferred, } #[derive(Clone, Copy)] @@ -350,6 +351,10 @@ fn server_large_value_alias( "J" => (ServerLargeValuePreviewKind::Text, Some("json")), "K" => (ServerLargeValuePreviewKind::Text, Some("jsonb")), "S" => (ServerLargeValuePreviewKind::Text, Some("tsvector")), + "C" => (ServerLargeValuePreviewKind::Deferred, Some("clob")), + "N" => (ServerLargeValuePreviewKind::Deferred, Some("nclob")), + "L" => (ServerLargeValuePreviewKind::Deferred, Some("blob")), + "F" => (ServerLargeValuePreviewKind::Deferred, Some("bfile")), _ => return None, }; Some((source_index, Some(preview_kind), source_type)) @@ -361,6 +366,7 @@ fn server_large_value_marker(value: &serde_json::Value) -> Option<(ServerLargeVa "T" => ServerLargeValuePreviewKind::Text, "B" => ServerLargeValuePreviewKind::Binary, "V" => ServerLargeValuePreviewKind::Vector, + "D" => ServerLargeValuePreviewKind::Deferred, _ => return None, }; Some((kind, preview_size.parse::().ok()?.max(1))) @@ -374,6 +380,9 @@ fn truncate_server_large_value_preview( let serde_json::Value::String(text) = value else { return false; }; + if matches!(kind, ServerLargeValuePreviewKind::Deferred) { + return true; + } if matches!(kind, ServerLargeValuePreviewKind::Vector) { let truncated = text.chars().count() > preview_size; let vector_text = if truncated { @@ -397,7 +406,7 @@ fn truncate_server_large_value_preview( .strip_prefix("0x") .filter(|hex| hex.len() > preview_size.saturating_mul(2)) .map(|_| 2usize.saturating_add(preview_size.saturating_mul(2))), - ServerLargeValuePreviewKind::Vector => unreachable!(), + ServerLargeValuePreviewKind::Vector | ServerLargeValuePreviewKind::Deferred => unreachable!(), }; let Some(truncate_at) = truncate_at else { return false; @@ -976,6 +985,7 @@ pub fn agent_execute_query_params( let mut params = serde_json::json!({ "sql": sql, "maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)), + "deferLobs": options.table_data_preview, }); if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) { params["database"] = serde_json::json!(database); @@ -1005,6 +1015,7 @@ pub fn agent_execute_query_page_params( "sql": sql, "pageSize": agent_protocol_row_count(options.page_size.unwrap_or(MAX_ROWS)), "maxRows": agent_protocol_row_count(options.max_rows.unwrap_or(MAX_ROWS)), + "deferLobs": options.table_data_preview, }); if let Some(database) = database.map(str::trim).filter(|database| !database.is_empty()) { params["database"] = serde_json::json!(database); @@ -6921,6 +6932,7 @@ for line in sys.stdin: fetch_size: Some(250), row_offset: Some(100), timeout_secs: Some(600), + table_data_preview: true, ..Default::default() }, ); @@ -6932,6 +6944,7 @@ for line in sys.stdin: assert_eq!(params["fetchSize"], 250); assert_eq!(params["rowOffset"], 100); assert_eq!(params["timeoutSecs"], 600); + assert_eq!(params["deferLobs"], true); } #[test] @@ -7101,6 +7114,7 @@ for line in sys.stdin: fetch_size: Some(250), row_offset: Some(100), timeout_secs: Some(600), + table_data_preview: true, ..Default::default() }, ); @@ -7113,6 +7127,7 @@ for line in sys.stdin: assert_eq!(params["rowOffset"], 100); assert_eq!(params["timeoutSecs"], 600); assert_eq!(params["maxRows"], MAX_ROWS); + assert_eq!(params["deferLobs"], true); } #[test] @@ -7355,6 +7370,47 @@ for line in sys.stdin: assert_eq!(cells.len(), 1); } + #[test] + fn extracts_deferred_oracle_clob_markers_without_changing_placeholder() { + let mut result = db::QueryResult { + columns: vec![ + "id".to_string(), + "payload".to_string(), + format!("{}C_1", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX), + ], + column_types: vec!["number".to_string(), "varchar2".to_string(), "varchar2".to_string()], + column_sortables: vec![true; 3], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), + rows: vec![ + vec![serde_json::json!(1), serde_json::json!(""), serde_json::json!("D:1")], + vec![serde_json::json!(2), serde_json::Value::Null, serde_json::Value::Null], + ], + affected_rows: 0, + execution_time_ms: 0, + truncated: false, + session_id: None, + has_more: false, + elasticsearch_raw_body: None, + messages: Vec::new(), + }; + + let cells = extract_server_large_value_markers(&mut result); + + assert_eq!(result.columns, vec!["id", "payload"]); + assert_eq!(result.column_types, vec!["number", "clob"]); + assert_eq!(result.rows[0], vec![serde_json::json!(1), serde_json::json!("")]); + assert_eq!(result.rows[1], vec![serde_json::json!(2), serde_json::Value::Null]); + assert_eq!( + cells, + vec![db::LargeValueCell { + row_index: 0, + column_index: 1, + original_bytes: SERVER_LARGE_VALUE_UNKNOWN_BYTES, + }] + ); + } + #[test] fn restores_pgvector_previews_as_arrays_and_marks_only_truncated_values() { let marker = format!("{}V_0", crate::sql_dialect::DBX_LARGE_VALUE_BYTES_COLUMN_PREFIX);