diff --git a/agents/drivers/xugu/main_test.go b/agents/drivers/xugu/main_test.go index 7f941a1fd6..39ad15c0da 100644 --- a/agents/drivers/xugu/main_test.go +++ b/agents/drivers/xugu/main_test.go @@ -1010,6 +1010,11 @@ func TestXuguIndexScopeDDL(t *testing.T) { index: indexInfo{IndexType: &indexType}, want: " INDEXTYPE IS BTREE", }, + { + name: "spatial index preserves Xugu RTREE type", + index: indexInfo{IndexType: indexTypePtr("RTREE")}, + want: " INDEXTYPE IS RTREE", + }, { name: "local partition index", index: indexInfo{IndexType: &indexType, IsLocal: true}, @@ -1082,6 +1087,26 @@ func TestXuguIndexScopeDDL(t *testing.T) { } } +func TestXuguIndexTypeName(t *testing.T) { + for _, tc := range []struct { + value any + want string + }{ + {value: int64(0), want: "BTREE"}, + {value: int64(1), want: "RTREE"}, + {value: int64(2), want: "FULLTEXT"}, + {value: int64(3), want: "BITMAP"}, + {value: "RTREE", want: "RTREE"}, + {value: "vendor-specific", want: "vendor-specific"}, + } { + t.Run(fmt.Sprint(tc.value), func(t *testing.T) { + if got := indexTypeName(tc.value); got != tc.want { + t.Fatalf("indexTypeName(%v) = %q, want %q", tc.value, got, tc.want) + } + }) + } +} + func indexTypePtr(value string) *string { return &value } func TestXuguIndexPartitionDetailsStayInternalToTheGenericPayload(t *testing.T) { diff --git a/agents/drivers/xugu/spatial_index_live_test.go b/agents/drivers/xugu/spatial_index_live_test.go new file mode 100644 index 0000000000..20ccbf75dd --- /dev/null +++ b/agents/drivers/xugu/spatial_index_live_test.go @@ -0,0 +1,89 @@ +package main + +import ( + "os" + "strings" + "testing" +) + +// TestLiveXuguSpatialIndexDDL verifies the Xugu-specific spatial index +// contract against a real server. Xugu reports spatial indexes as RTREE and +// reconstructed DDL must retain INDEXTYPE IS RTREE; emitting PostgreSQL's +// USING GIST syntax would not be executable on XuguDB. +func TestLiveXuguSpatialIndexDDL(t *testing.T) { + if os.Getenv("XUGU_LIVE_TEST") != "1" { + t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test") + } + params := connectParams{ + Host: os.Getenv("XUGU_LIVE_HOST"), + Port: parsePort(os.Getenv("XUGU_LIVE_PORT")), + Database: os.Getenv("XUGU_LIVE_DATABASE"), + Username: os.Getenv("XUGU_LIVE_USERNAME"), + Password: os.Getenv("XUGU_LIVE_PASSWORD"), + } + if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" { + t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required") + } + + s := newServer() + db, err := openDB(params) + if err != nil { + t.Fatal(err) + } + s.db = db + s.params = params + s.currentDatabase = params.Database + defer s.disconnect() + + const ( + table = "DBX_SPATIAL_INDEX_LIVE_T" + index = "DBX_SPATIAL_INDEX_LIVE_I" + ) + _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) + defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }() + + for _, statement := range []string{ + "CREATE TABLE " + table + " (ID INTEGER, GEOM GEOMETRY)", + "CREATE INDEX " + index + " ON " + table + " (GEOM) INDEXTYPE IS RTREE", + } { + if err := s.execWithReconnect(statement); err != nil { + t.Fatalf("setup statement failed: %s: %v", statement, err) + } + } + + indexes, err := s.listIndexes(params.Username, table) + if err != nil { + t.Fatal(err) + } + var spatial *indexInfo + for i := range indexes { + if strings.EqualFold(indexes[i].Name, index) { + spatial = &indexes[i] + break + } + } + if spatial == nil { + t.Fatalf("spatial index %s was not listed: %#v", index, indexes) + } + if spatial.IndexType == nil || !strings.EqualFold(strings.TrimSpace(*spatial.IndexType), "RTREE") { + t.Fatalf("spatial index type = %#v, want RTREE: %#v", spatial.IndexType, *spatial) + } + if len(spatial.Columns) != 1 || !strings.EqualFold(spatial.Columns[0], "GEOM") { + t.Fatalf("spatial index columns = %#v, want [GEOM]", spatial.Columns) + } + + ddl, err := s.getTableDDL(params.Username, table) + if err != nil { + t.Fatal(err) + } + upperDDL := strings.ToUpper(ddl) + if !strings.Contains(upperDDL, "CREATE INDEX") || !strings.Contains(upperDDL, strings.ToUpper(index)) { + t.Fatalf("table DDL omitted spatial index: %s", ddl) + } + if !strings.Contains(upperDDL, "INDEXTYPE IS RTREE") { + t.Fatalf("table DDL did not preserve Xugu RTREE syntax: %s", ddl) + } + if strings.Contains(upperDDL, "USING GIST") { + t.Fatalf("table DDL emitted PostgreSQL-only GIST syntax: %s", ddl) + } +} diff --git a/agents/drivers/xugu/spatial_live_test.go b/agents/drivers/xugu/spatial_live_test.go new file mode 100644 index 0000000000..c357fcb6ea --- /dev/null +++ b/agents/drivers/xugu/spatial_live_test.go @@ -0,0 +1,186 @@ +package main + +import ( + "fmt" + "os" + "strings" + "testing" +) + +// TestLiveXuguSpatialQueryRegression exercises the value path used by the +// desktop map preview against a real Xugu server. It is opt-in because CI does +// not provide a Xugu service. The test deliberately covers both GEOMETRY and +// GEOGRAPHY columns, explicit SRIDs, EWKT input, and the multi-geometry types +// rendered by the shared map preview. +func TestLiveXuguSpatialQueryRegression(t *testing.T) { + if os.Getenv("XUGU_LIVE_TEST") != "1" { + t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test") + } + params := liveXuguParams(t) + s := newServer() + db, err := openDB(params) + if err != nil { + t.Fatal(err) + } + s.db = db + s.params = params + s.currentDatabase = params.Database + defer s.disconnect() + + const table = "DBX_SPATIAL_VALUE_LIVE_T" + _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) + defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }() + + for _, statement := range []string{ + "CREATE TABLE " + table + " (ID INTEGER, G_POINT GEOMETRY, G_MULTI GEOMETRY, G_MULTI_LINE GEOMETRY, G_COLLECTION GEOMETRY, G_GEO GEOGRAPHY)", + "INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'), ST_GeomFromEWKT('SRID=3857;MULTIPOINT((1 2),(3 4))'), ST_GeomFromEWKT('SRID=4326;MULTILINESTRING((0 0,1 1),(2 2,3 3))'), ST_GeomFromEWKT('SRID=4326;GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))'), ST_GeomFromEWKT('SRID=4326;POINT(116.397 39.908)'))", + "INSERT INTO " + table + " VALUES (2, ST_GeomFromText('LINESTRING(0 0,1 1,2 2)', 0), ST_GeomFromText('MULTIPOLYGON(((0 0,0 1,1 1,0 0)))', 4326), ST_GeomFromText('MULTILINESTRING((4 4,5 5),(6 6,7 7))', 4326), ST_GeomFromText('POLYGON((0 0,0 1,1 1,0 0))', 4326), ST_GeomFromText('POINT(10 20)', 4326))", + } { + if err := s.execWithReconnect(statement); err != nil { + t.Fatalf("setup statement failed: %s: %v", statement, err) + } + } + + result, err := s.executeSelect("SELECT ID, G_POINT, G_MULTI, G_MULTI_LINE, G_COLLECTION, G_GEO FROM "+table+" ORDER BY ID", 20, 0) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) != 2 { + t.Fatalf("row count = %d, want 2: %#v", len(result.Rows), result.Rows) + } + // The current Go driver reports direct GEOGRAPHY projections without a + // DatabaseTypeName. The decoder deliberately classifies a valid spatial + // payload as GEOMETRY in that case. This is sufficient for the shared map + // preview (which accepts GEOMETRY and GEOGRAPHY), while catalog metadata + // still exposes the source column as GEOGRAPHY. + wantTypes := []string{"INTEGER", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY", "GEOMETRY"} + for i, want := range wantTypes { + if i >= len(result.ColumnTypes) || !strings.EqualFold(result.ColumnTypes[i], want) { + t.Fatalf("column %d type = %#v, want %s (columns=%#v)", i, result.ColumnTypes, want, result.Columns) + } + } + if len(result.SpatialColumns) != 5 { + t.Fatalf("spatial column metadata = %#v, want five geometry/geography columns", result.SpatialColumns) + } + for rowIndex, row := range result.Rows { + for _, columnIndex := range []int{1, 2, 3, 4, 5} { + if row[columnIndex] == nil { + t.Fatalf("row %d column %d unexpectedly NULL: %#v", rowIndex, columnIndex, row) + } + if !isXuguWKT(fmt.Sprint(row[columnIndex])) { + t.Fatalf("row %d column %d was not decoded as WKT: %#v", rowIndex, columnIndex, row[columnIndex]) + } + } + } + if len(result.SpatialValues) != 2 || len(result.SpatialValues[0]) != 6 { + t.Fatalf("spatial cell metadata shape = %#v", result.SpatialValues) + } + if result.SpatialValues[0][1] == nil || *result.SpatialValues[0][1] != 4326 || result.SpatialValues[0][5] == nil || *result.SpatialValues[0][5] != 4326 { + t.Fatalf("row 1 SRIDs = %#v, want geometry/geography SRID 4326", result.SpatialValues[0]) + } + if result.SpatialValues[0][2] == nil || *result.SpatialValues[0][2] != 3857 { + t.Fatalf("row 1 multipoint SRID = %#v, want 3857", result.SpatialValues[0][2]) + } + raw, err := s.executeSelect("SELECT ST_AsBinary(G_POINT), ST_AsEWKB(G_POINT), ST_AsEWKT(G_GEO), ST_AsEWKB(G_MULTI), ST_AsEWKB(G_COLLECTION) FROM "+table+" WHERE ID = 1", 20, 0) + if err != nil { + t.Fatalf("WKB/EWKB/EWKT query failed: %v", err) + } + if len(raw.Rows) != 1 || len(raw.Rows[0]) != 5 { + t.Fatalf("unexpected WKB/EWKB/EWKT result: %#v", raw) + } + for columnIndex, wantSRID := range map[int]*uint32{0: nil, 1: uint32Pointer(4326)} { + bytes, ok := raw.Rows[0][columnIndex].(string) + if !ok { + t.Fatalf("raw WKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex]) + } + decoded, decodedOK := decodeXuguWKB([]byte(bytes)) + if !decodedOK || !isXuguWKT(decoded.WKT) || (wantSRID == nil && decoded.SRID != nil) || (wantSRID != nil && (decoded.SRID == nil || *decoded.SRID != *wantSRID)) { + t.Fatalf("raw WKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK) + } + } + decodedEWKT, sridEWKT, recognizedEWKT := decodeXuguSpatialValue(raw.Rows[0][2]) + if !recognizedEWKT || decodedEWKT != "POINT(116.397 39.908)" || sridEWKT == nil || *sridEWKT != 4326 { + t.Fatalf("EWKT column decoded as value=%#v srid=%#v recognized=%v", decodedEWKT, sridEWKT, recognizedEWKT) + } + for columnIndex, want := range map[int]struct { + wkt string + srid uint32 + }{ + 3: {wkt: "MULTIPOINT((1 2),(3 4))", srid: 3857}, + 4: {wkt: "GEOMETRYCOLLECTION(POINT(1 2),LINESTRING(3 4,5 6))", srid: 4326}, + } { + bytes, ok := raw.Rows[0][columnIndex].(string) + if !ok { + t.Fatalf("complex EWKB column %d type = %T, want binary string", columnIndex, raw.Rows[0][columnIndex]) + } + decoded, decodedOK := decodeXuguWKB([]byte(bytes)) + if !decodedOK || decoded.WKT != want.wkt || decoded.SRID == nil || *decoded.SRID != want.srid { + t.Fatalf("complex EWKB column %d decoded as value=%#v ok=%v", columnIndex, decoded, decodedOK) + } + } + t.Logf("Xugu WKB/EWKB/EWKT result: types=%#v columns=%#v rows=%#v spatialColumns=%#v spatialValues=%#v", raw.ColumnTypes, raw.Columns, raw.Rows, raw.SpatialColumns, raw.SpatialValues) + t.Logf("decoded Xugu spatial result: columns=%#v types=%#v spatialColumns=%#v values=%#v rows=%#v", result.Columns, result.ColumnTypes, result.SpatialColumns, result.SpatialValues, result.Rows) +} + +// TestLiveXuguSpatialReplay probes the literal form generated by the generic +// export path. Xugu accepts WKT constructors for replay; the test records +// whether a plain quoted WKT is accepted so export code can remain database +// specific instead of guessing from PostgreSQL syntax. +func TestLiveXuguSpatialReplay(t *testing.T) { + if os.Getenv("XUGU_LIVE_TEST") != "1" { + t.Skip("set XUGU_LIVE_TEST=1 with XUGU_LIVE_* connection settings to run the XuguDB integration test") + } + params := liveXuguParams(t) + s := newServer() + db, err := openDB(params) + if err != nil { + t.Fatal(err) + } + s.db = db + s.params = params + s.currentDatabase = params.Database + defer s.disconnect() + + const table = "DBX_SPATIAL_REPLAY_LIVE_T" + _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) + defer func() { _ = s.execWithReconnect("DROP TABLE IF EXISTS " + table) }() + for _, statement := range []string{ + "CREATE TABLE " + table + " (ID INTEGER, G GEOMETRY, GEO GEOGRAPHY)", + "INSERT INTO " + table + " VALUES (1, ST_GeomFromEWKT('SRID=4326;POINT(1 2)'), ST_GeomFromEWKT('SRID=4326;POINT(3 4)'))", + } { + if err := s.execWithReconnect(statement); err != nil { + t.Fatalf("setup statement failed: %s: %v", statement, err) + } + } + if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (2, 'POINT(5 6)', 'POINT(7 8)')"); err != nil { + t.Logf("plain quoted WKT is rejected by Xugu (expected on typed spatial columns): %v", err) + } else { + t.Log("plain quoted WKT is accepted by Xugu typed spatial columns") + } + if err := s.execWithReconnect("INSERT INTO " + table + " VALUES (3, ST_GeomFromEWKT('SRID=3857;POINT(9 10)'), ST_GeomFromEWKT('SRID=4326;POINT(11 12)'))"); err != nil { + t.Fatalf("Xugu spatial constructor replay failed: %v", err) + } + result, err := s.executeSelect("SELECT ID, ST_AsEWKT(G), ST_AsEWKT(GEO) FROM "+table+" ORDER BY ID", 20, 0) + if err != nil { + t.Fatal(err) + } + if len(result.Rows) < 2 { + t.Fatalf("spatial replay returned too few rows: %#v", result.Rows) + } + t.Logf("Xugu spatial replay result: types=%#v rows=%#v", result.ColumnTypes, result.Rows) +} + +func liveXuguParams(t *testing.T) connectParams { + t.Helper() + params := connectParams{ + Host: os.Getenv("XUGU_LIVE_HOST"), + Port: parsePort(os.Getenv("XUGU_LIVE_PORT")), + Database: os.Getenv("XUGU_LIVE_DATABASE"), + Username: os.Getenv("XUGU_LIVE_USERNAME"), + Password: os.Getenv("XUGU_LIVE_PASSWORD"), + } + if params.Host == "" || params.Database == "" || params.Username == "" || params.Password == "" { + t.Fatal("XUGU_LIVE_HOST, XUGU_LIVE_DATABASE, XUGU_LIVE_USERNAME, and XUGU_LIVE_PASSWORD are required") + } + return params +} diff --git a/apps/desktop/src/components/grid/DataGrid.vue b/apps/desktop/src/components/grid/DataGrid.vue index b2d3a1eaf3..97173a2418 100644 --- a/apps/desktop/src/components/grid/DataGrid.vue +++ b/apps/desktop/src/components/grid/DataGrid.vue @@ -7844,6 +7844,8 @@ const { columnComments: visibleColumnComments, allColumnComments, mongoDocuments: computed(() => props.result.mongo_copy_documents ?? props.result.mongo_documents), + spatialColumns: computed(() => props.result.spatial_columns), + spatialValues: computed(() => props.result.spatial_values), columnTypes: visibleColumnTypes, allColumnTypes, whereInput: computed(() => currentWhereInput()), diff --git a/apps/desktop/src/composables/useDataGridExport.ts b/apps/desktop/src/composables/useDataGridExport.ts index 9d04981f18..6b0584d034 100644 --- a/apps/desktop/src/composables/useDataGridExport.ts +++ b/apps/desktop/src/composables/useDataGridExport.ts @@ -86,6 +86,8 @@ export interface UseDataGridExportOptions { columnComments?: ComputedRef>; allColumnComments?: ComputedRef>; mongoDocuments?: ComputedRef; + spatialColumns?: ComputedRef; + spatialValues?: ComputedRef; columnTypes: ComputedRef | undefined>; allColumnTypes?: ComputedRef | undefined>; whereInput: ComputedRef; @@ -174,6 +176,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) { connectionId, database, context, + spatialColumns: spatialColumnsOption, + spatialValues: spatialValuesOption, whereInput, orderBy, columnTypes, @@ -335,12 +339,24 @@ export function useDataGridExport(options: UseDataGridExportOptions) { useFullExport = true, formatDateTime = true, headerMode: XlsxHeaderMode = "name", - ): Promise<{ columns: string[]; columnTypes: string[]; columnComments?: (string | null)[]; rows: CellValue[][] }> { + ): Promise<{ + columns: string[]; + columnTypes: string[]; + columnComments?: (string | null)[]; + spatialColumns?: QueryResult["spatial_columns"]; + spatialValues?: QueryResult["spatial_values"]; + rows: CellValue[][]; + }> { if (useFullExport && rowIds === undefined && fullExportResult && !hasCompleteLocalResult?.value) { const result = await fullExportResult(onProgress); if (result) { const columnComments = buildXlsxHeaderOverrides(result.columns, commentsForExportColumns(result.columns), headerMode); - return { ...applyGlobalDateTimeExportFormat({ columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }, formatDateTime), columnComments }; + return { + ...applyGlobalDateTimeExportFormat({ columns: result.columns, columnTypes: result.column_types ?? [], rows: result.rows }, formatDateTime), + columnComments, + spatialColumns: result.spatial_columns, + spatialValues: result.spatial_values, + }; } } // The full result is already in memory — export the raw QueryResult (all @@ -351,19 +367,30 @@ export function useDataGridExport(options: UseDataGridExportOptions) { if (useFullExport && rowIds === undefined && hasCompleteLocalResult?.value && completeLocalResult?.value) { const normalized = normalizeCompleteLocalResult(completeLocalResult.value); const columnComments = buildXlsxHeaderOverrides(normalized.columns, normalized.columnComments, headerMode); - return { ...applyGlobalDateTimeExportFormat({ columns: normalized.columns, columnTypes: normalized.columnTypes, rows: normalized.rows }, formatDateTime), columnComments }; + return { + ...applyGlobalDateTimeExportFormat({ columns: normalized.columns, columnTypes: normalized.columnTypes, rows: normalized.rows }, formatDateTime), + columnComments, + spatialColumns: completeLocalResult.value.spatial_columns, + spatialValues: completeLocalResult.value.spatial_values, + }; } const commentHeader = buildXlsxHeaderOverrides(columns.value, visibleXlsxColumnComments.value, headerMode); + const exportItems = await resolveVisibleRowValues(rowsToExport(rowIds)); + const visibleIndexes = visibleColumnIndexesOption?.value ?? columns.value.map((_, index) => index); + const spatialColumns = spatialColumnsOption?.value?.map((column) => ({ column_index: visibleIndexes.indexOf(column.column_index), srid: column.srid })).filter((column) => column.column_index >= 0); + const spatialValues = spatialValuesOption?.value; return { ...applyGlobalDateTimeExportFormat( { columns: columns.value, columnTypes: (columnTypes.value ?? []).map((type) => type ?? ""), - rows: (await resolveVisibleRowValues(rowsToExport(rowIds))).map((item) => item.data), + rows: exportItems.map((item) => item.data), }, formatDateTime, ), columnComments: commentHeader, + ...(spatialColumns?.length ? { spatialColumns } : {}), + ...(spatialValues?.length ? { spatialValues: exportItems.map((item) => visibleIndexes.map((index) => spatialValues[item.sourceIndex ?? -1]?.[index] ?? null)) } : {}), }; } @@ -1278,6 +1305,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) { tableName: tableMeta.value?.tableName || "table_name", columns: exportData.columns, columnTypes: exportData.columnTypes, + spatialColumns: exportData.spatialColumns, + spatialValues: exportData.spatialValues, rows: exportData.rows, }); await saveTextFile(content, exportFileName(tableMeta.value?.tableName || "export", "sql", { preferFallback: true }), "SQL", "sql"); @@ -1300,6 +1329,8 @@ export function useDataGridExport(options: UseDataGridExportOptions) { tableName: tableMeta.value?.tableName || "table_name", columns: exportData.columns, columnTypes: exportData.columnTypes, + spatialColumns: exportData.spatialColumns, + spatialValues: exportData.spatialValues, rows: exportData.rows, }); await saveTextFile(content, exportFileName("export-page", "sql", { page: true }), "SQL", "sql"); @@ -1315,17 +1346,26 @@ export function useDataGridExport(options: UseDataGridExportOptions) { await copyText(sql.value); } - function sqlInsertExportData(result: { columns: string[]; rows: CellValue[][] }): { + function sqlInsertExportData(result: { columns: string[]; rows: CellValue[][]; spatialColumns?: QueryResult["spatial_columns"]; spatialValues?: QueryResult["spatial_values"] }): { columns: string[]; columnTypes?: Array; + spatialColumns?: QueryResult["spatial_columns"]; + spatialValues?: QueryResult["spatial_values"]; rows: CellValue[][]; } { const exportColumns = context.value === "table-data" && tableMeta.value ? effectiveColumns(sourceColumns.value, result.columns) : result.columns; const columnIndexes = exportColumns.map((column, index) => ({ column, index })).filter((item): item is { column: string; index: number } => !!item.column); const exportColumnTypes = columnTypes.value?.length === result.columns.length ? columnTypes.value : undefined; + const indexBySource = new Map(columnIndexes.map((item, index) => [item.index, index])); + const spatialColumns = result.spatialColumns?.flatMap((column) => { + const columnIndex = indexBySource.get(column.column_index); + return columnIndex === undefined ? [] : [{ column_index: columnIndex, srid: column.srid }]; + }); return { columns: columnIndexes.map((item) => item.column), columnTypes: exportColumnTypes ? columnIndexes.map((item) => exportColumnTypes[item.index]) : undefined, + ...(spatialColumns?.length ? { spatialColumns } : {}), + ...(result.spatialValues?.length ? { spatialValues: result.spatialValues.map((row) => columnIndexes.map((item) => row[item.index] ?? null)) } : {}), rows: result.rows.map((row) => columnIndexes.map((item) => row[item.index] ?? null)), }; } diff --git a/apps/desktop/src/lib/__tests__/dataGrid/geometryMapPreview.spec.ts b/apps/desktop/src/lib/__tests__/dataGrid/geometryMapPreview.spec.ts index 79004bfaf0..c323b65f57 100644 --- a/apps/desktop/src/lib/__tests__/dataGrid/geometryMapPreview.spec.ts +++ b/apps/desktop/src/lib/__tests__/dataGrid/geometryMapPreview.spec.ts @@ -29,6 +29,29 @@ function context(): PreviewActionContext { } describe("buildGeometryMapFeatureCollection", () => { + it("treats GEOGRAPHY columns as map-preview layers and preserves SRID metadata", () => { + const ctx = context(); + ctx.result.column_types = ["GEOGRAPHY", "text"]; + ctx.result.columns = ["location", "name"]; + ctx.result.rows = [["SRID=4326;POINT(116.397 39.908)", "wgs84"]]; + ctx.result.spatial_columns = [{ column_index: 0, srid: 4326 }]; + ctx.result.spatial_values = [[4326, null]]; + ctx.displayRowRefs = [{ id: 1, sourceIndex: 0, isNew: false }]; + + expect(hasGeometryMapPreviewColumns(ctx.result)).toBe(true); + expect(buildGeometryMapFeatureCollection(ctx)).toEqual({ + type: "FeatureCollection", + detectedSrid: 4326, + features: [ + { + type: "Feature", + geometry: { type: "Point", coordinates: [116.397, 39.908] }, + properties: { _column: "location", _row: 0, _srid: 4326, name: "wgs84" }, + }, + ], + }); + }); + it("checks preview availability from column metadata without parsing result rows", () => { const result = context().result; expect(hasGeometryMapPreviewColumns(result)).toBe(true); diff --git a/apps/desktop/src/lib/export/databaseExport.ts b/apps/desktop/src/lib/export/databaseExport.ts index 77faf24e4e..7be6947079 100644 --- a/apps/desktop/src/lib/export/databaseExport.ts +++ b/apps/desktop/src/lib/export/databaseExport.ts @@ -18,6 +18,8 @@ export interface ExportedTableSql { columns: string[]; columnTypes?: Array; columnExtras?: Array; + spatialColumns?: QueryResult["spatial_columns"]; + spatialValues?: QueryResult["spatial_values"]; rows: QueryResult["rows"]; truncated?: boolean; } @@ -41,6 +43,8 @@ export interface BuildExportInsertStatementsOptions { columns: string[]; columnTypes?: Array; columnExtras?: Array; + spatialColumns?: QueryResult["spatial_columns"]; + spatialValues?: QueryResult["spatial_values"]; rows: QueryResult["rows"]; batchSize?: number; } diff --git a/apps/desktop/src/lib/export/exportFormats.ts b/apps/desktop/src/lib/export/exportFormats.ts index b759b1ef4f..9419e7833d 100644 --- a/apps/desktop/src/lib/export/exportFormats.ts +++ b/apps/desktop/src/lib/export/exportFormats.ts @@ -1,4 +1,4 @@ -import type { DatabaseType } from "@/types/database"; +import type { DatabaseType, QueryResult } from "@/types/database"; import * as api from "@/lib/backend/api"; export type ExportCellValue = string | number | boolean | null; @@ -35,6 +35,8 @@ export interface FormatSqlInsertOptions { qualifiedTableName?: string; columns: string[]; columnTypes?: Array; + spatialColumns?: QueryResult["spatial_columns"]; + spatialValues?: QueryResult["spatial_values"]; rows: ExportCellValue[][]; } diff --git a/crates/dbx-core/examples/data_transfer_bench.rs b/crates/dbx-core/examples/data_transfer_bench.rs index 4a96f0a9f1..ac82902943 100644 --- a/crates/dbx-core/examples/data_transfer_bench.rs +++ b/crates/dbx-core/examples/data_transfer_bench.rs @@ -100,6 +100,8 @@ fn run() -> Result<(), String> { columns: columns.clone(), column_types: vec![None; columns.len()], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: rows.clone(), batch_size: Some(options.batch_size), })?; diff --git a/crates/dbx-core/src/database_export.rs b/crates/dbx-core/src/database_export.rs index 4bd8981e87..56d2c30514 100644 --- a/crates/dbx-core/src/database_export.rs +++ b/crates/dbx-core/src/database_export.rs @@ -17,7 +17,7 @@ use crate::transfer::{ is_mysql_generated_column_extra, keyset_pagination_sql_with_identifier_quote, quote_identifier, quote_postgres_string_literal, wrap_dameng_identity_insert_sql_for_table, }; -use crate::types::ObjectSourceKind; +use crate::types::{ObjectSourceKind, SpatialColumn}; static EXPORT_CANCELLED: std::sync::LazyLock>> = std::sync::LazyLock::new(|| RwLock::new(HashSet::new())); @@ -288,6 +288,10 @@ pub struct ExportedTableSql { #[serde(default)] pub column_extras: Vec>, #[serde(default)] + pub spatial_columns: Vec, + #[serde(default)] + pub spatial_values: Vec>>, + #[serde(default)] pub rows: Vec>, #[serde(default)] pub truncated: bool, @@ -313,6 +317,10 @@ pub struct BuildExportInsertStatementsOptions { #[serde(default)] pub column_extras: Vec>, #[serde(default)] + pub spatial_columns: Vec, + #[serde(default)] + pub spatial_values: Vec>>, + #[serde(default)] pub rows: Vec>, #[serde(default, skip_serializing_if = "Option::is_none")] pub batch_size: Option, @@ -394,6 +402,9 @@ fn format_export_sql_literal_typed( if let Some(literal) = format_mysql_spatial_export_literal(value, database_type, column_type) { return literal; } + if let Some(literal) = format_xugu_spatial_export_literal(value, database_type, column_type) { + return literal; + } if is_mysql_compatible_export_literal_target(database_type) { if column_type.is_some_and(is_mysql_binary_export_type) { if let Some(literal) = format_mysql_binary_export_literal(value) { @@ -832,6 +843,50 @@ pub(crate) fn is_mysql_spatial_export_type(column_type: &str) -> bool { ) } +pub(crate) fn is_xugu_spatial_export_type(column_type: &str) -> bool { + let normalized = column_type.trim().to_ascii_lowercase(); + let base = normalized.split(['(', ':', ' ', '\t', '\n']).next().unwrap_or("").trim(); + matches!(base, "geometry" | "geography") +} + +/// Xugu returns spatial values as readable WKT/EWKT text. Plain WKT is +/// accepted by the server, but it cannot carry a non-zero SRID; database +/// exports therefore select EWKT and replay it through the Xugu constructor. +/// This branch is intentionally Xugu-only so PostgreSQL/PostGIS and other +/// dialects retain their existing export behavior. +pub(crate) fn format_xugu_spatial_export_literal( + value: &Value, + database_type: Option, + column_type: Option<&str>, +) -> Option { + format_xugu_spatial_export_literal_with_srid(value, database_type, column_type, None) +} + +fn format_xugu_spatial_export_literal_with_srid( + value: &Value, + database_type: Option, + column_type: Option<&str>, + srid: Option, +) -> Option { + if database_type != Some(DatabaseType::Xugu) || !column_type.is_some_and(is_xugu_spatial_export_type) { + return None; + } + if value.is_null() { + return Some("NULL".to_string()); + } + let text = value.as_str().map_or_else(|| value.to_string(), ToString::to_string); + let trimmed = text.trim_start(); + if trimmed.len() > 5 && trimmed[..5].eq_ignore_ascii_case("SRID=") { + return Some(format!("ST_GeomFromEWKT({})", quote_export_sql_string(&text))); + } + if let Some(srid) = srid.filter(|srid| *srid != 0) { + return Some(format!("ST_GeomFromEWKT({})", quote_export_sql_string(&format!("SRID={srid};{text}")))); + } + // Xugu accepts a plain WKT string for both GEOMETRY and GEOGRAPHY. Keep + // that form for SRID 0/legacy values rather than inventing a constructor. + Some(quote_export_sql_string(&text)) +} + /// Database exports encode MySQL spatial cells as `DBX_WKB::` while /// reading them. Keeping this marker internal lets the normal JSON row shape /// and all non-export query paths continue to expose readable WKT values. @@ -926,6 +981,31 @@ fn format_export_numeric_literal(value: &Value) -> Option { } } +fn export_column_type<'a>( + column_types: &'a [Option], + index: usize, + database_type: Option, + spatial_columns: &HashMap>, +) -> Option<&'a str> { + column_types.get(index).and_then(|value| value.as_deref()).filter(|value| !value.trim().is_empty()).or_else(|| { + (database_type == Some(DatabaseType::Xugu) && spatial_columns.contains_key(&index)).then_some("GEOMETRY") + }) +} + +fn format_export_sql_literal_typed_with_spatial( + value: &Value, + database_type: Option, + column_type: Option<&str>, + sqlserver_unicode_string: bool, + spatial_srid: Option, +) -> String { + if let Some(literal) = format_xugu_spatial_export_literal_with_srid(value, database_type, column_type, spatial_srid) + { + return literal; + } + format_export_sql_literal_typed(value, database_type, column_type, sqlserver_unicode_string) +} + fn is_export_numeric_literal(text: &str) -> bool { if text.trim() != text || text.is_empty() { return false; @@ -947,12 +1027,14 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption options.qualified_table_name.as_deref(), options.identifier_quote.as_deref(), )?; + let spatial_columns = + options.spatial_columns.iter().map(|column| (column.column_index, column.srid)).collect::>(); let insert_columns = options .columns .iter() .enumerate() .filter_map(|(index, column)| { - let column_type = options.column_types.get(index).and_then(|value| value.as_deref()); + let column_type = export_column_type(&options.column_types, index, options.database_type, &spatial_columns); is_export_insert_column( options.database_type, column, @@ -1023,7 +1105,7 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption *row_count = 0; }; - for row in options.rows { + for (row_index, row) in options.rows.into_iter().enumerate() { let mut rendered_row = String::with_capacity(insert_columns.len().saturating_mul(16).saturating_add(2)); rendered_row.push('('); for (column_index, (index, _, sqlserver_unicode_string)) in insert_columns.iter().enumerate() { @@ -1031,11 +1113,21 @@ pub fn build_export_insert_statements(options: BuildExportInsertStatementsOption rendered_row.push_str(", "); } let value = row.get(*index).unwrap_or(&Value::Null); - rendered_row.push_str(&format_export_sql_literal_typed( + let column_type = + export_column_type(&options.column_types, *index, options.database_type, &spatial_columns); + let spatial_srid = options + .spatial_values + .get(row_index) + .and_then(|values| values.get(*index)) + .copied() + .flatten() + .or_else(|| spatial_columns.get(index).copied().flatten()); + rendered_row.push_str(&format_export_sql_literal_typed_with_spatial( value, options.database_type, - options.column_types.get(*index).and_then(|value| value.as_deref()), + column_type, *sqlserver_unicode_string, + spatial_srid, )); } rendered_row.push(')'); @@ -1164,6 +1256,8 @@ pub fn build_database_sql_export(options: BuildDatabaseSqlExportOptions) -> Resu columns: table.columns, column_types: table.column_types, column_extras: table.column_extras, + spatial_columns: table.spatial_columns, + spatial_values: table.spatial_values, rows: table.rows, batch_size: Some(insert_batch_size), })?; @@ -1627,6 +1721,11 @@ fn database_export_select_list(columns: &[String], column_types: &[Option( columns: insert_columns.to_vec(), column_types: insert_column_types.to_vec(), column_extras: insert_column_extras.to_vec(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: insert_rows.to_vec(), batch_size: Some(DATABASE_EXPORT_INSERT_BATCH_SIZE), })?; @@ -2967,18 +3068,20 @@ mod tests { database_export_select_sql, database_export_total_objects, drop_table_if_exists_sql, ensure_export_destination_dir, export_destination_identity_mismatch, filter_export_table_infos, format_export_sql_literal, format_export_table_ddl, format_mysql_spatial_export_literal, - generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, generate_postgres_sequence_owner_ddl, - generate_postgres_sequence_setval_sql, is_postgres_extension_member_routine, mysql_database_export_preamble, - mysql_view_dependencies_from_rows, mysql_view_dependencies_sql, normalize_export_table_ddl, - record_export_destination_identity, record_export_error, replace_database_export_select_list, - sort_export_views_by_dependencies, split_postgres_export_table_triggers, write_database_export_rows, - BuildDatabaseSqlExportOptions, BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, - DatabaseExportRequest, DdlNormalizeOptions, ExportedTableSql, PostgresExportExtension, PostgresExportSequence, - PostgresExtensionMembers, DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT, + format_xugu_spatial_export_literal, generate_postgres_extension_ddl, generate_postgres_sequence_create_ddl, + generate_postgres_sequence_owner_ddl, generate_postgres_sequence_setval_sql, + is_postgres_extension_member_routine, mysql_database_export_preamble, mysql_view_dependencies_from_rows, + mysql_view_dependencies_sql, normalize_export_table_ddl, record_export_destination_identity, + record_export_error, replace_database_export_select_list, sort_export_views_by_dependencies, + split_postgres_export_table_triggers, write_database_export_rows, BuildDatabaseSqlExportOptions, + BuildExportInsertStatementsOptions, DatabaseExportObjectCounts, DatabaseExportRequest, DdlNormalizeOptions, + ExportedTableSql, PostgresExportExtension, PostgresExportSequence, PostgresExtensionMembers, + DATABASE_EXPORT_INSERT_BATCH_SIZE, DATABASE_EXPORT_ROW_LIMIT, }; use crate::connection::AppState; use crate::models::connection::DatabaseType; use crate::storage::Storage; + use crate::types::SpatialColumn; use crate::types::{ObjectInfo, ObjectSourceKind, TableInfo}; use serde_json::{json, Value}; use std::sync::{ @@ -3441,6 +3544,8 @@ mod tests { columns: vec!["location".to_string(), "shape".to_string()], column_types: vec![Some("point".to_string()), Some("geometry".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![ json!("DBX_WKB:4326:0101000000AE47E17A14AE5C4052B81E85EBF34240"), json!("DBX_WKB:0:0101000000000000000000F03F0000000000000040"), @@ -3508,6 +3613,85 @@ mod tests { assert!(sql.contains("ORDER BY `id` ASC LIMIT 1000"), "sql: {sql}"); } + #[test] + fn xugu_spatial_export_selects_ewkt_to_preserve_srid() { + let sql = database_export_select_sql( + &["id".to_string(), "shape".to_string(), "location".to_string(), "name".to_string()], + &[ + Some("INTEGER".to_string()), + Some("GEOMETRY".to_string()), + Some("GEOGRAPHY".to_string()), + Some("VARCHAR(32)".to_string()), + ], + "places", + "app", + &DatabaseType::Xugu, + ); + + assert_eq!( + sql, + "SELECT \"id\", ST_AsEWKT(\"shape\") AS \"shape\", ST_AsEWKT(\"location\") AS \"location\", \"name\" FROM \"app\".\"places\"" + ); + } + + #[test] + fn xugu_spatial_export_replays_ewkt_and_keeps_plain_wkt_compatible() { + assert_eq!( + format_xugu_spatial_export_literal( + &json!("SRID=3857;POINT(1 2)"), + Some(DatabaseType::Xugu), + Some("GEOMETRY"), + ), + Some("ST_GeomFromEWKT('SRID=3857;POINT(1 2)')".to_string()) + ); + assert_eq!( + format_xugu_spatial_export_literal(&json!("POINT(1 2)"), Some(DatabaseType::Xugu), Some("GEOGRAPHY"),), + Some("'POINT(1 2)'".to_string()) + ); + assert_eq!( + format_xugu_spatial_export_literal(&Value::Null, Some(DatabaseType::Xugu), Some("GEOMETRY")), + Some("NULL".to_string()) + ); + assert!(format_xugu_spatial_export_literal( + &json!("SRID=3857;POINT(1 2)"), + Some(DatabaseType::Xugu), + Some("VARCHAR") + ) + .is_none()); + assert!(format_xugu_spatial_export_literal( + &json!("SRID=3857;POINT(1 2)"), + Some(DatabaseType::Postgres), + Some("GEOMETRY") + ) + .is_none()); + } + + #[test] + fn xugu_spatial_metadata_recovers_missing_type_and_cell_srid() { + let statements = build_export_insert_statements(BuildExportInsertStatementsOptions { + database_type: Some(DatabaseType::Xugu), + identifier_quote: None, + schema: Some("app".to_string()), + table_name: Some("places".to_string()), + qualified_table_name: None, + columns: vec!["shape".to_string()], + column_types: vec![None], + column_extras: vec![None], + spatial_columns: vec![SpatialColumn { column_index: 0, srid: Some(3857) }], + spatial_values: vec![vec![Some(3857)]], + rows: vec![vec![json!("POINT(1 2)")]], + batch_size: None, + }) + .unwrap(); + + assert_eq!( + statements, + vec![ + r#"INSERT INTO "app"."places" ("shape") VALUES (ST_GeomFromEWKT('SRID=3857;POINT(1 2)'));"#.to_string() + ] + ); + } + #[test] fn database_specific_boolean_export_literals() { let sqlserver_statements = build_export_insert_statements(BuildExportInsertStatementsOptions { @@ -3519,6 +3703,8 @@ mod tests { columns: vec!["typed_true".to_string(), "untyped_false".to_string(), "typed_null".to_string()], column_types: vec![Some("bit".to_string()), None, Some("bit".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), }) @@ -3532,6 +3718,8 @@ mod tests { columns: vec!["enabled".to_string(), "disabled".to_string(), "unknown".to_string()], column_types: Vec::new(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), }) @@ -3574,6 +3762,8 @@ mod tests { Some("nvarchar(20)".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![ json!("张'三"), json!("中文"), @@ -3605,6 +3795,8 @@ mod tests { columns: vec!["body".to_string()], column_types: vec![Some("text".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("line1\nline2\tcol\rend\\slash\0\x1aO'Hara")]], batch_size: Some(10), }) @@ -3628,6 +3820,8 @@ mod tests { columns: vec!["payload".to_string()], column_types: vec![Some("longtext".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(long_value.clone())], vec![json!(long_value)]], batch_size: Some(100), }) @@ -3648,6 +3842,8 @@ mod tests { columns: vec!["id".to_string()], column_types: vec![Some("int".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: (0..1001).map(|id| vec![json!(id)]).collect(), batch_size: Some(2000), }) @@ -3669,6 +3865,8 @@ mod tests { columns: vec!["message".to_string()], column_types: vec![Some("varchar(255)".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("first\nsecond\tthird")]], batch_size: Some(10), }) @@ -3688,6 +3886,8 @@ mod tests { columns: vec!["body".to_string()], column_types: vec![Some("text".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("line1\nline2\tend")]], batch_size: Some(10), }) @@ -3712,6 +3912,8 @@ mod tests { Some("text".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("line1\rline2"), json!("O'Hara"), json!(r"C:\tmp"), json!("plain")]], batch_size: Some(10), }) @@ -3736,6 +3938,8 @@ mod tests { columns: vec!["payload".to_string()], column_types: vec![Some("jsonb".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(r#"{"text":"say \"hi\"","path":"C:\\tmp","quote":"O'Hara"}"#)]], batch_size: Some(10), }) @@ -3770,6 +3974,8 @@ mod tests { Some("text[]".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!([1.2, 3.4]), json!(["5", "6"]), json!(["x", "y"])]], batch_size: Some(10), }) @@ -3794,6 +4000,8 @@ mod tests { columns: vec!["id".to_string(), "name".to_string()], column_types: Vec::new(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("O'Hara")], vec![json!(3), json!("Linus")]], batch_size: Some(2), }) @@ -3819,6 +4027,8 @@ mod tests { columns: vec!["ID".to_string(), "NAME".to_string()], column_types: Vec::new(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")], vec![json!(2), json!("Linus")]], batch_size: Some(100), }) @@ -3844,6 +4054,8 @@ mod tests { columns: vec!["__DBX_ROWID".to_string(), "ID".to_string(), "NAME".to_string()], column_types: vec![Some("VARCHAR2".to_string()), Some("NUMBER".to_string()), Some("VARCHAR2".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("AAAPr9AAEAAAAGfAAA"), json!(1), json!("Ada")]], batch_size: Some(100), }) @@ -3863,6 +4075,8 @@ mod tests { columns: vec!["__DBX_ROWID".to_string(), "ID".to_string(), "NAME".to_string()], column_types: vec![Some("VARCHAR2".to_string()), Some("NUMBER".to_string()), Some("VARCHAR2".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("*AAABk1AAEAAAAAgAAA"), json!(1), json!("Ada")]], batch_size: Some(100), }) @@ -3882,6 +4096,8 @@ mod tests { columns: vec!["__DBX_ROWID".to_string(), "name".to_string()], column_types: Vec::new(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(7), json!("Ada")]], batch_size: Some(100), }) @@ -3901,6 +4117,8 @@ mod tests { columns: vec!["ID".to_string(), "CREATED_ON".to_string(), "RAW_TEXT".to_string()], column_types: vec![Some("NUMBER".to_string()), Some("DATE".to_string()), Some("VARCHAR2(64)".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![ vec![json!(1), json!("2022-08-25T09:58:43Z"), json!("2022-08-25T09:58:43Z")], vec![json!(2), json!("2022-08-25T00:00:00Z"), json!("2022-08-25T00:00:00Z")], @@ -3954,6 +4172,8 @@ mod tests { Some("TIMESTAMP".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![ json!("2022-08-25 09:58:43.123456"), json!("2022-08-26T10:59:44Z"), @@ -4002,6 +4222,8 @@ mod tests { columns: vec!["created_at".to_string(), "recorded_at".to_string()], column_types: vec![Some("timestamp".to_string()), Some("timestamp with time zone".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("2022-08-25T09:58:43.123456Z"), json!("2022-08-26T10:59:44+08:00")]], batch_size: Some(100), }) @@ -4027,6 +4249,8 @@ mod tests { columns: vec!["enabled".to_string(), "mask".to_string(), "label".to_string()], column_types: vec![Some("bit(1)".to_string()), Some("BIT(4)".to_string()), Some("varchar(20)".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("1"), json!("1010"), json!("1010")], vec![json!(false), json!(3), json!("off")]], batch_size: Some(10), }) @@ -4049,6 +4273,8 @@ mod tests { columns: vec!["ENABLED".to_string(), "DELETED".to_string(), "OPTIONAL".to_string()], column_types: vec![Some("BIT".to_string()), Some("bit".to_string()), Some("BIT".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(true), json!(false), Value::Null]], batch_size: Some(10), }) @@ -4078,6 +4304,8 @@ mod tests { ], column_types: vec![Some("VARCHAR".to_string()); 6], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![ json!("plain"), json!("eHall\0"), @@ -4113,6 +4341,8 @@ mod tests { columns: vec!["id".to_string(), "f_blob".to_string(), "note".to_string()], column_types: vec![Some("int".to_string()), Some("blob".to_string()), Some("varchar(64)".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![ vec![json!("1"), json!("0x68656c6c6f"), json!("0x68656c6c6f")], vec![json!("2"), json!("0X"), json!("1")], @@ -4145,6 +4375,8 @@ mod tests { Some("varchar(64)".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![ json!(1), json!("2026-06-12T10:11:12.123456789Z"), @@ -4177,6 +4409,8 @@ mod tests { Some("timestamp without time zone".to_string()), ], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T18:11:12+08:00")]], batch_size: Some(10), }) @@ -4201,6 +4435,8 @@ mod tests { columns: vec!["row_version".to_string(), "created_at".to_string()], column_types: vec![Some("timestamp".to_string()), Some("datetime2(3)".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!("2026-06-12T10:11:12Z"), json!("2026-06-12T10:11:12.1234567Z")]], batch_size: Some(10), }) @@ -4225,6 +4461,8 @@ mod tests { columns: vec!["id".to_string(), "title".to_string(), "search_vector".to_string()], column_types: vec![Some("integer".to_string()), Some("text".to_string()), Some("tsvector".to_string())], column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Hello"), json!("'hello':1A")]], batch_size: Some(10), }) @@ -4271,6 +4509,8 @@ mod tests { Some("stored generated".to_string()), Some("DEFAULT_GENERATED".to_string()), ], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(7), json!(2), json!(3.5), json!(7.0), json!(7.0), json!("2026-07-30 08:00:00")]], truncated: false, }], @@ -4336,6 +4576,8 @@ mod tests { columns: vec!["ID".to_string(), "NAME".to_string()], column_types: vec![Some("INT".to_string()), Some("VARCHAR(20)".to_string())], column_extras: vec![Some("identity".to_string()), None], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")]], batch_size: Some(10), }) @@ -4365,6 +4607,8 @@ mod tests { columns: vec!["id".to_string()], column_types: Vec::new(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1)]], truncated: true, }], @@ -4721,6 +4965,8 @@ mod tests { columns: vec!["id".to_string(), "event_type".to_string()], column_types: vec![None, None], column_extras: vec![None, None], + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("login")]], batch_size: Some(100), }) diff --git a/crates/dbx-core/src/query_result_export.rs b/crates/dbx-core/src/query_result_export.rs index 3971dc77c8..11c14da4b4 100644 --- a/crates/dbx-core/src/query_result_export.rs +++ b/crates/dbx-core/src/query_result_export.rs @@ -24,6 +24,7 @@ use crate::query_result_sql::{ }; use crate::table_export::TableExportProgress; use crate::transfer::keyset_pagination_sql; +use crate::types::SpatialColumn; use crate::xlsx_export::{ finish_streaming_xlsx_workbook, start_streaming_xlsx_workbook_with_options, StreamingXlsxWriter, XlsxWorksheetData, }; @@ -297,8 +298,10 @@ struct SqlInsertWriter { file: Option>, target: Option, pending_rows: Vec>, + pending_spatial_values: Vec>>, columns: Vec, column_types: Vec>, + spatial_columns: Vec, database_type: DatabaseType, schema: Option, table_name: String, @@ -323,8 +326,10 @@ impl SqlInsertWriter { file: Some(file), target: Some(target), pending_rows: Vec::new(), + pending_spatial_values: Vec::new(), columns: Vec::new(), column_types: Vec::new(), + spatial_columns: Vec::new(), database_type: request.database_type, schema: request.schema.clone(), table_name, @@ -339,14 +344,17 @@ impl SqlInsertWriter { &mut self, columns: Vec, result_column_types: &[String], + spatial_columns: &[SpatialColumn], request: &QueryResultExportRequest, ) { self.column_types = sql_insert_column_types(request, result_column_types); + self.spatial_columns = spatial_columns.to_vec(); self.columns = columns; } - fn write_row(&mut self, row: Vec) -> Result<(), String> { + fn write_row(&mut self, row: Vec, spatial_values: Option>>) -> Result<(), String> { self.pending_rows.push(row); + self.pending_spatial_values.push(spatial_values.unwrap_or_default()); if self.pending_rows.len() >= SQL_INSERT_BATCH_SIZE { self.flush_batch()?; } @@ -366,6 +374,8 @@ impl SqlInsertWriter { columns: self.columns.clone(), column_types: self.column_types.clone(), column_extras: Vec::new(), + spatial_columns: self.spatial_columns.clone(), + spatial_values: mem::take(&mut self.pending_spatial_values), rows: mem::take(&mut self.pending_rows), batch_size: Some(SQL_INSERT_BATCH_SIZE), })?; @@ -857,7 +867,7 @@ async fn export_query_result_core_inner( columns = result.columns.clone(); column_types = result.column_types.clone(); if let Some(writer) = sql_writer.as_mut() { - writer.set_columns(columns.clone(), &column_types, request); + writer.set_columns(columns.clone(), &column_types, &result.spatial_columns, request); } } let fetched_row_count = result.rows.len(); @@ -886,8 +896,8 @@ async fn export_query_result_core_inner( } } else if format == "sql" { let writer = sql_writer.as_mut().ok_or_else(|| "SQL export writer missing".to_string())?; - for row in formatted_rows.into_owned() { - writer.write_row(row)?; + for (row_index, row) in formatted_rows.into_owned().into_iter().enumerate() { + writer.write_row(row, result.spatial_values.get(row_index).cloned())?; } } else { if xlsx.is_none() { @@ -1060,7 +1070,7 @@ async fn try_export_postgres_query_result_stream( columns = stream_columns; temporal_column_types = column_types.clone(); if let Some(writer) = sql_writer.as_mut() { - writer.set_columns(columns.clone(), &column_types, request); + writer.set_columns(columns.clone(), &column_types, &[], request); } else if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1082,7 +1092,7 @@ async fn try_export_postgres_query_result_stream( request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted.into_owned())?; + writer.write_row(formatted.into_owned(), None)?; } else if let Some(file) = text_file.as_mut() { write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { @@ -1304,7 +1314,7 @@ async fn try_export_mysql_query_result_stream( columns = stream_columns; temporal_column_types = column_types.clone(); if let Some(writer) = sql_writer.as_mut() { - writer.set_columns(columns.clone(), &column_types, request); + writer.set_columns(columns.clone(), &column_types, &[], request); } else if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1326,7 +1336,7 @@ async fn try_export_mysql_query_result_stream( request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted.into_owned())?; + writer.write_row(formatted.into_owned(), None)?; } else if let Some(file) = text_file.as_mut() { write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { @@ -1519,7 +1529,7 @@ async fn try_export_clickhouse_query_result_stream( columns = stream_columns; temporal_column_types = column_types.clone(); if let Some(writer) = sql_writer.as_mut() { - writer.set_columns(columns.clone(), &column_types, request); + writer.set_columns(columns.clone(), &column_types, &[], request); } else if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1541,7 +1551,7 @@ async fn try_export_clickhouse_query_result_stream( request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted.into_owned())?; + writer.write_row(formatted.into_owned(), None)?; } else if let Some(file) = text_file.as_mut() { write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { @@ -1703,7 +1713,7 @@ async fn try_export_sqlserver_query_result_stream( columns = stream_columns.to_vec(); temporal_column_types = column_types.to_vec(); if let Some(writer) = sql_writer.as_mut() { - writer.set_columns(columns.clone(), &temporal_column_types, request); + writer.set_columns(columns.clone(), &temporal_column_types, &[], request); } else if let Some(file) = text_file.as_mut() { let header = format_text_export_header(format, &columns); file.write_all(header.as_bytes()).map_err(|e| format!("Failed to write export header: {e}"))?; @@ -1721,7 +1731,7 @@ async fn try_export_sqlserver_query_result_stream( request.date_time_format.as_deref(), ); if let Some(writer) = sql_writer.as_mut() { - writer.write_row(formatted.into_owned())?; + writer.write_row(formatted.into_owned(), None)?; } else if let Some(file) = text_file.as_mut() { write_text_export_row(file, format, formatted.as_ref(), &mut text_buffer)?; } else if let Some(writer) = xlsx.as_mut() { diff --git a/crates/dbx-core/src/table_export.rs b/crates/dbx-core/src/table_export.rs index 18508f4cdb..190c35990c 100644 --- a/crates/dbx-core/src/table_export.rs +++ b/crates/dbx-core/src/table_export.rs @@ -1213,6 +1213,8 @@ async fn try_export_native_table_stream( columns: col_names.to_vec(), column_types: column_types.to_vec(), column_extras: column_extras.to_vec(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: std::mem::take(pending_rows), batch_size: Some(SQL_INSERT_BATCH_SIZE), })?; @@ -2022,6 +2024,8 @@ async fn export_table_data_core_inner( columns: col_names.clone(), column_types: column_types.clone(), column_extras: column_extras.clone(), + spatial_columns: result.spatial_columns.clone(), + spatial_values: result.spatial_values.clone(), rows: result.rows.clone(), batch_size: Some(100), })?; @@ -2801,6 +2805,8 @@ mod tests { columns, column_types, column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: vec![vec![json!(1), json!("Ada")]], batch_size: Some(100), }) diff --git a/crates/dbx-core/tests/live_postgres_tsvector.rs b/crates/dbx-core/tests/live_postgres_tsvector.rs index 379e73469c..52cf046522 100644 --- a/crates/dbx-core/tests/live_postgres_tsvector.rs +++ b/crates/dbx-core/tests/live_postgres_tsvector.rs @@ -93,6 +93,8 @@ async fn postgres_tsvector_generated_columns_are_readable_and_omitted_from_inser columns: result.columns.clone(), column_types: result.column_types.iter().map(|value| Some(value.clone())).collect(), column_extras: Vec::new(), + spatial_columns: Vec::new(), + spatial_values: Vec::new(), rows: result.rows.clone(), batch_size: Some(10), })