feat(xugu): enable schema-scoped field lineage

This commit is contained in:
Elias
2026-10-12 16:17:01 +08:00
committed by GitHub
parent 2bf6af1ff0
commit d7df7fe3c0
21 changed files with 127 additions and 14 deletions
@@ -320,6 +320,7 @@ watch(
v-if="dialogs.showFieldLineageDialog.value"
v-model:open="dialogs.showFieldLineageDialog.value"
:prefill-connection-id="dialogs.lineagePrefillConnectionId.value"
:prefill-database-type="dialogs.lineagePrefillDatabaseType.value"
:prefill-database="dialogs.lineagePrefillDatabase.value"
:prefill-schema="dialogs.lineagePrefillSchema.value"
:prefill-table="dialogs.lineagePrefillTable.value"
@@ -10,12 +10,14 @@ import { Input } from "@/components/ui/input";
import { useToast } from "@/composables/useToast";
import { useConnectionStore } from "@/stores/connectionStore";
import * as api from "@/lib/backend/api";
import { analyzeFieldLineage, summarizeLineageCounts, type FieldLineageConfidence, type FieldLineageItem, type FieldLineageResult, type FieldLineageTable, type FieldLineageView } from "@/lib/diagram/fieldLineage";
import { analyzeFieldLineage, filterFieldLineageHistory, summarizeLineageCounts, type FieldLineageConfidence, type FieldLineageItem, type FieldLineageResult, type FieldLineageTable, type FieldLineageView } from "@/lib/diagram/fieldLineage";
import type { DatabaseType } from "@/types/database";
import { copyToClipboard } from "@/lib/common/clipboard";
const props = defineProps<{
open: boolean;
prefillConnectionId: string;
prefillDatabaseType?: DatabaseType;
prefillDatabase: string;
prefillSchema?: string;
prefillTable: string;
@@ -58,6 +60,7 @@ const result = ref<FieldLineageResult | null>(null);
const confidenceFilter = ref<"all" | FieldLineageConfidence>("all");
const searchText = ref("");
const copiedId = ref("");
const incompleteCoverage = ref({ skippedTables: 0, skippedViews: 0, omittedTables: 0, omittedViews: 0 });
let runId = 0;
const targetLabel = computed(() => {
@@ -66,6 +69,8 @@ const targetLabel = computed(() => {
});
const counts = computed(() => summarizeLineageCounts(result.value?.items ?? []));
const isXugu = computed(() => props.prefillDatabaseType === "xugu");
const hasIncompleteCoverage = computed(() => Object.values(incompleteCoverage.value).some((count) => count > 0));
const confidenceOptions = computed<Array<{ value: "all" | FieldLineageConfidence; label: string; count: number }>>(() => [
{ value: "all", label: t("lineage.all"), count: result.value?.items.length ?? 0 },
@@ -118,6 +123,7 @@ async function loadLineage() {
cancelled.value = false;
error.value = "";
result.value = null;
incompleteCoverage.value = { skippedTables: 0, skippedViews: 0, omittedTables: 0, omittedViews: 0 };
progressDone.value = 0;
progressTotal.value = 0;
@@ -126,14 +132,26 @@ async function loadLineage() {
if (isStale(currentRun)) return;
const schema = props.prefillSchema || props.prefillDatabase;
const tableInfos = prioritizeTargetTable(await api.listTables(props.prefillConnectionId, props.prefillDatabase, schema), props.prefillTable).slice(0, MAX_TABLES);
const viewInfos = tableInfos.filter((table) => table.table_type.toUpperCase().includes("VIEW")).slice(0, MAX_VIEW_DDLS);
progressTotal.value = tableInfos.length + viewInfos.length + 1;
const listedTables = prioritizeTargetTable(await api.listTables(props.prefillConnectionId, props.prefillDatabase, schema), props.prefillTable);
const tableInfos = listedTables.slice(0, MAX_TABLES);
const listedViews = tableInfos.filter((table) => table.table_type.toUpperCase().includes("VIEW"));
const viewInfos = listedViews.slice(0, MAX_VIEW_DDLS);
// Xugu's list_tables result contains both tables and views. A view is
// scanned below from its definition; sending it through table metadata
// can execute the view as a SELECT and incorrectly report the same broken
// view as both a skipped table and a skipped view.
const tableMetadataInfos = isXugu.value ? tableInfos.filter((table) => !table.table_type.toUpperCase().includes("VIEW")) : tableInfos;
if (isXugu.value) {
incompleteCoverage.value.omittedTables = Math.max(0, listedTables.length - tableInfos.length);
incompleteCoverage.value.omittedViews = Math.max(0, listedViews.length - viewInfos.length);
}
progressTotal.value = tableMetadataInfos.length + viewInfos.length + 1;
const tables: FieldLineageTable[] = [];
for (let i = 0; i < tableInfos.length; i += BATCH_SIZE) {
let skippedTables = 0;
for (let i = 0; i < tableMetadataInfos.length; i += BATCH_SIZE) {
if (isStale(currentRun)) return;
const batch = tableInfos.slice(i, i + BATCH_SIZE);
const batch = tableMetadataInfos.slice(i, i + BATCH_SIZE);
const loaded = await Promise.all(
batch.map(async (table) => {
try {
@@ -146,6 +164,7 @@ async function loadLineage() {
foreignKeys,
};
} catch {
skippedTables++;
return { schema, name: table.name, columns: [], foreignKeys: [] };
}
}),
@@ -155,19 +174,31 @@ async function loadLineage() {
}
const views: FieldLineageView[] = [];
let skippedViews = 0;
for (const view of viewInfos) {
if (isStale(currentRun)) return;
try {
const ddl = await api.getTableDdl(props.prefillConnectionId, props.prefillDatabase, schema, view.name);
const ddl = await api.getTableDdl(props.prefillConnectionId, props.prefillDatabase, schema, view.name, isXugu.value ? "VIEW" : undefined);
views.push({ schema, name: view.name, ddl });
} catch {
skippedViews++;
// Some drivers may not expose view DDL consistently; keep the rest of the lineage usable.
} finally {
progressDone.value++;
}
}
const histories = (await api.loadHistory(200, 0)).filter((entry) => !entry.database || entry.database === props.prefillDatabase).map((entry) => ({ id: entry.id, sql: entry.sql, executed_at: entry.executed_at }));
const historyEntries = await api.loadHistory(200, 0);
const scopedHistory = filterFieldLineageHistory(historyEntries, {
database: props.prefillDatabase,
connectionId: props.prefillConnectionId,
requireConnectionMatch: isXugu.value,
});
const histories = scopedHistory.map((entry) => ({ id: entry.id, sql: entry.sql, executed_at: entry.executed_at }));
if (isXugu.value) {
incompleteCoverage.value.skippedTables = skippedTables;
incompleteCoverage.value.skippedViews = skippedViews;
}
progressDone.value++;
if (isStale(currentRun)) return;
@@ -316,6 +347,19 @@ function openItemTarget(item: FieldLineageItem) {
</div>
<div class="min-h-0 flex-1 overflow-y-auto px-6 py-4">
<div v-if="isXugu" class="mb-3 rounded-md border border-amber-500/30 bg-amber-500/5 p-3 text-xs leading-5 text-muted-foreground">
{{ t("lineage.xuguScopeNotice", { schema: props.prefillSchema || props.prefillDatabase }) }}
</div>
<div v-if="isXugu && hasIncompleteCoverage" class="mb-3 rounded-md border border-amber-500/30 bg-amber-500/5 p-3 text-xs leading-5 text-amber-800 dark:text-amber-200">
{{
t("lineage.xuguPartialCoverage", {
skippedTables: incompleteCoverage.skippedTables,
skippedViews: incompleteCoverage.skippedViews,
omittedTables: incompleteCoverage.omittedTables,
omittedViews: incompleteCoverage.omittedViews,
})
}}
</div>
<div v-if="loading" class="rounded-md border bg-muted/20 p-4">
<div class="flex items-center gap-2 text-sm text-muted-foreground tabular-nums">
<Loader2 class="h-4 w-4 animate-spin" />
@@ -6,7 +6,7 @@ import { useToast } from "@/composables/useToast";
import { rememberExportPassphrase } from "@/lib/backend/exportPassphraseSession";
import { hasSidebarLayoutEntries } from "@/lib/sidebar/sidebarLayout";
import type { ConnectionConfigBundle } from "@/lib/connection/connectionConfigTransfer";
import type { ConnectionConfig, SidebarLayout } from "@/types/database";
import type { ConnectionConfig, DatabaseType, SidebarLayout } from "@/types/database";
const showTransferDialog = ref(false);
const transferTaskId = ref<string | null>(null);
@@ -92,6 +92,7 @@ const tableDataGeneratePrefillDatabase = ref("");
const tableDataGeneratePrefillSchema = ref("");
const tableDataGeneratePrefillTable = ref("");
const lineagePrefillConnectionId = ref("");
const lineagePrefillDatabaseType = ref<DatabaseType>();
const lineagePrefillDatabase = ref("");
const lineagePrefillSchema = ref("");
const lineagePrefillTable = ref("");
@@ -352,6 +353,7 @@ export function useDialogSources() {
(v) => {
if (v) {
lineagePrefillConnectionId.value = v.connectionId;
lineagePrefillDatabaseType.value = v.databaseType;
lineagePrefillDatabase.value = v.database;
lineagePrefillSchema.value = v.schema ?? "";
lineagePrefillTable.value = v.tableName;
@@ -667,6 +669,7 @@ export function useDialogSources() {
tableDataGeneratePrefillSchema,
tableDataGeneratePrefillTable,
lineagePrefillConnectionId,
lineagePrefillDatabaseType,
lineagePrefillDatabase,
lineagePrefillSchema,
lineagePrefillTable,
@@ -200,6 +200,7 @@ export function useSidebarTreeToolRuntime(options: SidebarTreeToolRuntimeOptions
if (node.type !== "column" || !node.connectionId || !node.database || !node.tableName || !column) return;
connectionStore.fieldLineageSource = {
connectionId: node.connectionId,
databaseType: connectionStore.getConfig(node.connectionId)?.db_type,
database: node.database,
schema: node.schema,
tableName: node.tableName,
@@ -70,6 +70,12 @@ const FEATURE_MESSAGES: FeatureMessages[] = [
{ feature: "plugin batch actions", keys: ["pluginPlatform.batchDuplicateSources", "pluginPlatform.batchRefreshFailed"], locales: EXCEPT_AZ_TR },
{ feature: "plugin marketplace sorting (#10078)", keys: under("pluginPlatform", ["sortBy", "sortByName", "sortByRecentlyUpdated", "sortByRecentlyListed", "sortByUpdatesFirst"]), locales: EXCEPT_AZ_TR, translated: true },
{ feature: "shared refresh action (#8768)", keys: ["common.refresh"] },
{
feature: "Xugu field lineage scope and coverage",
keys: under("lineage", ["xuguScopeNotice", "xuguPartialCoverage"]),
locales: ["en", "es", "it", "ja", "ko", "pt-BR", "ru", "zh-CN", "zh-TW"],
translated: true,
},
{ feature: "transfer bulk select", keys: under("transfer", ["bulkSelectObjects", "bulkSelectTitle", "bulkSelectHint", "bulkSelectPlaceholder", "bulkSelectConfirm", "bulkSelectMatched", "bulkSelectUnmatched", "noMatchingObjects"]), translated: true },
{ feature: "transfer background acknowledgement (#8649)", keys: ["transfer.backgroundStarted"], translated: true },
{ feature: "table info pin", keys: ["grid.pinTableInfo", "grid.unpinTableInfo"], translated: true },
+3
View File
@@ -3363,6 +3363,9 @@ export default {
title: "Field Lineage",
open: "View Field Lineage",
loading: "Reading metadata {done}/{total}",
xuguScopeNotice:
"Xugu field lineage is an assistive analysis: it scans tables and views only in the current schema ({schema}); SQL history is limited to entries for this connection found in the latest 200 locally stored records. Same-name fields indicate possible relevance, not a database dependency.",
xuguPartialCoverage: "Results may be incomplete: skipped {skippedTables} tables with metadata read failures and {skippedViews} view definitions; {omittedTables} tables and {omittedViews} views were also omitted by scan limits.",
empty: "No related lineage found",
noFiltered: "No results match the current filters",
targetField: "Current Field",
+3
View File
@@ -3265,6 +3265,9 @@ export default withEnglishFallback({
title: "Linaje de campo",
open: "Ver linaje de campo",
loading: "Leyendo metadatos {done}/{total}",
xuguScopeNotice:
"El linaje de campos de Xugu es un análisis orientativo: solo analiza las tablas y vistas del esquema actual ({schema}); el historial SQL se limita a las entradas de esta conexión entre los 200 registros locales más recientes. Los campos con el mismo nombre pueden estar relacionados, pero no constituyen una dependencia de la base de datos.",
xuguPartialCoverage: "Los resultados pueden estar incompletos: se omitieron {skippedTables} tablas por errores al leer sus metadatos y {skippedViews} definiciones de vista; además, los límites del análisis excluyeron {omittedTables} tablas y {omittedViews} vistas.",
empty: "Sin linaje relacionado",
noFiltered: "Ningún resultado coincide con los filtros actuales",
targetField: "Campo actual",
+3
View File
@@ -3117,6 +3117,9 @@ export default withEnglishFallback({
title: "Lineage dei Campi",
open: "Visualizza Lineage dei Campi",
loading: "Lettura metadati in corso {done}/{total}",
xuguScopeNotice:
"La lineage dei campi Xugu è un'analisi orientativa: esamina solo le tabelle e le viste dello schema corrente ({schema}); la cronologia SQL è limitata alle voci di questa connessione presenti negli ultimi 200 record salvati localmente. I campi con lo stesso nome possono essere correlati, ma non costituiscono una dipendenza del database.",
xuguPartialCoverage: "I risultati potrebbero essere incompleti: sono state ignorate {skippedTables} tabelle per errori di lettura dei metadati e {skippedViews} definizioni di vista; inoltre, i limiti di scansione hanno escluso {omittedTables} tabelle e {omittedViews} viste.",
empty: "Nessun lineage correlato trovato",
noFiltered: "Nessun risultato corrisponde ai filtri correnti",
targetField: "Campo Corrente",
+3
View File
@@ -3141,6 +3141,9 @@ export default withEnglishFallback({
title: "フィールドリネージ",
open: "フィールドリネージを表示",
loading: "メタデータを読み取り中 {done}/{total}",
xuguScopeNotice:
"Xugu のフィールドリネージは参考情報としての分析です。現在のスキーマ({schema})内のテーブルとビューのみをスキャンします。SQL 履歴は、ローカルに保存された最新 200 件のうち、この接続に属する記録に限定されます。同名のフィールドは関連している可能性を示すもので、データベース上の依存関係を意味するものではありません。",
xuguPartialCoverage: "結果が不完全な可能性があります。メタデータの読み取りに失敗したテーブル {skippedTables} 件とビュー定義 {skippedViews} 件をスキップしました。また、スキャン上限によりテーブル {omittedTables} 件とビュー {omittedViews} 件も対象外になりました。",
empty: "関連するリネージが見つかりません",
noFiltered: "現在のフィルターに一致する結果はありません",
targetField: "現在のフィールド",
+2
View File
@@ -3069,6 +3069,8 @@ export default withEnglishFallback({
title: "필드 계보",
open: "필드 계보 보기",
loading: "메타데이터 읽는 중 {done}/{total}",
xuguScopeNotice: "Xugu 필드 계보는 참고용 분석입니다. 현재 스키마({schema})의 테이블과 뷰만 검색합니다. SQL 기록은 로컬에 저장된 최신 200개 항목 중 현재 연결에 해당하는 항목으로 제한됩니다. 이름이 같은 필드는 관련 가능성을 나타낼 뿐, 데이터베이스 종속성을 의미하지는 않습니다.",
xuguPartialCoverage: "결과가 불완전할 수 있습니다. 메타데이터를 읽지 못한 테이블 {skippedTables}개와 뷰 정의 {skippedViews}개를 건너뛰었으며, 검색 한도로 인해 테이블 {omittedTables}개와 뷰 {omittedViews}개도 제외되었습니다.",
empty: "관련 계보를 찾을 수 없습니다",
noFiltered: "현재 필터와 일치하는 결과가 없습니다",
targetField: "현재 필드",
+3
View File
@@ -3117,6 +3117,9 @@ export default withEnglishFallback({
title: "Linhagem de Campo",
open: "Visualizar Linhagem de Campo",
loading: "Lendo metadados {done}/{total}",
xuguScopeNotice:
"A linhagem de campos do Xugu é uma análise de apoio: ela examina apenas as tabelas e visões do esquema atual ({schema}); o histórico SQL se limita às entradas desta conexão entre os 200 registros locais mais recentes. Campos com o mesmo nome podem estar relacionados, mas isso não constitui uma dependência do banco de dados.",
xuguPartialCoverage: "Os resultados podem estar incompletos: {skippedTables} tabelas foram ignoradas devido a falhas na leitura de metadados e {skippedViews} definições de visão não foram lidas; além disso, os limites da análise excluíram {omittedTables} tabelas e {omittedViews} visões.",
empty: "Nenhuma linhagem relacionada encontrada",
noFiltered: "Nenhum resultado corresponde aos filtros atuais",
targetField: "Campo Atual",
+3
View File
@@ -3220,6 +3220,9 @@ export default withEnglishFallback({
title: "Происхождение поля",
open: "Просмотреть происхождение поля",
loading: "Чтение метаданных {done}/{total}",
xuguScopeNotice:
"Происхождение полей Xugu — это вспомогательный анализ: сканируются только таблицы и представления текущей схемы ({schema}); история SQL ограничена записями этого подключения среди последних 200 записей, сохранённых локально. Совпадение имён полей может указывать на связь, но не означает зависимость на уровне базы данных.",
xuguPartialCoverage: "Результаты могут быть неполными: пропущено таблиц с ошибками чтения метаданных — {skippedTables}, определений представлений — {skippedViews}; кроме того, из-за ограничений сканирования не включено таблиц — {omittedTables}, представлений — {omittedViews}.",
empty: "Связанное происхождение не найдено",
noFiltered: "Нет результатов, соответствующих текущим фильтрам",
targetField: "Текущее поле",
+2
View File
@@ -3297,6 +3297,8 @@ export default withEnglishFallback({
title: "字段血缘",
open: "查看字段血缘",
loading: "正在读取元数据 {done}/{total}",
xuguScopeNotice: "虚谷字段血缘是辅助分析:仅扫描当前模式「{schema}」中的表和视图;SQL 历史只取本地最近 200 条记录中属于当前连接的部分。同名字段只表示可能相关,不等于数据库依赖。",
xuguPartialCoverage: "本次结果可能不完整:跳过 {skippedTables} 张元数据读取失败的表、{skippedViews} 个视图定义;另有 {omittedTables} 张表和 {omittedViews} 个视图因扫描上限未纳入。",
empty: "没有找到相关血缘",
noFiltered: "当前筛选条件下没有结果",
targetField: "当前字段",
+2
View File
@@ -3123,6 +3123,8 @@ export default withEnglishFallback({
title: "欄位血緣",
open: "檢視欄位血緣",
loading: "正在讀取 metadata {done}/{total}",
xuguScopeNotice: "虛谷欄位血緣是輔助分析:只掃描目前綱要({schema})中的資料表和檢視;SQL 歷史僅包含本機最近 200 筆記錄中屬於目前連線的項目。同名欄位只表示可能相關,並不代表資料庫相依關係。",
xuguPartialCoverage: "結果可能不完整:因 metadata 讀取失敗而略過 {skippedTables} 個資料表及 {skippedViews} 個檢視定義;另有 {omittedTables} 個資料表及 {omittedViews} 個檢視因掃描上限而未納入。",
empty: "沒有找到相關血緣",
noFiltered: "目前篩選條件下沒有結果",
targetField: "目前欄位",
@@ -16,7 +16,7 @@ describe("databaseDriverManifest", () => {
dataTransfer: true,
sqlFileExecution: true,
databaseCreate: false,
fieldLineage: false,
fieldLineage: true,
sqlExplain: true,
userAdmin: true,
driverManagement: true,
@@ -352,8 +352,10 @@ describe("supportsTableImport", () => {
});
describe("supportsFieldLineage", () => {
it("enables DB2 field lineage", () => {
it("enables field lineage only for databases that advertise support", () => {
expect(supportsFieldLineage("db2")).toBe(true);
expect(supportsFieldLineage("xugu")).toBe(true);
expect(supportsFieldLineage("redis")).toBe(false);
});
});
+19 -1
View File
@@ -1,5 +1,23 @@
import { describe, expect, it } from "vitest";
import { identifierInSql } from "@/lib/diagram/fieldLineage";
import { filterFieldLineageHistory, identifierInSql } from "@/lib/diagram/fieldLineage";
describe("filterFieldLineageHistory", () => {
const history = [
{ id: "same-connection", connection_id: "xugu-1", database: "SYSTEM", sql: "select 1" },
{ id: "other-connection", connection_id: "xugu-2", database: "SYSTEM", sql: "select 2" },
{ id: "other-database", connection_id: "xugu-1", database: "SHOP_DEMO", sql: "select 3" },
{ id: "legacy", database: "SYSTEM", sql: "select 4" },
{ id: "no-database", connection_id: "xugu-1", database: "", sql: "select 5" },
];
it("keeps the existing database-only scope for non-Xugu callers", () => {
expect(filterFieldLineageHistory(history, { database: "SYSTEM", connectionId: "xugu-1" }).map((entry) => entry.id)).toEqual(["same-connection", "other-connection", "legacy", "no-database"]);
});
it("can require an exact connection match for Xugu and excludes legacy unscoped history", () => {
expect(filterFieldLineageHistory(history, { database: "SYSTEM", connectionId: "xugu-1", requireConnectionMatch: true }).map((entry) => entry.id)).toEqual(["same-connection", "no-database"]);
});
});
describe("identifierInSql", () => {
it("matches unquoted identifiers at the start or after punctuation", () => {
@@ -29,6 +29,19 @@ export interface FieldLineageHistory {
executed_at?: string;
}
export interface FieldLineageHistoryEntry {
database?: string;
connection_id?: string;
}
export function filterFieldLineageHistory<T extends FieldLineageHistoryEntry>(entries: T[], options: { database: string; connectionId: string; requireConnectionMatch?: boolean }): T[] {
return entries.filter((entry) => {
if (entry.database && entry.database !== options.database) return false;
if (options.requireConnectionMatch && entry.connection_id !== options.connectionId) return false;
return true;
});
}
export interface FieldLineageItem {
id: string;
kind: FieldLineageKind;
@@ -660,6 +660,7 @@ export const useConnectionStore = defineStore("connection", () => {
} | null>(null);
const fieldLineageSource = ref<{
connectionId: string;
databaseType?: DatabaseType;
database: string;
schema?: string;
tableName: string;
@@ -2540,7 +2540,7 @@
"dataTransfer": true,
"sqlFileExecution": true,
"databaseCreate": false,
"fieldLineage": false,
"fieldLineage": true,
"sqlExplain": true,
"userAdmin": true,
"driverManagement": true
+1 -1
View File
@@ -30,7 +30,7 @@ capabilities:
dataTransfer: true
sqlFileExecution: true
databaseCreate: false
fieldLineage: false
fieldLineage: true
sqlExplain: true
userAdmin: true
driverManagement: true