busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Новый экранчик для обзора дашей

Browse Source
This commit is contained in:
busya
2026-02-23 15:54:20 +03:00
parent 0685f50ae7
commit 43dd97ecbf
35 changed files with 32880 additions and 828 deletions
Download Patch File Download Diff File Expand all files Collapse all files

297
backend/src/core/superset_client.py
View File

@@ -11,6 +11,7 @@
# [SECTION: IMPORTS]
import json
import re
import zipfile
from pathlib import Path
from typing import Dict, List, Optional, Tuple, Union, cast
@@ -120,6 +121,252 @@ class SupersetClient:
return result
# [/DEF:get_dashboards_summary:Function]
# [DEF:get_dashboard:Function]
# @PURPOSE: Fetches a single dashboard by ID.
# @PRE: Client is authenticated and dashboard_id exists.
# @POST: Returns dashboard payload from Superset API.
# @RETURN: Dict
def get_dashboard(self, dashboard_id: int) -> Dict:
with belief_scope("SupersetClient.get_dashboard", f"id={dashboard_id}"):
response = self.network.request(method="GET", endpoint=f"/dashboard/{dashboard_id}")
return cast(Dict, response)
# [/DEF:get_dashboard:Function]
# [DEF:get_chart:Function]
# @PURPOSE: Fetches a single chart by ID.
# @PRE: Client is authenticated and chart_id exists.
# @POST: Returns chart payload from Superset API.
# @RETURN: Dict
def get_chart(self, chart_id: int) -> Dict:
with belief_scope("SupersetClient.get_chart", f"id={chart_id}"):
response = self.network.request(method="GET", endpoint=f"/chart/{chart_id}")
return cast(Dict, response)
# [/DEF:get_chart:Function]
# [DEF:get_dashboard_detail:Function]
# @PURPOSE: Fetches detailed dashboard information including related charts and datasets.
# @PRE: Client is authenticated and dashboard_id exists.
# @POST: Returns dashboard metadata with charts and datasets lists.
# @RETURN: Dict
def get_dashboard_detail(self, dashboard_id: int) -> Dict:
with belief_scope("SupersetClient.get_dashboard_detail", f"id={dashboard_id}"):
dashboard_response = self.get_dashboard(dashboard_id)
dashboard_data = dashboard_response.get("result", dashboard_response)
charts: List[Dict] = []
datasets: List[Dict] = []
def extract_dataset_id_from_form_data(form_data: Optional[Dict]) -> Optional[int]:
if not isinstance(form_data, dict):
return None
datasource = form_data.get("datasource")
if isinstance(datasource, str):
matched = re.match(r"^(\d+)__", datasource)
if matched:
try:
return int(matched.group(1))
except ValueError:
return None
if isinstance(datasource, dict):
ds_id = datasource.get("id")
try:
return int(ds_id) if ds_id is not None else None
except (TypeError, ValueError):
return None
ds_id = form_data.get("datasource_id")
try:
return int(ds_id) if ds_id is not None else None
except (TypeError, ValueError):
return None
# Canonical endpoints from Superset OpenAPI:
# /dashboard/{id_or_slug}/charts and /dashboard/{id_or_slug}/datasets.
try:
charts_response = self.network.request(
method="GET",
endpoint=f"/dashboard/{dashboard_id}/charts"
)
charts_payload = charts_response.get("result", []) if isinstance(charts_response, dict) else []
for chart_obj in charts_payload:
if not isinstance(chart_obj, dict):
continue
chart_id = chart_obj.get("id")
if chart_id is None:
continue
form_data = chart_obj.get("form_data")
if isinstance(form_data, str):
try:
form_data = json.loads(form_data)
except Exception:
form_data = {}
dataset_id = extract_dataset_id_from_form_data(form_data) or chart_obj.get("datasource_id")
charts.append({
"id": int(chart_id),
"title": chart_obj.get("slice_name") or chart_obj.get("name") or f"Chart {chart_id}",
"viz_type": (form_data.get("viz_type") if isinstance(form_data, dict) else None),
"dataset_id": int(dataset_id) if dataset_id is not None else None,
"last_modified": chart_obj.get("changed_on"),
"overview": chart_obj.get("description") or (form_data.get("viz_type") if isinstance(form_data, dict) else None) or "Chart",
})
except Exception as e:
app_logger.warning("[get_dashboard_detail][Warning] Failed to fetch dashboard charts: %s", e)
try:
datasets_response = self.network.request(
method="GET",
endpoint=f"/dashboard/{dashboard_id}/datasets"
)
datasets_payload = datasets_response.get("result", []) if isinstance(datasets_response, dict) else []
for dataset_obj in datasets_payload:
if not isinstance(dataset_obj, dict):
continue
dataset_id = dataset_obj.get("id")
if dataset_id is None:
continue
db_payload = dataset_obj.get("database")
db_name = db_payload.get("database_name") if isinstance(db_payload, dict) else None
table_name = dataset_obj.get("table_name") or dataset_obj.get("datasource_name") or dataset_obj.get("name") or f"Dataset {dataset_id}"
schema = dataset_obj.get("schema")
fq_name = f"{schema}.{table_name}" if schema else table_name
datasets.append({
"id": int(dataset_id),
"table_name": table_name,
"schema": schema,
"database": db_name or dataset_obj.get("database_name") or "Unknown",
"last_modified": dataset_obj.get("changed_on"),
"overview": fq_name,
})
except Exception as e:
app_logger.warning("[get_dashboard_detail][Warning] Failed to fetch dashboard datasets: %s", e)
# Fallback: derive chart IDs from layout metadata if dashboard charts endpoint fails.
if not charts:
raw_position_json = dashboard_data.get("position_json")
chart_ids_from_position = set()
if isinstance(raw_position_json, str) and raw_position_json:
try:
parsed_position = json.loads(raw_position_json)
chart_ids_from_position.update(self._extract_chart_ids_from_layout(parsed_position))
except Exception:
pass
elif isinstance(raw_position_json, dict):
chart_ids_from_position.update(self._extract_chart_ids_from_layout(raw_position_json))
raw_json_metadata = dashboard_data.get("json_metadata")
if isinstance(raw_json_metadata, str) and raw_json_metadata:
try:
parsed_metadata = json.loads(raw_json_metadata)
chart_ids_from_position.update(self._extract_chart_ids_from_layout(parsed_metadata))
except Exception:
pass
elif isinstance(raw_json_metadata, dict):
chart_ids_from_position.update(self._extract_chart_ids_from_layout(raw_json_metadata))
app_logger.info(
"[get_dashboard_detail][State] Extracted %s fallback chart IDs from layout (dashboard_id=%s)",
len(chart_ids_from_position),
dashboard_id,
)
for chart_id in sorted(chart_ids_from_position):
try:
chart_response = self.get_chart(int(chart_id))
chart_data = chart_response.get("result", chart_response)
charts.append({
"id": int(chart_id),
"title": chart_data.get("slice_name") or chart_data.get("name") or f"Chart {chart_id}",
"viz_type": chart_data.get("viz_type"),
"dataset_id": chart_data.get("datasource_id"),
"last_modified": chart_data.get("changed_on"),
"overview": chart_data.get("description") or chart_data.get("viz_type") or "Chart",
})
except Exception as e:
app_logger.warning("[get_dashboard_detail][Warning] Failed to resolve fallback chart %s: %s", chart_id, e)
# Backfill datasets from chart datasource IDs.
dataset_ids_from_charts = {
c.get("dataset_id")
for c in charts
if c.get("dataset_id") is not None
}
known_dataset_ids = {d.get("id") for d in datasets}
missing_dataset_ids = [ds_id for ds_id in dataset_ids_from_charts if ds_id not in known_dataset_ids]
for dataset_id in missing_dataset_ids:
try:
dataset_response = self.get_dataset(int(dataset_id))
dataset_data = dataset_response.get("result", dataset_response)
db_payload = dataset_data.get("database")
db_name = db_payload.get("database_name") if isinstance(db_payload, dict) else None
table_name = dataset_data.get("table_name") or f"Dataset {dataset_id}"
schema = dataset_data.get("schema")
fq_name = f"{schema}.{table_name}" if schema else table_name
datasets.append({
"id": int(dataset_id),
"table_name": table_name,
"schema": schema,
"database": db_name or "Unknown",
"last_modified": dataset_data.get("changed_on_utc") or dataset_data.get("changed_on"),
"overview": fq_name,
})
except Exception as e:
app_logger.warning("[get_dashboard_detail][Warning] Failed to resolve dataset %s: %s", dataset_id, e)
unique_charts = {}
for chart in charts:
unique_charts[chart["id"]] = chart
unique_datasets = {}
for dataset in datasets:
unique_datasets[dataset["id"]] = dataset
return {
"id": dashboard_data.get("id", dashboard_id),
"title": dashboard_data.get("dashboard_title") or dashboard_data.get("title") or f"Dashboard {dashboard_id}",
"slug": dashboard_data.get("slug"),
"url": dashboard_data.get("url"),
"description": dashboard_data.get("description") or "",
"last_modified": dashboard_data.get("changed_on_utc") or dashboard_data.get("changed_on"),
"published": dashboard_data.get("published"),
"charts": list(unique_charts.values()),
"datasets": list(unique_datasets.values()),
"chart_count": len(unique_charts),
"dataset_count": len(unique_datasets),
}
# [/DEF:get_dashboard_detail:Function]
# [DEF:_extract_chart_ids_from_layout:Function]
# @PURPOSE: Traverses dashboard layout metadata and extracts chart IDs from common keys.
# @PRE: payload can be dict/list/scalar.
# @POST: Returns a set of chart IDs found in nested structures.
def _extract_chart_ids_from_layout(self, payload: Union[Dict, List, str, int, None]) -> set:
with belief_scope("_extract_chart_ids_from_layout"):
found = set()
def walk(node):
if isinstance(node, dict):
for key, value in node.items():
if key in ("chartId", "chart_id", "slice_id", "sliceId"):
try:
found.add(int(value))
except (TypeError, ValueError):
pass
if key == "id" and isinstance(value, str):
match = re.match(r"^CHART-(\d+)$", value)
if match:
try:
found.add(int(match.group(1)))
except ValueError:
pass
walk(value)
elif isinstance(node, list):
for item in node:
walk(item)
walk(payload)
return found
# [/DEF:_extract_chart_ids_from_layout:Function]
# [DEF:export_dashboard:Function]
# @PURPOSE: Экспортирует дашборд в виде ZIP-архива.
# @PARAM: dashboard_id (int) - ID дашборда для экспорта.
@@ -246,6 +493,15 @@ class SupersetClient:
# @RELATION: CALLS -> self.network.request (for related_objects)
def get_dataset_detail(self, dataset_id: int) -> Dict:
with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"):
def as_bool(value, default=False):
if value is None:
return default
if isinstance(value, bool):
return value
if isinstance(value, str):
return value.strip().lower() in ("1", "true", "yes", "y", "on")
return bool(value)
# Get base dataset info
response = self.get_dataset(dataset_id)
@@ -259,12 +515,15 @@ class SupersetClient:
columns = dataset.get("columns", [])
column_info = []
for col in columns:
col_id = col.get("id")
if col_id is None:
continue
column_info.append({
"id": col.get("id"),
"id": int(col_id),
"name": col.get("column_name"),
"type": col.get("type"),
"is_dttm": col.get("is_dttm", False),
"is_active": col.get("is_active", True),
"is_dttm": as_bool(col.get("is_dttm"), default=False),
"is_active": as_bool(col.get("is_active"), default=True),
"description": col.get("description", "")
})
@@ -286,11 +545,25 @@ class SupersetClient:
dashboards_data = []
for dash in dashboards_data:
linked_dashboards.append({
"id": dash.get("id"),
"title": dash.get("dashboard_title") or dash.get("title", "Unknown"),
"slug": dash.get("slug")
})
if isinstance(dash, dict):
dash_id = dash.get("id")
if dash_id is None:
continue
linked_dashboards.append({
"id": int(dash_id),
"title": dash.get("dashboard_title") or dash.get("title", f"Dashboard {dash_id}"),
"slug": dash.get("slug")
})
else:
try:
dash_id = int(dash)
except (TypeError, ValueError):
continue
linked_dashboards.append({
"id": dash_id,
"title": f"Dashboard {dash_id}",
"slug": None
})
except Exception as e:
app_logger.warning(f"[get_dataset_detail][Warning] Failed to fetch related dashboards: {e}")
linked_dashboards = []
@@ -302,14 +575,18 @@ class SupersetClient:
"id": dataset.get("id"),
"table_name": dataset.get("table_name"),
"schema": dataset.get("schema"),
"database": dataset.get("database", {}).get("database_name", "Unknown"),
"database": (
dataset.get("database", {}).get("database_name", "Unknown")
if isinstance(dataset.get("database"), dict)
else dataset.get("database_name") or "Unknown"
),
"description": dataset.get("description", ""),
"columns": column_info,
"column_count": len(column_info),
"sql": sql,
"linked_dashboards": linked_dashboards,
"linked_dashboard_count": len(linked_dashboards),
"is_sqllab_view": dataset.get("is_sqllab_view", False),
"is_sqllab_view": as_bool(dataset.get("is_sqllab_view"), default=False),
"created_on": dataset.get("created_on"),
"changed_on": dataset.get("changed_on")
}