busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: restore legacy data and add typed task result views

Browse Source
This commit is contained in:
busya
2026-02-21 23:17:56 +03:00
parent 0cf0ef25f1
commit 6ffdf5f8a4
10 changed files with 411 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

91
backend/src/plugins/backup.py
View File

@@ -154,10 +154,10 @@ class BackupPlugin(PluginBase):
log.info(f"Starting backup for environment: {env}")
try:
config_manager = get_config_manager()
if not config_manager.has_environments():
raise ValueError("No Superset environments configured. Please add an environment in Settings.")
try:
config_manager = get_config_manager()
if not config_manager.has_environments():
raise ValueError("No Superset environments configured. Please add an environment in Settings.")
env_config = config_manager.get_environment(env)
if not env_config:
@@ -180,16 +180,27 @@ class BackupPlugin(PluginBase):
superset_log.info("No dashboard filter applied - backing up all dashboards")
dashboard_meta = all_dashboard_meta
if dashboard_count == 0:
log.info("No dashboards to back up")
return
total = len(dashboard_meta)
for idx, db in enumerate(dashboard_meta, 1):
dashboard_id = db.get('id')
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
if not dashboard_id:
continue
if dashboard_count == 0:
log.info("No dashboards to back up")
return {
"status": "NO_DASHBOARDS",
"environment": env,
"backup_root": str(backup_path / env.upper()),
"total_dashboards": 0,
"backed_up_dashboards": 0,
"failed_dashboards": 0,
"dashboards": [],
"failures": []
}
total = len(dashboard_meta)
backed_up_dashboards = []
failed_dashboards = []
for idx, db in enumerate(dashboard_meta, 1):
dashboard_id = db.get('id')
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
if not dashboard_id:
continue
# Report progress
progress_pct = (idx / total) * 100
@@ -210,21 +221,41 @@ class BackupPlugin(PluginBase):
unpack=False
)
archive_exports(str(dashboard_dir), policy=RetentionPolicy())
storage_log.debug(f"Archived dashboard: {dashboard_title}")
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
log.error(f"Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}")
continue
consolidate_archive_folders(backup_path / env.upper())
remove_empty_directories(str(backup_path / env.upper()))
log.info(f"Backup completed successfully for {env}")
except (RequestException, IOError, KeyError) as e:
log.error(f"Fatal error during backup for {env}: {e}")
raise e
archive_exports(str(dashboard_dir), policy=RetentionPolicy())
storage_log.debug(f"Archived dashboard: {dashboard_title}")
backed_up_dashboards.append({
"id": dashboard_id,
"title": dashboard_title,
"path": str(dashboard_dir)
})
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
log.error(f"Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}")
failed_dashboards.append({
"id": dashboard_id,
"title": dashboard_title,
"error": str(db_error)
})
continue
consolidate_archive_folders(backup_path / env.upper())
remove_empty_directories(str(backup_path / env.upper()))
log.info(f"Backup completed successfully for {env}")
return {
"status": "SUCCESS" if not failed_dashboards else "PARTIAL_SUCCESS",
"environment": env,
"backup_root": str(backup_path / env.upper()),
"total_dashboards": total,
"backed_up_dashboards": len(backed_up_dashboards),
"failed_dashboards": len(failed_dashboards),
"dashboards": backed_up_dashboards,
"failures": failed_dashboards
}
except (RequestException, IOError, KeyError) as e:
log.error(f"Fatal error during backup for {env}: {e}")
raise e
# [/DEF:execute:Function]
# [/DEF:BackupPlugin:Class]
# [/DEF:BackupPlugin:Module]
# [/DEF:BackupPlugin:Module]

147
backend/src/plugins/migration.py
View File

@@ -165,11 +165,11 @@ class MigrationPlugin(PluginBase):
superset_log = log.with_source("superset_api") if context else log
migration_log = log.with_source("migration") if context else log
log.info("Starting migration task.")
log.debug(f"Params: {params}")
try:
with belief_scope("execute"):
log.info("Starting migration task.")
log.debug(f"Params: {params}")
try:
with belief_scope("execute"):
config_manager = get_config_manager()
environments = config_manager.get_environments()
@@ -192,11 +192,20 @@ class MigrationPlugin(PluginBase):
from_env_name = src_env.name
to_env_name = tgt_env.name
log.info(f"Resolved environments: {from_env_name} -> {to_env_name}")
from_c = SupersetClient(src_env)
to_c = SupersetClient(tgt_env)
log.info(f"Resolved environments: {from_env_name} -> {to_env_name}")
migration_result = {
"status": "SUCCESS",
"source_environment": from_env_name,
"target_environment": to_env_name,
"selected_dashboards": 0,
"migrated_dashboards": [],
"failed_dashboards": [],
"mapping_count": 0
}
from_c = SupersetClient(src_env)
to_c = SupersetClient(tgt_env)
if not from_c or not to_c:
raise ValueError(f"Clients not initialized for environments: {from_env_name}, {to_env_name}")
@@ -204,20 +213,24 @@ class MigrationPlugin(PluginBase):
_, all_dashboards = from_c.get_dashboards()
dashboards_to_migrate = []
if selected_ids:
dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids]
elif dashboard_regex:
regex_str = str(dashboard_regex)
dashboards_to_migrate = [
if selected_ids:
dashboards_to_migrate = [d for d in all_dashboards if d["id"] in selected_ids]
elif dashboard_regex:
regex_str = str(dashboard_regex)
dashboards_to_migrate = [
d for d in all_dashboards if re.search(regex_str, d["dashboard_title"], re.IGNORECASE)
]
else:
log.warning("No selection criteria provided (selected_ids or dashboard_regex).")
return
if not dashboards_to_migrate:
log.warning("No dashboards found matching criteria.")
return
else:
log.warning("No selection criteria provided (selected_ids or dashboard_regex).")
migration_result["status"] = "NO_SELECTION"
return migration_result
if not dashboards_to_migrate:
log.warning("No dashboards found matching criteria.")
migration_result["status"] = "NO_MATCHES"
return migration_result
migration_result["selected_dashboards"] = len(dashboards_to_migrate)
# Get mappings from params
db_mapping = params.get("db_mappings", {})
@@ -238,17 +251,18 @@ class MigrationPlugin(PluginBase):
DatabaseMapping.target_env_id == tgt_env_db.id
).all()
# Provided mappings override stored ones
stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}
stored_map_dict.update(db_mapping)
db_mapping = stored_map_dict
log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
finally:
db.close()
engine = MigrationEngine()
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
stored_map_dict = {m.source_db_uuid: m.target_db_uuid for m in stored_mappings}
stored_map_dict.update(db_mapping)
db_mapping = stored_map_dict
log.info(f"Loaded {len(stored_mappings)} database mappings from database.")
finally:
db.close()
migration_result["mapping_count"] = len(db_mapping)
engine = MigrationEngine()
for dash in dashboards_to_migrate:
dash_id, dash_slug, title = dash["id"], dash.get("slug"), dash["dashboard_title"]
try:
exported_content, _ = from_c.export_dashboard(dash_id)
@@ -279,13 +293,22 @@ class MigrationPlugin(PluginBase):
db.close()
success = engine.transform_zip(str(tmp_zip_path), str(tmp_new_zip), db_mapping, strip_databases=False)
if success:
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
else:
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
superset_log.info(f"Dashboard {title} imported.")
except Exception as exc:
if success:
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
migration_result["migrated_dashboards"].append({
"id": dash_id,
"title": title
})
else:
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
migration_result["failed_dashboards"].append({
"id": dash_id,
"title": title,
"error": "Failed to transform ZIP"
})
superset_log.info(f"Dashboard {title} imported.")
except Exception as exc:
# Check for password error
error_msg = str(exc)
# The error message from Superset is often a JSON string inside a string.
@@ -324,22 +347,34 @@ class MigrationPlugin(PluginBase):
passwords = task.params.get("passwords", {})
# Retry import with password
if passwords:
app_logger.info(f"[MigrationPlugin][Action] Retrying import for {title} with provided passwords.")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
app_logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported after password injection.")
# Clear passwords from params after use for security
if "passwords" in task.params:
del task.params["passwords"]
continue
app_logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
app_logger.info("[MigrationPlugin][Exit] Migration finished.")
except Exception as e:
app_logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
raise e
if passwords:
app_logger.info(f"[MigrationPlugin][Action] Retrying import for {title} with provided passwords.")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
app_logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported after password injection.")
migration_result["migrated_dashboards"].append({
"id": dash_id,
"title": title
})
# Clear passwords from params after use for security
if "passwords" in task.params:
del task.params["passwords"]
continue
app_logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
migration_result["failed_dashboards"].append({
"id": dash_id,
"title": title,
"error": str(exc)
})
app_logger.info("[MigrationPlugin][Exit] Migration finished.")
if migration_result["failed_dashboards"]:
migration_result["status"] = "PARTIAL_SUCCESS"
return migration_result
except Exception as e:
app_logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
raise e
# [/DEF:MigrationPlugin.execute:Action]
# [/DEF:execute:Function]
# [/DEF:MigrationPlugin:Class]
# [/DEF:MigrationPlugin:Module]
# [/DEF:MigrationPlugin:Module]