399 lines
20 KiB
Python
399 lines
20 KiB
Python
# [DEF:backend.src.plugins.git_plugin:Module]
|
||
#
|
||
# @SEMANTICS: git, plugin, dashboard, version_control, sync, deploy
|
||
# @PURPOSE: Предоставляет плагин для версионирования и развертывания дашбордов Superset.
|
||
# @LAYER: Plugin
|
||
# @RELATION: INHERITS_FROM -> src.core.plugin_base.PluginBase
|
||
# @RELATION: USES -> src.services.git_service.GitService
|
||
# @RELATION: USES -> src.core.superset_client.SupersetClient
|
||
# @RELATION: USES -> src.core.config_manager.ConfigManager
|
||
# @RELATION: USES -> TaskContext
|
||
#
|
||
# @INVARIANT: Все операции с Git должны выполняться через GitService.
|
||
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
|
||
|
||
# [SECTION: IMPORTS]
|
||
import os
|
||
import io
|
||
import shutil
|
||
import zipfile
|
||
from pathlib import Path
|
||
from typing import Dict, Any, Optional
|
||
from src.core.plugin_base import PluginBase
|
||
from src.services.git_service import GitService
|
||
from src.core.logger import logger as app_logger, belief_scope
|
||
from src.core.config_manager import ConfigManager
|
||
from src.core.superset_client import SupersetClient
|
||
from src.core.task_manager.context import TaskContext
|
||
# [/SECTION]
|
||
|
||
# [DEF:GitPlugin:Class]
|
||
# @PURPOSE: Реализация плагина Git Integration для управления версиями дашбордов.
|
||
class GitPlugin(PluginBase):
|
||
|
||
# [DEF:__init__:Function]
|
||
# @PURPOSE: Инициализирует плагин и его зависимости.
|
||
# @PRE: config.json exists or shared config_manager is available.
|
||
# @POST: Инициализированы git_service и config_manager.
|
||
def __init__(self):
|
||
with belief_scope("GitPlugin.__init__"):
|
||
app_logger.info("Initializing GitPlugin.")
|
||
self.git_service = GitService()
|
||
|
||
# Robust config path resolution:
|
||
# 1. Try absolute path from src/dependencies.py style if possible
|
||
# 2. Try relative paths based on common execution patterns
|
||
if os.path.exists("../config.json"):
|
||
config_path = "../config.json"
|
||
elif os.path.exists("config.json"):
|
||
config_path = "config.json"
|
||
else:
|
||
# Fallback to the one initialized in dependencies if we can import it
|
||
try:
|
||
from src.dependencies import config_manager
|
||
self.config_manager = config_manager
|
||
app_logger.info("GitPlugin initialized using shared config_manager.")
|
||
return
|
||
except Exception:
|
||
config_path = "config.json"
|
||
|
||
self.config_manager = ConfigManager(config_path)
|
||
app_logger.info(f"GitPlugin initialized with {config_path}")
|
||
# [/DEF:__init__:Function]
|
||
|
||
@property
|
||
# [DEF:id:Function]
|
||
# @PURPOSE: Returns the plugin identifier.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Returns 'git-integration'.
|
||
def id(self) -> str:
|
||
with belief_scope("GitPlugin.id"):
|
||
return "git-integration"
|
||
# [/DEF:id:Function]
|
||
|
||
@property
|
||
# [DEF:name:Function]
|
||
# @PURPOSE: Returns the plugin name.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Returns the human-readable name.
|
||
def name(self) -> str:
|
||
with belief_scope("GitPlugin.name"):
|
||
return "Git Integration"
|
||
# [/DEF:name:Function]
|
||
|
||
@property
|
||
# [DEF:description:Function]
|
||
# @PURPOSE: Returns the plugin description.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Returns the plugin's purpose description.
|
||
def description(self) -> str:
|
||
with belief_scope("GitPlugin.description"):
|
||
return "Version control for Superset dashboards"
|
||
# [/DEF:description:Function]
|
||
|
||
@property
|
||
# [DEF:version:Function]
|
||
# @PURPOSE: Returns the plugin version.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Returns the version string.
|
||
def version(self) -> str:
|
||
with belief_scope("GitPlugin.version"):
|
||
return "0.1.0"
|
||
# [/DEF:version:Function]
|
||
|
||
@property
|
||
# [DEF:ui_route:Function]
|
||
# @PURPOSE: Returns the frontend route for the git plugin.
|
||
# @RETURN: str - "/git"
|
||
def ui_route(self) -> str:
|
||
with belief_scope("GitPlugin.ui_route"):
|
||
return "/git"
|
||
# [/DEF:ui_route:Function]
|
||
|
||
# [DEF:get_schema:Function]
|
||
# @PURPOSE: Возвращает JSON-схему параметров для выполнения задач плагина.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Returns a JSON schema dictionary.
|
||
# @RETURN: Dict[str, Any] - Схема параметров.
|
||
def get_schema(self) -> Dict[str, Any]:
|
||
with belief_scope("GitPlugin.get_schema"):
|
||
return {
|
||
"type": "object",
|
||
"properties": {
|
||
"operation": {"type": "string", "enum": ["sync", "deploy", "history"]},
|
||
"dashboard_id": {"type": "integer"},
|
||
"environment_id": {"type": "string"},
|
||
"source_env_id": {"type": "string"}
|
||
},
|
||
"required": ["operation", "dashboard_id"]
|
||
}
|
||
# [/DEF:get_schema:Function]
|
||
|
||
# [DEF:initialize:Function]
|
||
# @PURPOSE: Выполняет начальную настройку плагина.
|
||
# @PRE: GitPlugin is initialized.
|
||
# @POST: Плагин готов к выполнению задач.
|
||
async def initialize(self):
|
||
with belief_scope("GitPlugin.initialize"):
|
||
app_logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
|
||
|
||
# [DEF:execute:Function]
|
||
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
|
||
# @PRE: task_data содержит 'operation' и 'dashboard_id'.
|
||
# @POST: Возвращает результат выполнения операции.
|
||
# @PARAM: task_data (Dict[str, Any]) - Данные задачи.
|
||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||
# @RETURN: Dict[str, Any] - Статус и сообщение.
|
||
# @RELATION: CALLS -> self._handle_sync
|
||
# @RELATION: CALLS -> self._handle_deploy
|
||
async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:
|
||
with belief_scope("GitPlugin.execute"):
|
||
operation = task_data.get("operation")
|
||
dashboard_id = task_data.get("dashboard_id")
|
||
|
||
# Use TaskContext logger if available, otherwise fall back to app_logger
|
||
log = context.logger if context else app_logger
|
||
|
||
# Create sub-loggers for different components
|
||
git_log = log.with_source("git") if context else log
|
||
superset_log = log.with_source("superset_api") if context else log
|
||
|
||
log.info(f"Executing operation: {operation} for dashboard {dashboard_id}")
|
||
|
||
if operation == "sync":
|
||
source_env_id = task_data.get("source_env_id")
|
||
result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)
|
||
elif operation == "deploy":
|
||
env_id = task_data.get("environment_id")
|
||
result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)
|
||
elif operation == "history":
|
||
result = {"status": "success", "message": "History available via API"}
|
||
else:
|
||
log.error(f"Unknown operation: {operation}")
|
||
raise ValueError(f"Unknown operation: {operation}")
|
||
|
||
log.info(f"Operation {operation} completed.")
|
||
return result
|
||
# [/DEF:execute:Function]
|
||
|
||
# [DEF:_handle_sync:Function]
|
||
# @PURPOSE: Экспортирует дашборд из Superset и распаковывает в Git-репозиторий.
|
||
# @PRE: Репозиторий для дашборда должен существовать.
|
||
# @POST: Файлы в репозитории обновлены до текущего состояния в Superset.
|
||
# @PARAM: dashboard_id (int) - ID дашборда.
|
||
# @PARAM: source_env_id (Optional[str]) - ID исходного окружения.
|
||
# @RETURN: Dict[str, str] - Результат синхронизации.
|
||
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
|
||
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
|
||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
|
||
async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:
|
||
with belief_scope("GitPlugin._handle_sync"):
|
||
try:
|
||
# 1. Получение репозитория
|
||
repo = self.git_service.get_repo(dashboard_id)
|
||
repo_path = Path(repo.working_dir)
|
||
git_log.info(f"Target repo path: {repo_path}")
|
||
|
||
# 2. Настройка клиента Superset
|
||
env = self._get_env(source_env_id)
|
||
client = SupersetClient(env)
|
||
client.authenticate()
|
||
|
||
# 3. Экспорт дашборда
|
||
superset_log.info(f"Exporting dashboard {dashboard_id} from {env.name}")
|
||
zip_bytes, _ = client.export_dashboard(dashboard_id)
|
||
|
||
# 4. Распаковка с выравниванием структуры (flattening)
|
||
git_log.info(f"Unpacking export to {repo_path}")
|
||
|
||
# Список папок/файлов, которые мы ожидаем от Superset
|
||
managed_dirs = ["dashboards", "charts", "datasets", "databases"]
|
||
managed_files = ["metadata.yaml"]
|
||
|
||
# Очистка старых данных перед распаковкой, чтобы не оставалось "призраков"
|
||
for d in managed_dirs:
|
||
d_path = repo_path / d
|
||
if d_path.exists() and d_path.is_dir():
|
||
shutil.rmtree(d_path)
|
||
for f in managed_files:
|
||
f_path = repo_path / f
|
||
if f_path.exists():
|
||
f_path.unlink()
|
||
|
||
with zipfile.ZipFile(io.BytesIO(zip_bytes)) as zf:
|
||
# Superset экспортирует всё в подпапку dashboard_export_timestamp/
|
||
# Нам нужно найти это имя папки
|
||
namelist = zf.namelist()
|
||
if not namelist:
|
||
raise ValueError("Export ZIP is empty")
|
||
|
||
root_folder = namelist[0].split('/')[0]
|
||
git_log.info(f"Detected root folder in ZIP: {root_folder}")
|
||
|
||
for member in zf.infolist():
|
||
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
|
||
# Убираем префикс папки
|
||
relative_path = member.filename[len(root_folder)+1:]
|
||
target_path = repo_path / relative_path
|
||
|
||
if member.is_dir():
|
||
target_path.mkdir(parents=True, exist_ok=True)
|
||
else:
|
||
target_path.parent.mkdir(parents=True, exist_ok=True)
|
||
with zf.open(member) as source, open(target_path, "wb") as target:
|
||
shutil.copyfileobj(source, target)
|
||
|
||
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
|
||
try:
|
||
repo.git.add(A=True)
|
||
app_logger.info("[_handle_sync][Action] Changes staged in git")
|
||
except Exception as ge:
|
||
app_logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}")
|
||
|
||
app_logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.")
|
||
return {"status": "success", "message": "Dashboard synced and flattened in local repository"}
|
||
|
||
except Exception as e:
|
||
app_logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}")
|
||
raise
|
||
# [/DEF:_handle_sync:Function]
|
||
|
||
# [DEF:_handle_deploy:Function]
|
||
# @PURPOSE: Упаковывает репозиторий в ZIP и импортирует в целевое окружение Superset.
|
||
# @PRE: environment_id должен соответствовать настроенному окружению.
|
||
# @POST: Дашборд импортирован в целевой Superset.
|
||
# @PARAM: dashboard_id (int) - ID дашборда.
|
||
# @PARAM: env_id (str) - ID целевого окружения.
|
||
# @PARAM: log - Main logger instance.
|
||
# @PARAM: git_log - Git-specific logger instance.
|
||
# @PARAM: superset_log - Superset API-specific logger instance.
|
||
# @RETURN: Dict[str, Any] - Результат деплоя.
|
||
# @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.
|
||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
|
||
async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:
|
||
with belief_scope("GitPlugin._handle_deploy"):
|
||
try:
|
||
if not env_id:
|
||
raise ValueError("Target environment ID required for deployment")
|
||
|
||
# 1. Получение репозитория
|
||
repo = self.git_service.get_repo(dashboard_id)
|
||
repo_path = Path(repo.working_dir)
|
||
|
||
# 2. Упаковка в ZIP
|
||
git_log.info(f"Packing repository {repo_path} for deployment.")
|
||
zip_buffer = io.BytesIO()
|
||
|
||
# Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)
|
||
root_dir_name = f"dashboard_export_{dashboard_id}"
|
||
|
||
with zipfile.ZipFile(zip_buffer, "w", zipfile.ZIP_DEFLATED) as zf:
|
||
for root, dirs, files in os.walk(repo_path):
|
||
if ".git" in dirs:
|
||
dirs.remove(".git")
|
||
for file in files:
|
||
if file == ".git" or file.endswith(".zip"):
|
||
continue
|
||
file_path = Path(root) / file
|
||
# Prepend the root directory name to the archive path
|
||
arcname = Path(root_dir_name) / file_path.relative_to(repo_path)
|
||
zf.write(file_path, arcname)
|
||
|
||
zip_buffer.seek(0)
|
||
|
||
# 3. Настройка клиента Superset
|
||
env = self.config_manager.get_environment(env_id)
|
||
if not env:
|
||
raise ValueError(f"Environment {env_id} not found")
|
||
|
||
client = SupersetClient(env)
|
||
client.authenticate()
|
||
|
||
# 4. Импорт
|
||
temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip"
|
||
git_log.info(f"Saving temporary zip to {temp_zip_path}")
|
||
with open(temp_zip_path, "wb") as f:
|
||
f.write(zip_buffer.getvalue())
|
||
|
||
try:
|
||
app_logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}")
|
||
result = client.import_dashboard(temp_zip_path)
|
||
app_logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.")
|
||
return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result}
|
||
finally:
|
||
if temp_zip_path.exists():
|
||
os.remove(temp_zip_path)
|
||
|
||
except Exception as e:
|
||
app_logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}")
|
||
raise
|
||
# [/DEF:_handle_deploy:Function]
|
||
|
||
# [DEF:_get_env:Function]
|
||
# @PURPOSE: Вспомогательный метод для получения конфигурации окружения.
|
||
# @PARAM: env_id (Optional[str]) - ID окружения.
|
||
# @PRE: env_id is a string or None.
|
||
# @POST: Returns an Environment object from config or DB.
|
||
# @RETURN: Environment - Объект конфигурации окружения.
|
||
def _get_env(self, env_id: Optional[str] = None):
|
||
with belief_scope("GitPlugin._get_env"):
|
||
app_logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}")
|
||
|
||
# Priority 1: ConfigManager (config.json)
|
||
if env_id:
|
||
env = self.config_manager.get_environment(env_id)
|
||
if env:
|
||
app_logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}")
|
||
return env
|
||
|
||
# Priority 2: Database (DeploymentEnvironment)
|
||
from src.core.database import SessionLocal
|
||
from src.models.git import DeploymentEnvironment
|
||
|
||
db = SessionLocal()
|
||
try:
|
||
if env_id:
|
||
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
|
||
else:
|
||
# If no ID, try to find active or any environment in DB
|
||
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
|
||
if not db_env:
|
||
db_env = db.query(DeploymentEnvironment).first()
|
||
|
||
if db_env:
|
||
app_logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}")
|
||
from src.core.config_models import Environment
|
||
# Use token as password for SupersetClient
|
||
return Environment(
|
||
id=db_env.id,
|
||
name=db_env.name,
|
||
url=db_env.superset_url,
|
||
username="admin",
|
||
password=db_env.superset_token,
|
||
verify_ssl=True
|
||
)
|
||
finally:
|
||
db.close()
|
||
|
||
# Priority 3: ConfigManager Default (if no env_id provided)
|
||
envs = self.config_manager.get_environments()
|
||
if envs:
|
||
if env_id:
|
||
# If env_id was provided but not found in DB or specifically by ID in config,
|
||
# but we have other envs, maybe it's one of them?
|
||
env = next((e for e in envs if e.id == env_id), None)
|
||
if env:
|
||
app_logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list")
|
||
return env
|
||
|
||
if not env_id:
|
||
app_logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}")
|
||
return envs[0]
|
||
|
||
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
|
||
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
|
||
# [/DEF:_get_env:Function]
|
||
|
||
# [/DEF:initialize:Function]
|
||
# [/DEF:GitPlugin:Class]
|
||
# [/DEF:backend.src.plugins.git_plugin:Module] |