busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

linter + новые таски

Browse Source
This commit is contained in:
busya
2026-02-10 12:53:01 +03:00
parent 794cc55fe7
commit 76b98fcf8f
73 changed files with 2298 additions and 726 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
backend/src/plugins/backup.py
View File

@@ -95,7 +95,7 @@ class BackupPlugin(PluginBase):
with belief_scope("get_schema"):
config_manager = get_config_manager()
envs = [e.name for e in config_manager.get_environments()]
default_path = config_manager.get_config().settings.storage.root_path
config_manager.get_config().settings.storage.root_path
return {
"type": "object",

3
backend/src/plugins/git/llm_extension.py
View File

@@ -5,10 +5,9 @@
# @LAYER: Domain
# @RELATION: DEPENDS_ON -> backend.src.plugins.llm_analysis.service.LLMClient
from typing import List, Optional
from typing import List
from tenacity import retry, stop_after_attempt, wait_exponential
from ..llm_analysis.service import LLMClient
from ..llm_analysis.models import LLMProviderType
from ...core.logger import belief_scope, logger
# [DEF:GitLLMExtension:Class]

35
backend/src/plugins/git_plugin.py
View File

@@ -54,7 +54,7 @@ class GitPlugin(PluginBase):
self.config_manager = config_manager
app_logger.info("GitPlugin initialized using shared config_manager.")
return
except:
except Exception:
config_path = "config.json"
self.config_manager = ConfigManager(config_path)
@@ -135,7 +135,7 @@ class GitPlugin(PluginBase):
# @POST: Плагин готов к выполнению задач.
async def initialize(self):
with belief_scope("GitPlugin.initialize"):
logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
app_logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
# [DEF:execute:Function]
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
@@ -246,15 +246,15 @@ class GitPlugin(PluginBase):
# 5. Автоматический staging изменений (не коммит, чтобы юзер мог проверить diff)
try:
repo.git.add(A=True)
logger.info(f"[_handle_sync][Action] Changes staged in git")
app_logger.info("[_handle_sync][Action] Changes staged in git")
except Exception as ge:
logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}")
app_logger.warning(f"[_handle_sync][Action] Failed to stage changes: {ge}")
logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.")
app_logger.info(f"[_handle_sync][Coherence:OK] Dashboard {dashboard_id} synced successfully.")
return {"status": "success", "message": "Dashboard synced and flattened in local repository"}
except Exception as e:
logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}")
app_logger.error(f"[_handle_sync][Coherence:Failed] Sync failed: {e}")
raise
# [/DEF:_handle_sync:Function]
@@ -292,7 +292,8 @@ class GitPlugin(PluginBase):
if ".git" in dirs:
dirs.remove(".git")
for file in files:
if file == ".git" or file.endswith(".zip"): continue
if file == ".git" or file.endswith(".zip"):
continue
file_path = Path(root) / file
# Prepend the root directory name to the archive path
arcname = Path(root_dir_name) / file_path.relative_to(repo_path)
@@ -315,16 +316,16 @@ class GitPlugin(PluginBase):
f.write(zip_buffer.getvalue())
try:
logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}")
app_logger.info(f"[_handle_deploy][Action] Importing dashboard to {env.name}")
result = client.import_dashboard(temp_zip_path)
logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.")
app_logger.info(f"[_handle_deploy][Coherence:OK] Deployment successful for dashboard {dashboard_id}.")
return {"status": "success", "message": f"Dashboard deployed to {env.name}", "details": result}
finally:
if temp_zip_path.exists():
os.remove(temp_zip_path)
except Exception as e:
logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}")
app_logger.error(f"[_handle_deploy][Coherence:Failed] Deployment failed: {e}")
raise
# [/DEF:_handle_deploy:Function]
@@ -336,13 +337,13 @@ class GitPlugin(PluginBase):
# @RETURN: Environment - Объект конфигурации окружения.
def _get_env(self, env_id: Optional[str] = None):
with belief_scope("GitPlugin._get_env"):
logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}")
app_logger.info(f"[_get_env][Entry] Fetching environment for ID: {env_id}")
# Priority 1: ConfigManager (config.json)
if env_id:
env = self.config_manager.get_environment(env_id)
if env:
logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}")
app_logger.info(f"[_get_env][Exit] Found environment by ID in ConfigManager: {env.name}")
return env
# Priority 2: Database (DeploymentEnvironment)
@@ -355,12 +356,12 @@ class GitPlugin(PluginBase):
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.id == env_id).first()
else:
# If no ID, try to find active or any environment in DB
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active == True).first()
db_env = db.query(DeploymentEnvironment).filter(DeploymentEnvironment.is_active).first()
if not db_env:
db_env = db.query(DeploymentEnvironment).first()
if db_env:
logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}")
app_logger.info(f"[_get_env][Exit] Found environment in DB: {db_env.name}")
from src.core.config_models import Environment
# Use token as password for SupersetClient
return Environment(
@@ -382,14 +383,14 @@ class GitPlugin(PluginBase):
# but we have other envs, maybe it's one of them?
env = next((e for e in envs if e.id == env_id), None)
if env:
logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list")
app_logger.info(f"[_get_env][Exit] Found environment {env_id} in ConfigManager list")
return env
if not env_id:
logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}")
app_logger.info(f"[_get_env][Exit] Using first environment from ConfigManager: {envs[0].name}")
return envs[0]
logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
app_logger.error(f"[_get_env][Coherence:Failed] No environments configured (searched config.json and DB). env_id={env_id}")
raise ValueError("No environments configured. Please add a Superset Environment in Settings.")
# [/DEF:_get_env:Function]

2
backend/src/plugins/llm_analysis/__init__.py
View File

@@ -9,4 +9,6 @@ LLM Analysis Plugin for automated dashboard validation and dataset documentation
from .plugin import DashboardValidationPlugin, DocumentationPlugin
__all__ = ['DashboardValidationPlugin', 'DocumentationPlugin']
# [/DEF:backend/src/plugins/llm_analysis/__init__.py:Module]

8
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -10,15 +10,13 @@
# @RELATION: USES -> TaskContext
# @INVARIANT: All LLM interactions must be executed as asynchronous tasks.
from typing import Dict, Any, Optional, List
from typing import Dict, Any, Optional
import os
import json
import logging
from datetime import datetime, timedelta
from ...core.plugin_base import PluginBase
from ...core.logger import belief_scope, logger
from ...core.database import SessionLocal
from ...core.config_manager import ConfigManager
from ...services.llm_provider import LLMProviderService
from ...core.superset_client import SupersetClient
from .service import ScreenshotService, LLMClient
@@ -97,7 +95,7 @@ class DashboardValidationPlugin(PluginBase):
log.error(f"LLM Provider {provider_id} not found")
raise ValueError(f"LLM Provider {provider_id} not found")
llm_log.debug(f"Retrieved provider config:")
llm_log.debug("Retrieved provider config:")
llm_log.debug(f" Provider ID: {db_provider.id}")
llm_log.debug(f" Provider Name: {db_provider.name}")
llm_log.debug(f" Provider Type: {db_provider.provider_type}")
@@ -299,7 +297,7 @@ class DocumentationPlugin(PluginBase):
log.error(f"LLM Provider {provider_id} not found")
raise ValueError(f"LLM Provider {provider_id} not found")
llm_log.debug(f"Retrieved provider config:")
llm_log.debug("Retrieved provider config:")
llm_log.debug(f" Provider ID: {db_provider.id}")
llm_log.debug(f" Provider Name: {db_provider.name}")
llm_log.debug(f" Provider Type: {db_provider.provider_type}")

39
backend/src/plugins/llm_analysis/service.py
View File

@@ -12,12 +12,12 @@ import asyncio
import base64
import json
import io
from typing import List, Optional, Dict, Any
from typing import List, Dict, Any
from PIL import Image
from playwright.async_api import async_playwright
from openai import AsyncOpenAI, RateLimitError, AuthenticationError as OpenAIAuthenticationError
from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception
from .models import LLMProviderType, ValidationResult, ValidationStatus, DetectedIssue
from .models import LLMProviderType
from ...core.logger import belief_scope, logger
from ...core.config_models import Environment
@@ -96,7 +96,7 @@ class ScreenshotService:
"password": ['input[name="password"]', 'input#password', 'input[placeholder*="Password"]', 'input[type="password"]'],
"submit": ['button[type="submit"]', 'button#submit', '.btn-primary', 'input[type="submit"]']
}
logger.info(f"[DEBUG] Attempting to find login form elements...")
logger.info("[DEBUG] Attempting to find login form elements...")
try:
# Find and fill username
@@ -190,27 +190,27 @@ class ScreenshotService:
try:
# Wait for the dashboard grid to be present
await page.wait_for_selector('.dashboard-component, .dashboard-header, [data-test="dashboard-grid"]', timeout=30000)
logger.info(f"[DEBUG] Dashboard container loaded")
logger.info("[DEBUG] Dashboard container loaded")
# Wait for charts to finish loading (Superset uses loading spinners/skeletons)
# We wait until loading indicators disappear or a timeout occurs
try:
# Wait for loading indicators to disappear
await page.wait_for_selector('.loading, .ant-skeleton, .spinner', state="hidden", timeout=60000)
logger.info(f"[DEBUG] Loading indicators hidden")
except:
logger.warning(f"[DEBUG] Timeout waiting for loading indicators to hide")
logger.info("[DEBUG] Loading indicators hidden")
except Exception:
logger.warning("[DEBUG] Timeout waiting for loading indicators to hide")
# Wait for charts to actually render their content (e.g., ECharts, NVD3)
# We look for common chart containers that should have content
try:
await page.wait_for_selector('.chart-container canvas, .slice_container svg, .superset-chart-canvas, .grid-content .chart-container', timeout=60000)
logger.info(f"[DEBUG] Chart content detected")
except:
logger.warning(f"[DEBUG] Timeout waiting for chart content")
logger.info("[DEBUG] Chart content detected")
except Exception:
logger.warning("[DEBUG] Timeout waiting for chart content")
# Additional check: wait for all chart containers to have non-empty content
logger.info(f"[DEBUG] Waiting for all charts to have rendered content...")
logger.info("[DEBUG] Waiting for all charts to have rendered content...")
await page.wait_for_function("""() => {
const charts = document.querySelectorAll('.chart-container, .slice_container');
if (charts.length === 0) return true; // No charts to wait for
@@ -223,10 +223,10 @@ class ScreenshotService:
return hasCanvas || hasSvg || hasContent;
});
}""", timeout=60000)
logger.info(f"[DEBUG] All charts have rendered content")
logger.info("[DEBUG] All charts have rendered content")
# Scroll to bottom and back to top to trigger lazy loading of all charts
logger.info(f"[DEBUG] Scrolling to trigger lazy loading...")
logger.info("[DEBUG] Scrolling to trigger lazy loading...")
await page.evaluate("""async () => {
const delay = ms => new Promise(resolve => setTimeout(resolve, ms));
for (let i = 0; i < document.body.scrollHeight; i += 500) {
@@ -241,7 +241,7 @@ class ScreenshotService:
logger.warning(f"[DEBUG] Dashboard content wait failed: {e}, proceeding anyway after delay")
# Final stabilization delay - increased for complex dashboards
logger.info(f"[DEBUG] Final stabilization delay...")
logger.info("[DEBUG] Final stabilization delay...")
await asyncio.sleep(15)
# Logic to handle tabs and full-page capture
@@ -251,7 +251,8 @@ class ScreenshotService:
processed_tabs = set()
async def switch_tabs(depth=0):
if depth > 3: return # Limit recursion depth
if depth > 3:
return # Limit recursion depth
tab_selectors = [
'.ant-tabs-nav-list .ant-tabs-tab',
@@ -262,7 +263,8 @@ class ScreenshotService:
found_tabs = []
for selector in tab_selectors:
found_tabs = await page.locator(selector).all()
if found_tabs: break
if found_tabs:
break
if found_tabs:
logger.info(f"[DEBUG][TabSwitching] Found {len(found_tabs)} tabs at depth {depth}")
@@ -292,7 +294,8 @@ class ScreenshotService:
if "ant-tabs-tab-active" not in (await first_tab.get_attribute("class") or ""):
await first_tab.click()
await asyncio.sleep(1)
except: pass
except Exception:
pass
await switch_tabs()
@@ -423,7 +426,7 @@ class LLMClient:
self.default_model = default_model
# DEBUG: Log initialization parameters (without exposing full API key)
logger.info(f"[LLMClient.__init__] Initializing LLM client:")
logger.info("[LLMClient.__init__] Initializing LLM client:")
logger.info(f"[LLMClient.__init__] Provider Type: {provider_type}")
logger.info(f"[LLMClient.__init__] Base URL: {base_url}")
logger.info(f"[LLMClient.__init__] Default Model: {default_model}")

22
backend/src/plugins/migration.py
View File

@@ -7,15 +7,13 @@
# @RELATION: DEPENDS_ON -> superset_tool.utils
# @RELATION: USES -> TaskContext
from typing import Dict, Any, List, Optional
from pathlib import Path
import zipfile
from typing import Dict, Any, Optional
import re
from ..core.plugin_base import PluginBase
from ..core.logger import belief_scope, logger as app_logger
from ..core.superset_client import SupersetClient
from ..core.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
from ..core.utils.fileio import create_temp_file
from ..dependencies import get_config_manager
from ..core.migration_engine import MigrationEngine
from ..core.database import SessionLocal
@@ -151,8 +149,8 @@ class MigrationPlugin(PluginBase):
dashboard_regex = params.get("dashboard_regex")
replace_db_config = params.get("replace_db_config", False)
from_db_id = params.get("from_db_id")
to_db_id = params.get("to_db_id")
params.get("from_db_id")
params.get("to_db_id")
# [DEF:MigrationPlugin.execute:Action]
# @PURPOSE: Execute the migration logic with proper task logging.
@@ -301,7 +299,7 @@ class MigrationPlugin(PluginBase):
if match_alt:
db_name = match_alt.group(1)
logger.warning(f"[MigrationPlugin][Action] Detected missing password for database: {db_name}")
app_logger.warning(f"[MigrationPlugin][Action] Detected missing password for database: {db_name}")
if task_id:
input_request = {
@@ -320,19 +318,19 @@ class MigrationPlugin(PluginBase):
# Retry import with password
if passwords:
logger.info(f"[MigrationPlugin][Action] Retrying import for {title} with provided passwords.")
app_logger.info(f"[MigrationPlugin][Action] Retrying import for {title} with provided passwords.")
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug, passwords=passwords)
logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported after password injection.")
app_logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported after password injection.")
# Clear passwords from params after use for security
if "passwords" in task.params:
del task.params["passwords"]
continue
logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
app_logger.error(f"[MigrationPlugin][Failure] Failed to migrate dashboard {title}: {exc}", exc_info=True)
logger.info("[MigrationPlugin][Exit] Migration finished.")
app_logger.info("[MigrationPlugin][Exit] Migration finished.")
except Exception as e:
logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
app_logger.critical(f"[MigrationPlugin][Failure] Fatal error during migration: {e}", exc_info=True)
raise e
# [/DEF:MigrationPlugin.execute:Action]
# [/DEF:execute:Function]

4
backend/src/plugins/search.py
View File

@@ -8,7 +8,7 @@
# [SECTION: IMPORTS]
import re
from typing import Dict, Any, List, Optional
from typing import Dict, Any, Optional
from ..core.plugin_base import PluginBase
from ..core.superset_client import SupersetClient
from ..core.logger import logger, belief_scope
@@ -116,7 +116,7 @@ class SearchPlugin(PluginBase):
log = context.logger if context else logger
# Create sub-loggers for different components
superset_log = log.with_source("superset_api") if context else log
log.with_source("superset_api") if context else log
search_log = log.with_source("search") if context else log
if not env_name or not search_query:

4
backend/src/plugins/storage/plugin.py
View File

@@ -19,7 +19,7 @@ from fastapi import UploadFile
from ...core.plugin_base import PluginBase
from ...core.logger import belief_scope, logger
from ...models.storage import StoredFile, FileCategory, StorageConfig
from ...models.storage import StoredFile, FileCategory
from ...dependencies import get_config_manager
from ...core.task_manager.context import TaskContext
# [/SECTION]
@@ -126,7 +126,7 @@ class StoragePlugin(PluginBase):
# Create sub-loggers for different components
storage_log = log.with_source("storage") if context else log
filesystem_log = log.with_source("filesystem") if context else log
log.with_source("filesystem") if context else log
storage_log.info(f"Executing with params: {params}")
# [/DEF:execute:Function]