busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Improve dashboard LLM validation UX and report flow

Browse Source
This commit is contained in:
busya
2026-02-26 17:53:41 +03:00
parent 5ec1254336
commit f4612c0737
10 changed files with 1199 additions and 30 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
backend/src/plugins/llm_analysis/plugin.py
View File

@@ -30,6 +30,34 @@ from ...services.llm_prompt_templates import (
render_prompt,
)
# [DEF:_is_masked_or_invalid_api_key:Function]
# @PURPOSE: Guards against placeholder or malformed API keys in runtime.
# @PRE: value may be None.
# @POST: Returns True when value cannot be used for authenticated provider calls.
def _is_masked_or_invalid_api_key(value: Optional[str]) -> bool:
key = (value or "").strip()
if not key:
return True
if key in {"********", "EMPTY_OR_NONE"}:
return True
# Most provider tokens are significantly longer; short values are almost always placeholders.
return len(key) < 16
# [/DEF:_is_masked_or_invalid_api_key:Function]
# [DEF:_json_safe_value:Function]
# @PURPOSE: Recursively normalize payload values for JSON serialization.
# @PRE: value may be nested dict/list with datetime values.
# @POST: datetime values are converted to ISO strings.
def _json_safe_value(value: Any):
if isinstance(value, datetime):
return value.isoformat()
if isinstance(value, dict):
return {k: _json_safe_value(v) for k, v in value.items()}
if isinstance(value, list):
return [_json_safe_value(v) for v in value]
return value
# [/DEF:_json_safe_value:Function]
# [DEF:DashboardValidationPlugin:Class]
# @PURPOSE: Plugin for automated dashboard health analysis using LLMs.
# @RELATION: IMPLEMENTS -> backend.src.core.plugin_base.PluginBase
@@ -70,6 +98,7 @@ class DashboardValidationPlugin(PluginBase):
# @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database.
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
with belief_scope("execute", f"plugin_id={self.id}"):
validation_started_at = datetime.utcnow()
# Use TaskContext logger if available, otherwise fall back to app logger
log = context.logger if context else logger
@@ -118,11 +147,10 @@ class DashboardValidationPlugin(PluginBase):
llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
# Check if API key was successfully decrypted
if not api_key:
if _is_masked_or_invalid_api_key(api_key):
raise ValueError(
f"Failed to decrypt API key for provider {provider_id}. "
f"The provider may have been encrypted with a different encryption key. "
f"Please update the provider with a new API key through the UI."
f"Invalid API key for provider {provider_id}. "
"Please open LLM provider settings and save a real API key (not masked placeholder)."
)
# 3. Capture Screenshot
@@ -135,12 +163,15 @@ class DashboardValidationPlugin(PluginBase):
filename = f"{dashboard_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
screenshot_path = os.path.join(screenshots_dir, filename)
screenshot_started_at = datetime.utcnow()
screenshot_log.info(f"Capturing screenshot for dashboard {dashboard_id}")
await screenshot_service.capture_dashboard(dashboard_id, screenshot_path)
screenshot_log.debug(f"Screenshot saved to: {screenshot_path}")
screenshot_finished_at = datetime.utcnow()
# 4. Fetch Logs (from Environment /api/v1/log/)
logs = []
logs_fetch_started_at = datetime.utcnow()
try:
client = SupersetClient(env)
@@ -181,6 +212,7 @@ class DashboardValidationPlugin(PluginBase):
except Exception as e:
superset_log.warning(f"Failed to fetch logs from environment: {e}")
logs = [f"Error fetching remote logs: {str(e)}"]
logs_fetch_finished_at = datetime.utcnow()
# 5. Analyze with LLM
llm_client = LLMClient(
@@ -196,11 +228,13 @@ class DashboardValidationPlugin(PluginBase):
"dashboard_validation_prompt",
DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"],
)
llm_call_started_at = datetime.utcnow()
analysis = await llm_client.analyze_dashboard(
screenshot_path,
logs,
prompt_template=dashboard_prompt,
)
llm_call_finished_at = datetime.utcnow()
# Log analysis summary to task logs for better visibility
llm_log.info(f"[ANALYSIS_SUMMARY] Status: {analysis['status']}")
@@ -218,6 +252,35 @@ class DashboardValidationPlugin(PluginBase):
screenshot_path=screenshot_path,
raw_response=str(analysis)
)
validation_finished_at = datetime.utcnow()
result_payload = _json_safe_value(validation_result.dict())
result_payload["screenshot_paths"] = [screenshot_path]
result_payload["logs_sent_to_llm"] = logs
result_payload["logs_sent_count"] = len(logs)
result_payload["prompt_template"] = dashboard_prompt
result_payload["provider"] = {
"id": db_provider.id,
"name": db_provider.name,
"type": db_provider.provider_type,
"base_url": db_provider.base_url,
"model": db_provider.default_model,
}
result_payload["environment_id"] = env_id
result_payload["timings"] = {
"validation_started_at": validation_started_at.isoformat(),
"validation_finished_at": validation_finished_at.isoformat(),
"validation_duration_ms": int((validation_finished_at - validation_started_at).total_seconds() * 1000),
"screenshot_started_at": screenshot_started_at.isoformat(),
"screenshot_finished_at": screenshot_finished_at.isoformat(),
"screenshot_duration_ms": int((screenshot_finished_at - screenshot_started_at).total_seconds() * 1000),
"logs_fetch_started_at": logs_fetch_started_at.isoformat(),
"logs_fetch_finished_at": logs_fetch_finished_at.isoformat(),
"logs_fetch_duration_ms": int((logs_fetch_finished_at - logs_fetch_started_at).total_seconds() * 1000),
"llm_call_started_at": llm_call_started_at.isoformat(),
"llm_call_finished_at": llm_call_finished_at.isoformat(),
"llm_call_duration_ms": int((llm_call_finished_at - llm_call_started_at).total_seconds() * 1000),
}
db_record = ValidationRecord(
dashboard_id=validation_result.dashboard_id,
@@ -225,7 +288,7 @@ class DashboardValidationPlugin(PluginBase):
summary=validation_result.summary,
issues=[issue.dict() for issue in validation_result.issues],
screenshot_path=validation_result.screenshot_path,
raw_response=validation_result.raw_response
raw_response=json.dumps(result_payload, ensure_ascii=False)
)
db.add(db_record)
db.commit()
@@ -240,7 +303,7 @@ class DashboardValidationPlugin(PluginBase):
# Final log to ensure all analysis is visible in task logs
log.info(f"Validation completed for dashboard {dashboard_id}. Status: {validation_result.status.value}")
return validation_result.dict()
return result_payload
finally:
db.close()
@@ -328,11 +391,10 @@ class DocumentationPlugin(PluginBase):
llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
# Check if API key was successfully decrypted
if not api_key:
if _is_masked_or_invalid_api_key(api_key):
raise ValueError(
f"Failed to decrypt API key for provider {provider_id}. "
f"The provider may have been encrypted with a different encryption key. "
f"Please update the provider with a new API key through the UI."
f"Invalid API key for provider {provider_id}. "
"Please open LLM provider settings and save a real API key (not masked placeholder)."
)
# 3. Fetch Metadata (US2 / T024)