From 40e6d8cd4c21e42f623aaea2c1b0817e00879ca2 Mon Sep 17 00:00:00 2001 From: busya Date: Mon, 23 Feb 2026 20:20:25 +0300 Subject: [PATCH] chat worked --- .../routes/__tests__/test_assistant_api.py | 23 + backend/src/api/routes/assistant.py | 393 +++++++++++++++++- backend/src/api/routes/git.py | 15 +- backend/src/api/routes/settings.py | 40 +- backend/src/core/config_models.py | 17 +- backend/src/plugins/git/llm_extension.py | 26 +- backend/src/plugins/llm_analysis/plugin.py | 44 +- backend/src/plugins/llm_analysis/service.py | 28 +- .../__tests__/test_llm_prompt_templates.py | 62 +++ backend/src/services/llm_prompt_templates.py | 94 +++++ frontend/src/components/DashboardGrid.svelte | 32 +- .../src/components/MissingMappingModal.svelte | 6 +- .../src/components/git/BranchSelector.svelte | 18 +- .../src/components/git/CommitHistory.svelte | 8 +- .../src/components/git/CommitModal.svelte | 18 +- .../components/git/ConflictResolver.svelte | 10 +- .../src/components/git/DeploymentModal.svelte | 14 +- frontend/src/components/git/GitManager.svelte | 36 +- frontend/src/components/llm/DocPreview.svelte | 18 +- .../src/components/llm/ProviderConfig.svelte | 38 +- .../provider_config.integration.test.js | 44 ++ .../src/components/storage/FileUpload.svelte | 14 +- .../assistant/AssistantChatPanel.svelte | 2 +- .../lib/components/layout/TaskDrawer.svelte | 12 +- .../lib/components/layout/TopNavbar.svelte | 2 +- frontend/src/lib/i18n/locales/en.json | 6 + frontend/src/lib/i18n/locales/ru.json | 6 + .../routes/admin/settings/llm/+page.svelte | 111 ++++- frontend/src/routes/settings/+page.svelte | 92 ++++ 29 files changed, 1033 insertions(+), 196 deletions(-) create mode 100644 backend/src/services/__tests__/test_llm_prompt_templates.py create mode 100644 backend/src/services/llm_prompt_templates.py create mode 100644 frontend/src/components/llm/__tests__/provider_config.integration.test.js diff --git a/backend/src/api/routes/__tests__/test_assistant_api.py b/backend/src/api/routes/__tests__/test_assistant_api.py index 25e9514..6e936d7 100644 --- a/backend/src/api/routes/__tests__/test_assistant_api.py +++ b/backend/src/api/routes/__tests__/test_assistant_api.py @@ -368,4 +368,27 @@ def test_status_query_without_task_id_returns_latest_user_task(): # [/DEF:test_status_query_without_task_id_returns_latest_user_task:Function] +# [DEF:test_llm_validation_missing_dashboard_returns_needs_clarification:Function] +# @PURPOSE: LLM validation command without resolvable dashboard id must request clarification instead of generic failure. +# @PRE: Command intent resolves to run_llm_validation but dashboard id cannot be inferred. +# @POST: Assistant response state is needs_clarification with guidance text. +def test_llm_validation_missing_dashboard_returns_needs_clarification(): + _clear_assistant_state() + response = _run_async( + assistant_module.send_message( + request=assistant_module.AssistantMessageRequest( + message="Я хочу сделать валидацию дашборда test1" + ), + current_user=_admin_user(), + task_manager=_FakeTaskManager(), + config_manager=_FakeConfigManager(), + db=_FakeDb(), + ) + ) + + assert response.state == "needs_clarification" + assert "Укажите" in response.text or "Missing dashboard_id" in response.text + + +# [/DEF:test_llm_validation_missing_dashboard_returns_needs_clarification:Function] # [/DEF:backend.src.api.routes.__tests__.test_assistant_api:Module] diff --git a/backend/src/api/routes/assistant.py b/backend/src/api/routes/assistant.py index dacad81..6efefd9 100644 --- a/backend/src/api/routes/assistant.py +++ b/backend/src/api/routes/assistant.py @@ -9,6 +9,7 @@ from __future__ import annotations +import json import re import uuid from datetime import datetime, timedelta @@ -26,6 +27,9 @@ from ...core.config_manager import ConfigManager from ...core.database import get_db from ...services.git_service import GitService from ...services.llm_provider import LLMProviderService +from ...core.superset_client import SupersetClient +from ...plugins.llm_analysis.service import LLMClient +from ...plugins.llm_analysis.models import LLMProviderType from ...schemas.auth import User from ...models.assistant import ( AssistantAuditRecord, @@ -100,6 +104,17 @@ USER_ACTIVE_CONVERSATION: Dict[str, str] = {} CONFIRMATIONS: Dict[str, ConfirmationRecord] = {} ASSISTANT_AUDIT: Dict[str, List[Dict[str, Any]]] = {} +INTENT_PERMISSION_CHECKS: Dict[str, List[Tuple[str, str]]] = { + "get_task_status": [("tasks", "READ")], + "create_branch": [("plugin:git", "EXECUTE")], + "commit_changes": [("plugin:git", "EXECUTE")], + "deploy_dashboard": [("plugin:git", "EXECUTE")], + "execute_migration": [("plugin:migration", "EXECUTE"), ("plugin:superset-migration", "EXECUTE")], + "run_backup": [("plugin:superset-backup", "EXECUTE"), ("plugin:backup", "EXECUTE")], + "run_llm_validation": [("plugin:llm_dashboard_validation", "EXECUTE")], + "run_llm_documentation": [("plugin:llm_documentation", "EXECUTE")], +} + # [DEF:_append_history:Function] # @PURPOSE: Append conversation message to in-memory history buffer. @@ -387,6 +402,69 @@ def _resolve_provider_id(provider_token: Optional[str], db: Session) -> Optional # [/DEF:_resolve_provider_id:Function] +# [DEF:_get_default_environment_id:Function] +# @PURPOSE: Resolve default environment id from settings or first configured environment. +# @PRE: config_manager returns environments list. +# @POST: Returns default environment id or None when environment list is empty. +def _get_default_environment_id(config_manager: ConfigManager) -> Optional[str]: + configured = config_manager.get_environments() + if not configured: + return None + preferred = None + if hasattr(config_manager, "get_config"): + try: + preferred = config_manager.get_config().settings.default_environment_id + except Exception: + preferred = None + if preferred and any(env.id == preferred for env in configured): + return preferred + explicit_default = next((env.id for env in configured if getattr(env, "is_default", False)), None) + return explicit_default or configured[0].id +# [/DEF:_get_default_environment_id:Function] + + +# [DEF:_resolve_dashboard_id_by_ref:Function] +# @PURPOSE: Resolve dashboard id by title or slug reference in selected environment. +# @PRE: dashboard_ref is a non-empty string-like token. +# @POST: Returns dashboard id when uniquely matched, otherwise None. +def _resolve_dashboard_id_by_ref( + dashboard_ref: Optional[str], + env_id: Optional[str], + config_manager: ConfigManager, +) -> Optional[int]: + if not dashboard_ref or not env_id: + return None + env = next((item for item in config_manager.get_environments() if item.id == env_id), None) + if not env: + return None + + needle = dashboard_ref.strip().lower() + try: + client = SupersetClient(env) + _, dashboards = client.get_dashboards(query={"page_size": 200}) + except Exception as exc: + logger.warning(f"[assistant.dashboard_resolve][failed] ref={dashboard_ref} env={env_id} error={exc}") + return None + + exact = next( + ( + d for d in dashboards + if str(d.get("slug", "")).lower() == needle + or str(d.get("dashboard_title", "")).lower() == needle + or str(d.get("title", "")).lower() == needle + ), + None, + ) + if exact: + return int(exact.get("id")) + + partial = [d for d in dashboards if needle in str(d.get("dashboard_title", d.get("title", ""))).lower()] + if len(partial) == 1 and partial[0].get("id") is not None: + return int(partial[0]["id"]) + return None +# [/DEF:_resolve_dashboard_id_by_ref:Function] + + # [DEF:_parse_command:Function] # @PURPOSE: Deterministically parse RU/EN command text into intent payload. # @PRE: message contains raw user text and config manager resolves environments. @@ -396,6 +474,10 @@ def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any lower = text.lower() dashboard_id = _extract_id(lower, [r"(?:дашборд\w*|dashboard)\s*(?:id\s*)?(\d+)"]) + dashboard_ref = _extract_id( + lower, + [r"(?:дашборд\w*|dashboard)\s*(?:id\s*)?([a-zа-я0-9._-]+)"], + ) dataset_id = _extract_id(lower, [r"(?:датасет\w*|dataset)\s*(?:id\s*)?(\d+)"]) # Accept short and long task ids (e.g., task-1, task-abc123, UUIDs). task_id = _extract_id(lower, [r"(task[-_a-z0-9]{1,}|[0-9a-f]{8}-[0-9a-f-]{27,})"]) @@ -500,6 +582,7 @@ def _parse_command(message: str, config_manager: ConfigManager) -> Dict[str, Any "operation": "run_llm_validation", "entities": { "dashboard_id": int(dashboard_id) if dashboard_id else None, + "dashboard_ref": dashboard_ref if (dashboard_ref and not dashboard_ref.isdigit()) else None, "environment": env_match, "provider": provider_match, }, @@ -553,24 +636,272 @@ def _check_any_permission(current_user: User, checks: List[Tuple[str, str]]): # [/DEF:_check_any_permission:Function] +# [DEF:_has_any_permission:Function] +# @PURPOSE: Check whether user has at least one permission tuple from the provided list. +# @PRE: current_user and checks list are valid. +# @POST: Returns True when at least one permission check passes. +def _has_any_permission(current_user: User, checks: List[Tuple[str, str]]) -> bool: + try: + _check_any_permission(current_user, checks) + return True + except HTTPException: + return False +# [/DEF:_has_any_permission:Function] + + +# [DEF:_build_tool_catalog:Function] +# @PURPOSE: Build current-user tool catalog for LLM planner with operation contracts and defaults. +# @PRE: current_user is authenticated; config/db are available. +# @POST: Returns list of executable tools filtered by permission and runtime availability. +def _build_tool_catalog(current_user: User, config_manager: ConfigManager, db: Session) -> List[Dict[str, Any]]: + envs = config_manager.get_environments() + default_env_id = _get_default_environment_id(config_manager) + providers = LLMProviderService(db).get_all_providers() + active_provider = next((p.id for p in providers if p.is_active), None) + fallback_provider = active_provider or (providers[0].id if providers else None) + + candidates: List[Dict[str, Any]] = [ + { + "operation": "get_task_status", + "domain": "status", + "description": "Get task status by task_id or latest user task", + "required_entities": [], + "optional_entities": ["task_id"], + "risk_level": "safe", + "requires_confirmation": False, + }, + { + "operation": "create_branch", + "domain": "git", + "description": "Create git branch for dashboard", + "required_entities": ["dashboard_id", "branch_name"], + "optional_entities": [], + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "commit_changes", + "domain": "git", + "description": "Commit dashboard repository changes", + "required_entities": ["dashboard_id"], + "optional_entities": ["message"], + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "deploy_dashboard", + "domain": "git", + "description": "Deploy dashboard to target environment", + "required_entities": ["dashboard_id", "environment"], + "optional_entities": [], + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "execute_migration", + "domain": "migration", + "description": "Run dashboard migration between environments", + "required_entities": ["dashboard_id", "source_env", "target_env"], + "optional_entities": [], + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "run_backup", + "domain": "backup", + "description": "Run backup for environment or specific dashboard", + "required_entities": ["environment"], + "optional_entities": ["dashboard_id"], + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "run_llm_validation", + "domain": "llm", + "description": "Run LLM dashboard validation", + "required_entities": ["dashboard_id"], + "optional_entities": ["dashboard_ref", "environment", "provider"], + "defaults": {"environment": default_env_id, "provider": fallback_provider}, + "risk_level": "guarded", + "requires_confirmation": False, + }, + { + "operation": "run_llm_documentation", + "domain": "llm", + "description": "Generate dataset documentation via LLM", + "required_entities": ["dataset_id"], + "optional_entities": ["environment", "provider"], + "defaults": {"environment": default_env_id, "provider": fallback_provider}, + "risk_level": "guarded", + "requires_confirmation": False, + }, + ] + + available: List[Dict[str, Any]] = [] + for tool in candidates: + checks = INTENT_PERMISSION_CHECKS.get(tool["operation"], []) + if checks and not _has_any_permission(current_user, checks): + continue + available.append(tool) + return available +# [/DEF:_build_tool_catalog:Function] + + +# [DEF:_coerce_intent_entities:Function] +# @PURPOSE: Normalize intent entity value types from LLM output to route-compatible values. +# @PRE: intent contains entities dict or missing entities. +# @POST: Returned intent has numeric ids coerced where possible and string values stripped. +def _coerce_intent_entities(intent: Dict[str, Any]) -> Dict[str, Any]: + entities = intent.get("entities") + if not isinstance(entities, dict): + intent["entities"] = {} + entities = intent["entities"] + for key in ("dashboard_id", "dataset_id"): + value = entities.get(key) + if isinstance(value, str) and value.strip().isdigit(): + entities[key] = int(value.strip()) + for key, value in list(entities.items()): + if isinstance(value, str): + entities[key] = value.strip() + return intent +# [/DEF:_coerce_intent_entities:Function] + + +# [DEF:_clarification_text_for_intent:Function] +# @PURPOSE: Convert technical missing-parameter errors into user-facing clarification prompts. +# @PRE: state was classified as needs_clarification for current intent/error combination. +# @POST: Returned text is human-readable and actionable for target operation. +def _clarification_text_for_intent(intent: Optional[Dict[str, Any]], detail_text: str) -> str: + operation = (intent or {}).get("operation") + guidance_by_operation: Dict[str, str] = { + "run_llm_validation": ( + "Нужно уточнение для запуска LLM-валидации: Укажите дашборд (id или slug), окружение и провайдер LLM." + ), + "run_llm_documentation": ( + "Нужно уточнение для генерации документации: Укажите dataset_id, окружение и провайдер LLM." + ), + "create_branch": "Нужно уточнение: укажите dashboard_id и имя ветки.", + "commit_changes": "Нужно уточнение: укажите dashboard_id для коммита.", + "deploy_dashboard": "Нужно уточнение: укажите dashboard_id и целевое окружение.", + "execute_migration": "Нужно уточнение: укажите dashboard_id, source_env и target_env.", + "run_backup": "Нужно уточнение: укажите окружение для бэкапа.", + } + return guidance_by_operation.get(operation, detail_text) +# [/DEF:_clarification_text_for_intent:Function] + + +# [DEF:_plan_intent_with_llm:Function] +# @PURPOSE: Use active LLM provider to select best tool/operation from dynamic catalog. +# @PRE: tools list contains allowed operations for current user. +# @POST: Returns normalized intent dict when planning succeeds; otherwise None. +async def _plan_intent_with_llm( + message: str, + tools: List[Dict[str, Any]], + db: Session, + config_manager: ConfigManager, +) -> Optional[Dict[str, Any]]: + if not tools: + return None + + llm_service = LLMProviderService(db) + providers = llm_service.get_all_providers() + provider = next((p for p in providers if p.is_active), None) + if not provider: + return None + api_key = llm_service.get_decrypted_api_key(provider.id) + if not api_key: + return None + + planner = LLMClient( + provider_type=LLMProviderType(provider.provider_type), + api_key=api_key, + base_url=provider.base_url, + default_model=provider.default_model, + ) + + system_instruction = ( + "You are a deterministic intent planner for backend tools.\n" + "Choose exactly one operation from available_tools or return clarify.\n" + "Output strict JSON object:\n" + "{" + "\"domain\": string, " + "\"operation\": string, " + "\"entities\": object, " + "\"confidence\": number, " + "\"risk_level\": \"safe\"|\"guarded\"|\"dangerous\", " + "\"requires_confirmation\": boolean" + "}\n" + "Rules:\n" + "- Use only operation names from available_tools.\n" + "- If input is ambiguous, operation must be \"clarify\" with low confidence.\n" + "- Keep entities minimal and factual.\n" + ) + payload = { + "available_tools": tools, + "user_message": message, + "known_environments": [{"id": e.id, "name": e.name} for e in config_manager.get_environments()], + } + try: + response = await planner.get_json_completion( + [ + {"role": "system", "content": system_instruction}, + {"role": "user", "content": json.dumps(payload, ensure_ascii=False)}, + ] + ) + except Exception as exc: + logger.warning(f"[assistant.planner][fallback] LLM planner unavailable: {exc}") + return None + if not isinstance(response, dict): + return None + + operation = response.get("operation") + valid_ops = {tool["operation"] for tool in tools} + if operation == "clarify": + return { + "domain": "unknown", + "operation": "clarify", + "entities": {}, + "confidence": float(response.get("confidence", 0.3)), + "risk_level": "safe", + "requires_confirmation": False, + } + if operation not in valid_ops: + return None + + by_operation = {tool["operation"]: tool for tool in tools} + selected = by_operation[operation] + intent = { + "domain": response.get("domain") or selected["domain"], + "operation": operation, + "entities": response.get("entities", {}), + "confidence": float(response.get("confidence", 0.75)), + "risk_level": response.get("risk_level") or selected["risk_level"], + "requires_confirmation": bool(response.get("requires_confirmation", selected["requires_confirmation"])), + } + intent = _coerce_intent_entities(intent) + + defaults = selected.get("defaults") or {} + for key, value in defaults.items(): + if value and not intent["entities"].get(key): + intent["entities"][key] = value + + if operation in {"deploy_dashboard", "execute_migration"}: + env_token = intent["entities"].get("environment") or intent["entities"].get("target_env") + if _is_production_env(env_token, config_manager): + intent["risk_level"] = "dangerous" + intent["requires_confirmation"] = True + return intent +# [/DEF:_plan_intent_with_llm:Function] + + # [DEF:_authorize_intent:Function] # @PURPOSE: Validate user permissions for parsed intent before confirmation/dispatch. # @PRE: intent.operation is present for known assistant command domains. # @POST: Returns if authorized; raises HTTPException(403) when denied. def _authorize_intent(intent: Dict[str, Any], current_user: User): operation = intent.get("operation") - checks_map: Dict[str, List[Tuple[str, str]]] = { - "get_task_status": [("tasks", "READ")], - "create_branch": [("plugin:git", "EXECUTE")], - "commit_changes": [("plugin:git", "EXECUTE")], - "deploy_dashboard": [("plugin:git", "EXECUTE")], - "execute_migration": [("plugin:migration", "EXECUTE"), ("plugin:superset-migration", "EXECUTE")], - "run_backup": [("plugin:superset-backup", "EXECUTE"), ("plugin:backup", "EXECUTE")], - "run_llm_validation": [("plugin:llm_dashboard_validation", "EXECUTE")], - "run_llm_documentation": [("plugin:llm_documentation", "EXECUTE")], - } - if operation in checks_map: - _check_any_permission(current_user, checks_map[operation]) + if operation in INTENT_PERMISSION_CHECKS: + _check_any_permission(current_user, INTENT_PERMISSION_CHECKS[operation]) # [/DEF:_authorize_intent:Function] @@ -708,11 +1039,20 @@ async def _dispatch_intent( if operation == "run_llm_validation": _check_any_permission(current_user, [("plugin:llm_dashboard_validation", "EXECUTE")]) + env_id = _resolve_env_id(entities.get("environment"), config_manager) or _get_default_environment_id(config_manager) dashboard_id = entities.get("dashboard_id") - env_id = _resolve_env_id(entities.get("environment"), config_manager) + if not dashboard_id: + dashboard_id = _resolve_dashboard_id_by_ref( + entities.get("dashboard_ref"), + env_id, + config_manager, + ) provider_id = _resolve_provider_id(entities.get("provider"), db) if not dashboard_id or not env_id or not provider_id: - raise HTTPException(status_code=400, detail="Missing dashboard_id/environment/provider") + raise HTTPException( + status_code=422, + detail="Missing dashboard_id/environment/provider. Укажите ID/slug дашборда или окружение.", + ) task = await task_manager.create_task( plugin_id="llm_dashboard_validation", @@ -782,7 +1122,14 @@ async def send_message( _append_history(user_id, conversation_id, "user", request.message) _persist_message(db, user_id, conversation_id, "user", request.message) - intent = _parse_command(request.message, config_manager) + tools_catalog = _build_tool_catalog(current_user, config_manager, db) + intent = None + try: + intent = await _plan_intent_with_llm(request.message, tools_catalog, db, config_manager) + except Exception as exc: + logger.warning(f"[assistant.planner][fallback] Planner error: {exc}") + if not intent: + intent = _parse_command(request.message, config_manager) confidence = float(intent.get("confidence", 0.0)) if intent.get("domain") == "unknown" or confidence < 0.6: @@ -886,8 +1233,18 @@ async def send_message( created_at=datetime.utcnow(), ) except HTTPException as exc: - state = "denied" if exc.status_code == status.HTTP_403_FORBIDDEN else "failed" - text = str(exc.detail) + detail_text = str(exc.detail) + is_clarification_error = exc.status_code in (400, 422) and ( + detail_text.lower().startswith("missing") + or "укажите" in detail_text.lower() + ) + if exc.status_code == status.HTTP_403_FORBIDDEN: + state = "denied" + elif is_clarification_error: + state = "needs_clarification" + else: + state = "failed" + text = _clarification_text_for_intent(intent, detail_text) if state == "needs_clarification" else detail_text _append_history(user_id, conversation_id, "assistant", text, state=state) _persist_message(db, user_id, conversation_id, "assistant", text, state=state, metadata={"intent": intent}) audit_payload = {"decision": state, "message": request.message, "intent": intent, "error": text} @@ -899,7 +1256,7 @@ async def send_message( state=state, text=text, intent=intent, - actions=[], + actions=[AssistantAction(type="rephrase", label="Rephrase command")] if state == "needs_clarification" else [], created_at=datetime.utcnow(), ) # [/DEF:send_message:Function] diff --git a/backend/src/api/routes/git.py b/backend/src/api/routes/git.py index 9c9fef7..100719d 100644 --- a/backend/src/api/routes/git.py +++ b/backend/src/api/routes/git.py @@ -25,6 +25,7 @@ from src.api.routes.git_schemas import ( ) from src.services.git_service import GitService from src.core.logger import logger, belief_scope +from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, normalize_llm_settings router = APIRouter(tags=["git"]) git_service = GitService() @@ -406,6 +407,7 @@ async def get_repository_diff( async def generate_commit_message( dashboard_id: int, db: Session = Depends(get_db), + config_manager = Depends(get_config_manager), _ = Depends(has_permission("plugin:git", "EXECUTE")) ): with belief_scope("generate_commit_message"): @@ -445,7 +447,16 @@ async def generate_commit_message( # 4. Generate Message from ...plugins.git.llm_extension import GitLLMExtension extension = GitLLMExtension(client) - message = await extension.suggest_commit_message(diff, history) + llm_settings = normalize_llm_settings(config_manager.get_config().settings.llm) + git_prompt = llm_settings["prompts"].get( + "git_commit_prompt", + DEFAULT_LLM_PROMPTS["git_commit_prompt"], + ) + message = await extension.suggest_commit_message( + diff, + history, + prompt_template=git_prompt, + ) return {"message": message} except Exception as e: @@ -453,4 +464,4 @@ async def generate_commit_message( raise HTTPException(status_code=400, detail=str(e)) # [/DEF:generate_commit_message:Function] -# [/DEF:backend.src.api.routes.git:Module] \ No newline at end of file +# [/DEF:backend.src.api.routes.git:Module] diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index fffb92d..02b13f3 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -16,9 +16,10 @@ from pydantic import BaseModel from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig from ...models.storage import StorageConfig from ...dependencies import get_config_manager, has_permission -from ...core.config_manager import ConfigManager -from ...core.logger import logger, belief_scope -from ...core.superset_client import SupersetClient +from ...core.config_manager import ConfigManager +from ...core.logger import logger, belief_scope +from ...core.superset_client import SupersetClient +from ...services.llm_prompt_templates import normalize_llm_settings # [/SECTION] # [DEF:LoggingConfigResponse:Class] @@ -38,13 +39,14 @@ router = APIRouter() # @POST: Returns masked AppConfig. # @RETURN: AppConfig - The current configuration. @router.get("", response_model=AppConfig) -async def get_settings( +async def get_settings( config_manager: ConfigManager = Depends(get_config_manager), _ = Depends(has_permission("admin:settings", "READ")) ): with belief_scope("get_settings"): logger.info("[get_settings][Entry] Fetching all settings") - config = config_manager.get_config().copy(deep=True) + config = config_manager.get_config().copy(deep=True) + config.settings.llm = normalize_llm_settings(config.settings.llm) # Mask passwords for env in config.environments: if env.password: @@ -279,7 +281,7 @@ async def update_logging_config( # [/DEF:update_logging_config:Function] # [DEF:ConsolidatedSettingsResponse:Class] -class ConsolidatedSettingsResponse(BaseModel): +class ConsolidatedSettingsResponse(BaseModel): environments: List[dict] connections: List[dict] llm: dict @@ -294,7 +296,7 @@ class ConsolidatedSettingsResponse(BaseModel): # @POST: Returns all consolidated settings. # @RETURN: ConsolidatedSettingsResponse - All settings categories. @router.get("/consolidated", response_model=ConsolidatedSettingsResponse) -async def get_consolidated_settings( +async def get_consolidated_settings( config_manager: ConfigManager = Depends(get_config_manager), _ = Depends(has_permission("admin:settings", "READ")) ): @@ -323,14 +325,16 @@ async def get_consolidated_settings( finally: db.close() - return ConsolidatedSettingsResponse( - environments=[env.dict() for env in config.environments], - connections=config.settings.connections, - llm=config.settings.llm, - llm_providers=llm_providers_list, - logging=config.settings.logging.dict(), - storage=config.settings.storage.dict() - ) + normalized_llm = normalize_llm_settings(config.settings.llm) + + return ConsolidatedSettingsResponse( + environments=[env.dict() for env in config.environments], + connections=config.settings.connections, + llm=normalized_llm, + llm_providers=llm_providers_list, + logging=config.settings.logging.dict(), + storage=config.settings.storage.dict() + ) # [/DEF:get_consolidated_settings:Function] # [DEF:update_consolidated_settings:Function] @@ -353,9 +357,9 @@ async def update_consolidated_settings( if "connections" in settings_patch: current_settings.connections = settings_patch["connections"] - # Update LLM if provided - if "llm" in settings_patch: - current_settings.llm = settings_patch["llm"] + # Update LLM if provided + if "llm" in settings_patch: + current_settings.llm = normalize_llm_settings(settings_patch["llm"]) # Update Logging if provided if "logging" in settings_patch: diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 8c7307d..80bceb8 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -6,9 +6,10 @@ # @RELATION: READS_FROM -> app_configurations (database) # @RELATION: USED_BY -> ConfigManager -from pydantic import BaseModel, Field -from typing import List, Optional -from ..models.storage import StorageConfig +from pydantic import BaseModel, Field +from typing import List, Optional +from ..models.storage import StorageConfig +from ..services.llm_prompt_templates import DEFAULT_LLM_PROMPTS # [DEF:Schedule:DataClass] # @PURPOSE: Represents a backup schedule configuration. @@ -44,12 +45,18 @@ class LoggingConfig(BaseModel): # [DEF:GlobalSettings:DataClass] # @PURPOSE: Represents global application settings. -class GlobalSettings(BaseModel): +class GlobalSettings(BaseModel): storage: StorageConfig = Field(default_factory=StorageConfig) default_environment_id: Optional[str] = None logging: LoggingConfig = Field(default_factory=LoggingConfig) connections: List[dict] = [] - llm: dict = Field(default_factory=lambda: {"providers": [], "default_provider": ""}) + llm: dict = Field( + default_factory=lambda: { + "providers": [], + "default_provider": "", + "prompts": dict(DEFAULT_LLM_PROMPTS), + } + ) # Task retention settings task_retention_days: int = 30 diff --git a/backend/src/plugins/git/llm_extension.py b/backend/src/plugins/git/llm_extension.py index 079389a..d621269 100644 --- a/backend/src/plugins/git/llm_extension.py +++ b/backend/src/plugins/git/llm_extension.py @@ -9,6 +9,7 @@ from typing import List from tenacity import retry, stop_after_attempt, wait_exponential from ..llm_analysis.service import LLMClient from ...core.logger import belief_scope, logger +from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt # [DEF:GitLLMExtension:Class] # @PURPOSE: Provides LLM capabilities to the Git plugin. @@ -26,21 +27,18 @@ class GitLLMExtension: wait=wait_exponential(multiplier=1, min=2, max=10), reraise=True ) - async def suggest_commit_message(self, diff: str, history: List[str]) -> str: + async def suggest_commit_message( + self, + diff: str, + history: List[str], + prompt_template: str = DEFAULT_LLM_PROMPTS["git_commit_prompt"], + ) -> str: with belief_scope("suggest_commit_message"): history_text = "\n".join(history) - prompt = f""" - Generate a concise and professional git commit message based on the following diff and recent history. - Use Conventional Commits format (e.g., feat: ..., fix: ..., docs: ...). - - Recent History: - {history_text} - - Diff: - {diff} - - Commit Message: - """ + prompt = render_prompt( + prompt_template, + {"history": history_text, "diff": diff}, + ) logger.debug(f"[suggest_commit_message] Calling LLM with model: {self.client.default_model}") response = await self.client.client.chat.completions.create( @@ -63,4 +61,4 @@ class GitLLMExtension: # [/DEF:suggest_commit_message:Function] # [/DEF:GitLLMExtension:Class] -# [/DEF:backend/src/plugins/git/llm_extension:Module] \ No newline at end of file +# [/DEF:backend/src/plugins/git/llm_extension:Module] diff --git a/backend/src/plugins/llm_analysis/plugin.py b/backend/src/plugins/llm_analysis/plugin.py index d3583a5..c260f2c 100644 --- a/backend/src/plugins/llm_analysis/plugin.py +++ b/backend/src/plugins/llm_analysis/plugin.py @@ -23,6 +23,11 @@ from .service import ScreenshotService, LLMClient from .models import LLMProviderType, ValidationStatus, ValidationResult, DetectedIssue from ...models.llm import ValidationRecord from ...core.task_manager.context import TaskContext +from ...services.llm_prompt_templates import ( + DEFAULT_LLM_PROMPTS, + normalize_llm_settings, + render_prompt, +) # [DEF:DashboardValidationPlugin:Class] # @PURPOSE: Plugin for automated dashboard health analysis using LLMs. @@ -181,7 +186,16 @@ class DashboardValidationPlugin(PluginBase): ) llm_log.info(f"Analyzing dashboard {dashboard_id} with LLM") - analysis = await llm_client.analyze_dashboard(screenshot_path, logs) + llm_settings = normalize_llm_settings(config_mgr.get_config().settings.llm) + dashboard_prompt = llm_settings["prompts"].get( + "dashboard_validation_prompt", + DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"], + ) + analysis = await llm_client.analyze_dashboard( + screenshot_path, + logs, + prompt_template=dashboard_prompt, + ) # Log analysis summary to task logs for better visibility llm_log.info(f"[ANALYSIS_SUMMARY] Status: {analysis['status']}") @@ -341,22 +355,18 @@ class DocumentationPlugin(PluginBase): default_model=db_provider.default_model ) - prompt = f""" - Generate professional documentation for the following dataset and its columns. - Dataset: {dataset.get('table_name')} - Columns: {columns_data} - - Provide the documentation in JSON format: - {{ - "dataset_description": "General description of the dataset", - "column_descriptions": [ - {{ - "name": "column_name", - "description": "Generated description" - }} - ] - }} - """ + llm_settings = normalize_llm_settings(config_mgr.get_config().settings.llm) + documentation_prompt = llm_settings["prompts"].get( + "documentation_prompt", + DEFAULT_LLM_PROMPTS["documentation_prompt"], + ) + prompt = render_prompt( + documentation_prompt, + { + "dataset_name": dataset.get("table_name") or "", + "columns_json": json.dumps(columns_data, ensure_ascii=False), + }, + ) # Using a generic chat completion for text-only US2 llm_log.info(f"Generating documentation for dataset {dataset_id}") diff --git a/backend/src/plugins/llm_analysis/service.py b/backend/src/plugins/llm_analysis/service.py index e1e2ab3..b4b78bc 100644 --- a/backend/src/plugins/llm_analysis/service.py +++ b/backend/src/plugins/llm_analysis/service.py @@ -20,6 +20,7 @@ from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_excep from .models import LLMProviderType from ...core.logger import belief_scope, logger from ...core.config_models import Environment +from ...services.llm_prompt_templates import DEFAULT_LLM_PROMPTS, render_prompt # [DEF:ScreenshotService:Class] # @PURPOSE: Handles capturing screenshots of Superset dashboards. @@ -548,7 +549,12 @@ class LLMClient: # @PRE: screenshot_path exists, logs is a list of strings. # @POST: Returns a structured analysis dictionary (status, summary, issues). # @SIDE_EFFECT: Reads screenshot file and calls external LLM API. - async def analyze_dashboard(self, screenshot_path: str, logs: List[str]) -> Dict[str, Any]: + async def analyze_dashboard( + self, + screenshot_path: str, + logs: List[str], + prompt_template: str = DEFAULT_LLM_PROMPTS["dashboard_validation_prompt"], + ) -> Dict[str, Any]: with belief_scope("analyze_dashboard"): # Optimize image to reduce token count (US1 / T023) # Gemini/Gemma models have limits on input tokens, and large images contribute significantly. @@ -582,25 +588,7 @@ class LLMClient: base_64_image = base64.b64encode(image_file.read()).decode('utf-8') log_text = "\n".join(logs) - prompt = f""" - Analyze the attached dashboard screenshot and the following execution logs for health and visual issues. - - Logs: - {log_text} - - Provide the analysis in JSON format with the following structure: - {{ - "status": "PASS" | "WARN" | "FAIL", - "summary": "Short summary of findings", - "issues": [ - {{ - "severity": "WARN" | "FAIL", - "message": "Description of the issue", - "location": "Optional location info (e.g. chart name)" - }} - ] - }} - """ + prompt = render_prompt(prompt_template, {"logs": log_text}) messages = [ { diff --git a/backend/src/services/__tests__/test_llm_prompt_templates.py b/backend/src/services/__tests__/test_llm_prompt_templates.py new file mode 100644 index 0000000..c909669 --- /dev/null +++ b/backend/src/services/__tests__/test_llm_prompt_templates.py @@ -0,0 +1,62 @@ +# [DEF:backend.src.services.__tests__.test_llm_prompt_templates:Module] +# @TIER: STANDARD +# @SEMANTICS: tests, llm, prompts, templates, settings +# @PURPOSE: Validate normalization and rendering behavior for configurable LLM prompt templates. +# @LAYER: Domain Tests +# @RELATION: DEPENDS_ON -> backend.src.services.llm_prompt_templates +# @INVARIANT: All required prompt keys remain available after normalization. + +from src.services.llm_prompt_templates import ( + DEFAULT_LLM_PROMPTS, + normalize_llm_settings, + render_prompt, +) + + +# [DEF:test_normalize_llm_settings_adds_default_prompts:Function] +# @TIER: STANDARD +# @PURPOSE: Ensure legacy/partial llm settings are expanded with all prompt defaults. +# @PRE: Input llm settings do not contain complete prompts object. +# @POST: Returned structure includes required prompt templates with fallback defaults. +def test_normalize_llm_settings_adds_default_prompts(): + normalized = normalize_llm_settings({"default_provider": "x"}) + + assert "prompts" in normalized + assert normalized["default_provider"] == "x" + for key in DEFAULT_LLM_PROMPTS: + assert key in normalized["prompts"] + assert isinstance(normalized["prompts"][key], str) +# [/DEF:test_normalize_llm_settings_adds_default_prompts:Function] + + +# [DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] +# @TIER: STANDARD +# @PURPOSE: Ensure user-customized prompt values are preserved during normalization. +# @PRE: Input llm settings contain custom prompt override. +# @POST: Custom prompt value remains unchanged in normalized output. +def test_normalize_llm_settings_keeps_custom_prompt_values(): + custom = "Doc for {dataset_name} using {columns_json}" + normalized = normalize_llm_settings( + {"prompts": {"documentation_prompt": custom}} + ) + + assert normalized["prompts"]["documentation_prompt"] == custom +# [/DEF:test_normalize_llm_settings_keeps_custom_prompt_values:Function] + + +# [DEF:test_render_prompt_replaces_known_placeholders:Function] +# @TIER: STANDARD +# @PURPOSE: Ensure template placeholders are deterministically replaced. +# @PRE: Template contains placeholders matching provided variables. +# @POST: Rendered prompt string contains substituted values. +def test_render_prompt_replaces_known_placeholders(): + rendered = render_prompt( + "Hello {name}, diff={diff}", + {"name": "bot", "diff": "A->B"}, + ) + + assert rendered == "Hello bot, diff=A->B" +# [/DEF:test_render_prompt_replaces_known_placeholders:Function] + + +# [/DEF:backend.src.services.__tests__.test_llm_prompt_templates:Module] diff --git a/backend/src/services/llm_prompt_templates.py b/backend/src/services/llm_prompt_templates.py new file mode 100644 index 0000000..c446575 --- /dev/null +++ b/backend/src/services/llm_prompt_templates.py @@ -0,0 +1,94 @@ +# [DEF:backend.src.services.llm_prompt_templates:Module] +# @TIER: STANDARD +# @SEMANTICS: llm, prompts, templates, settings +# @PURPOSE: Provide default LLM prompt templates and normalization helpers for runtime usage. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> backend.src.core.config_manager +# @INVARIANT: All required prompt template keys are always present after normalization. + +from __future__ import annotations + +from copy import deepcopy +from typing import Dict, Any + + +# [DEF:DEFAULT_LLM_PROMPTS:Constant] +# @TIER: STANDARD +# @PURPOSE: Default prompt templates used by documentation, dashboard validation, and git commit generation. +DEFAULT_LLM_PROMPTS: Dict[str, str] = { + "dashboard_validation_prompt": ( + "Analyze the attached dashboard screenshot and the following execution logs for health and visual issues.\n\n" + "Logs:\n" + "{logs}\n\n" + "Provide the analysis in JSON format with the following structure:\n" + "{\n" + ' "status": "PASS" | "WARN" | "FAIL",\n' + ' "summary": "Short summary of findings",\n' + ' "issues": [\n' + " {\n" + ' "severity": "WARN" | "FAIL",\n' + ' "message": "Description of the issue",\n' + ' "location": "Optional location info (e.g. chart name)"\n' + " }\n" + " ]\n" + "}" + ), + "documentation_prompt": ( + "Generate professional documentation for the following dataset and its columns.\n" + "Dataset: {dataset_name}\n" + "Columns: {columns_json}\n\n" + "Provide the documentation in JSON format:\n" + "{\n" + ' "dataset_description": "General description of the dataset",\n' + ' "column_descriptions": [\n' + " {\n" + ' "name": "column_name",\n' + ' "description": "Generated description"\n' + " }\n" + " ]\n" + "}" + ), + "git_commit_prompt": ( + "Generate a concise and professional git commit message based on the following diff and recent history.\n" + "Use Conventional Commits format (e.g., feat: ..., fix: ..., docs: ...).\n\n" + "Recent History:\n" + "{history}\n\n" + "Diff:\n" + "{diff}\n\n" + "Commit Message:" + ), +} +# [/DEF:DEFAULT_LLM_PROMPTS:Constant] + + +# [DEF:normalize_llm_settings:Function] +# @TIER: STANDARD +# @PURPOSE: Ensure llm settings contain stable schema with prompts section and default templates. +# @PRE: llm_settings is dictionary-like value or None. +# @POST: Returned dict contains prompts with all required template keys. +def normalize_llm_settings(llm_settings: Any) -> Dict[str, Any]: + normalized: Dict[str, Any] = {"providers": [], "default_provider": "", "prompts": {}} + if isinstance(llm_settings, dict): + normalized.update({k: v for k, v in llm_settings.items() if k in ("providers", "default_provider", "prompts")}) + prompts = normalized.get("prompts") if isinstance(normalized.get("prompts"), dict) else {} + merged_prompts = deepcopy(DEFAULT_LLM_PROMPTS) + merged_prompts.update({k: v for k, v in prompts.items() if isinstance(v, str) and v.strip()}) + normalized["prompts"] = merged_prompts + return normalized +# [/DEF:normalize_llm_settings:Function] + + +# [DEF:render_prompt:Function] +# @TIER: STANDARD +# @PURPOSE: Render prompt template using deterministic placeholder replacement with graceful fallback. +# @PRE: template is a string and variables values are already stringifiable. +# @POST: Returns rendered prompt text with known placeholders substituted. +def render_prompt(template: str, variables: Dict[str, Any]) -> str: + rendered = template + for key, value in variables.items(): + rendered = rendered.replace("{" + key + "}", str(value)) + return rendered +# [/DEF:render_prompt:Function] + + +# [/DEF:backend.src.services.llm_prompt_templates:Module] diff --git a/frontend/src/components/DashboardGrid.svelte b/frontend/src/components/DashboardGrid.svelte index 9bf8356..11239ee 100644 --- a/frontend/src/components/DashboardGrid.svelte +++ b/frontend/src/components/DashboardGrid.svelte @@ -26,18 +26,18 @@ // [/SECTION] // [SECTION: STATE] - let filterText = ""; - let currentPage = 0; - let pageSize = 20; - let sortColumn: keyof DashboardMetadata = "title"; - let sortDirection: "asc" | "desc" = "asc"; + let filterText = $state(""); + let currentPage = $state(0); + let pageSize = $state(20); + let sortColumn: keyof DashboardMetadata = $state("title"); + let sortDirection: "asc" | "desc" = $state("asc"); // [/SECTION] // [SECTION: UI STATE] - let showGitManager = false; - let gitDashboardId: number | null = null; - let gitDashboardTitle = ""; - let validatingIds: Set = new Set(); + let showGitManager = $state(false); + let gitDashboardId: number | null = $state(null); + let gitDashboardTitle = $state(""); + let validatingIds: Set = $state(new Set()); // [/SECTION] // [DEF:handleValidate:Function] @@ -48,7 +48,7 @@ if (validatingIds.has(dashboard.id)) return; validatingIds.add(dashboard.id); - validatingIds = validatingIds; // Trigger reactivity + validatingIds = new Set(validatingIds); try { // TODO: Get provider_id from settings or prompt user @@ -83,7 +83,7 @@ toast(e.message || "Validation failed to start", "error"); } finally { validatingIds.delete(dashboard.id); - validatingIds = validatingIds; + validatingIds = new Set(validatingIds); } } // [/DEF:handleValidate:Function] @@ -221,14 +221,14 @@ type="checkbox" checked={allSelected} indeterminate={someSelected && !allSelected} - on:change={(e) => + onchange={(e) => handleSelectAll((e.target as HTMLInputElement).checked)} class="h-4 w-4 text-blue-600 border-gray-300 rounded focus:ring-blue-500" /> handleSort("title")} + onclick={() => handleSort("title")} > {$t.dashboard.title} {sortColumn === "title" @@ -239,7 +239,7 @@ handleSort("last_modified")} + onclick={() => handleSort("last_modified")} > {$t.dashboard.last_modified} {sortColumn === "last_modified" @@ -250,7 +250,7 @@ handleSort("status")} + onclick={() => handleSort("status")} > {$t.dashboard.status} {sortColumn === "status" @@ -276,7 +276,7 @@ + onchange={(e) => handleSelectionChange( dashboard.id, (e.target as HTMLInputElement).checked, diff --git a/frontend/src/components/MissingMappingModal.svelte b/frontend/src/components/MissingMappingModal.svelte index 5cffe09..1d9bbec 100644 --- a/frontend/src/components/MissingMappingModal.svelte +++ b/frontend/src/components/MissingMappingModal.svelte @@ -23,7 +23,7 @@ // [/SECTION] - let selectedTargetUuid = ""; + let selectedTargetUuid = $state(""); const dispatch = createEventDispatcher(); // [DEF:resolve:Function] @@ -94,7 +94,7 @@
+ +
+ +
+ {/if} - \ No newline at end of file + diff --git a/frontend/src/routes/settings/+page.svelte b/frontend/src/routes/settings/+page.svelte index fd05f82..6583978 100644 --- a/frontend/src/routes/settings/+page.svelte +++ b/frontend/src/routes/settings/+page.svelte @@ -20,6 +20,15 @@ import { addToast } from "$lib/toasts"; import ProviderConfig from "../../components/llm/ProviderConfig.svelte"; + const DEFAULT_LLM_PROMPTS = { + dashboard_validation_prompt: + "Analyze the attached dashboard screenshot and the following execution logs for health and visual issues.\\n\\nLogs:\\n{logs}\\n\\nProvide the analysis in JSON format with the following structure:\\n{\\n \\\"status\\\": \\\"PASS\\\" | \\\"WARN\\\" | \\\"FAIL\\\",\\n \\\"summary\\\": \\\"Short summary of findings\\\",\\n \\\"issues\\\": [\\n {\\n \\\"severity\\\": \\\"WARN\\\" | \\\"FAIL\\\",\\n \\\"message\\\": \\\"Description of the issue\\\",\\n \\\"location\\\": \\\"Optional location info (e.g. chart name)\\\"\\n }\\n ]\\n}", + documentation_prompt: + "Generate professional documentation for the following dataset and its columns.\\nDataset: {dataset_name}\\nColumns: {columns_json}\\n\\nProvide the documentation in JSON format:\\n{\\n \\\"dataset_description\\\": \\\"General description of the dataset\\\",\\n \\\"column_descriptions\\\": [\\n {\\n \\\"name\\\": \\\"column_name\\\",\\n \\\"description\\\": \\\"Generated description\\\"\\n }\\n ]\\n}", + git_commit_prompt: + "Generate a concise and professional git commit message based on the following diff and recent history.\\nUse Conventional Commits format (e.g., feat: ..., fix: ..., docs: ...).\\n\\nRecent History:\\n{history}\\n\\nDiff:\\n{diff}\\n\\nCommit Message:", + }; + // State let activeTab = "environments"; let settings = null; @@ -53,6 +62,7 @@ error = null; try { const response = await api.getConsolidatedSettings(); + response.llm = normalizeLlmSettings(response.llm); settings = response; } catch (err) { error = err.message || "Failed to load settings"; @@ -62,6 +72,20 @@ } } + function normalizeLlmSettings(llm) { + const normalized = { + providers: [], + default_provider: "", + prompts: { ...DEFAULT_LLM_PROMPTS }, + ...(llm || {}), + }; + normalized.prompts = { + ...DEFAULT_LLM_PROMPTS, + ...(llm?.prompts || {}), + }; + return normalized; + } + // Handle tab change function handleTabChange(tab) { activeTab = tab; @@ -78,6 +102,7 @@ async function handleSave() { console.log("[SettingsPage][Action] Saving settings"); try { + settings.llm = normalizeLlmSettings(settings.llm); // In a real app we might want to only send the changed section, // but updateConsolidatedSettings expects full object or we can use specific endpoints. // For now we use the consolidated update. @@ -644,6 +669,73 @@ providers={settings.llm_providers || []} onSave={loadSettings} /> + +
+

+ {$t.settings?.llm_prompts_title || "LLM Prompt Templates"} +

+

+ {$t.settings?.llm_prompts_description || + "Edit reusable prompts used for documentation, dashboard validation, and git commit generation."} +

+ +
+
+ + +
+ +
+ + +
+ +
+ + +
+
+ +
+ +
+
{:else if activeTab === "storage"}