Compare commits
2 Commits
017-llm-an
...
235b0e3c9f
| Author | SHA1 | Date | |
|---|---|---|---|
| 235b0e3c9f | |||
| e6087bd3c1 |
117595
backend/logs/app.log.1
117595
backend/logs/app.log.1
File diff suppressed because it is too large
Load Diff
Binary file not shown.
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.api.routes.environments:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: api, environments, superset, databases
|
||||
# @PURPOSE: API endpoints for listing environments and their databases.
|
||||
# @LAYER: API
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.api.routes.git:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: git, routes, api, fastapi, repository, deployment
|
||||
# @PURPOSE: Provides FastAPI endpoints for Git integration operations.
|
||||
# @LAYER: API
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.api.routes.mappings:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: api, mappings, database, fuzzy-matching
|
||||
# @PURPOSE: API endpoints for managing database mappings and getting suggestions.
|
||||
# @LAYER: API
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:backend.src.api.routes.migration:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: api, migration, dashboards
|
||||
# @PURPOSE: API endpoints for migration operations.
|
||||
# @LAYER: API
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:PluginsRouter:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: api, router, plugins, list
|
||||
# @PURPOSE: Defines the FastAPI router for plugin-related endpoints, allowing clients to list available plugins.
|
||||
# @LAYER: UI (API)
|
||||
|
||||
@@ -12,15 +12,25 @@
|
||||
# [SECTION: IMPORTS]
|
||||
from fastapi import APIRouter, Depends, HTTPException
|
||||
from typing import List
|
||||
from ...core.config_models import AppConfig, Environment, GlobalSettings
|
||||
from pydantic import BaseModel
|
||||
from ...core.config_models import AppConfig, Environment, GlobalSettings, LoggingConfig
|
||||
from ...models.storage import StorageConfig
|
||||
from ...dependencies import get_config_manager, has_permission
|
||||
from ...core.config_manager import ConfigManager
|
||||
from ...core.logger import logger, belief_scope
|
||||
from ...core.logger import logger, belief_scope, get_task_log_level
|
||||
from ...core.superset_client import SupersetClient
|
||||
import os
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:LoggingConfigResponse:Class]
|
||||
# @PURPOSE: Response model for logging configuration with current task log level.
|
||||
# @SEMANTICS: logging, config, response
|
||||
class LoggingConfigResponse(BaseModel):
|
||||
level: str
|
||||
task_log_level: str
|
||||
enable_belief_state: bool
|
||||
# [/DEF:LoggingConfigResponse:Class]
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
# [DEF:get_settings:Function]
|
||||
@@ -223,5 +233,50 @@ async def test_environment_connection(
|
||||
return {"status": "error", "message": str(e)}
|
||||
# [/DEF:test_environment_connection:Function]
|
||||
|
||||
# [DEF:get_logging_config:Function]
|
||||
# @PURPOSE: Retrieves current logging configuration.
|
||||
# @PRE: Config manager is available.
|
||||
# @POST: Returns logging configuration.
|
||||
# @RETURN: LoggingConfigResponse - The current logging config.
|
||||
@router.get("/logging", response_model=LoggingConfigResponse)
|
||||
async def get_logging_config(
|
||||
config_manager: ConfigManager = Depends(get_config_manager),
|
||||
_ = Depends(has_permission("admin:settings", "READ"))
|
||||
):
|
||||
with belief_scope("get_logging_config"):
|
||||
logging_config = config_manager.get_config().settings.logging
|
||||
return LoggingConfigResponse(
|
||||
level=logging_config.level,
|
||||
task_log_level=logging_config.task_log_level,
|
||||
enable_belief_state=logging_config.enable_belief_state
|
||||
)
|
||||
# [/DEF:get_logging_config:Function]
|
||||
|
||||
# [DEF:update_logging_config:Function]
|
||||
# @PURPOSE: Updates logging configuration.
|
||||
# @PRE: New logging config is provided.
|
||||
# @POST: Logging configuration is updated and saved.
|
||||
# @PARAM: config (LoggingConfig) - The new logging configuration.
|
||||
# @RETURN: LoggingConfigResponse - The updated logging config.
|
||||
@router.patch("/logging", response_model=LoggingConfigResponse)
|
||||
async def update_logging_config(
|
||||
config: LoggingConfig,
|
||||
config_manager: ConfigManager = Depends(get_config_manager),
|
||||
_ = Depends(has_permission("admin:settings", "WRITE"))
|
||||
):
|
||||
with belief_scope("update_logging_config"):
|
||||
logger.info(f"[update_logging_config][Entry] Updating logging config: level={config.level}, task_log_level={config.task_log_level}")
|
||||
|
||||
# Get current settings and update logging config
|
||||
settings = config_manager.get_config().settings
|
||||
settings.logging = config
|
||||
config_manager.update_global_settings(settings)
|
||||
|
||||
return LoggingConfigResponse(
|
||||
level=config.level,
|
||||
task_log_level=config.task_log_level,
|
||||
enable_belief_state=config.enable_belief_state
|
||||
)
|
||||
# [/DEF:update_logging_config:Function]
|
||||
|
||||
# [/DEF:SettingsRouter:Module]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:storage_routes:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: storage, files, upload, download, backup, repository
|
||||
# @PURPOSE: API endpoints for file storage management (backups and repositories).
|
||||
# @LAYER: API
|
||||
|
||||
@@ -1,14 +1,16 @@
|
||||
# [DEF:TasksRouter:Module]
|
||||
# @SEMANTICS: api, router, tasks, create, list, get
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: api, router, tasks, create, list, get, logs
|
||||
# @PURPOSE: Defines the FastAPI router for task-related endpoints, allowing clients to create, list, and get the status of tasks.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the TaskManager. It is included by the main app.
|
||||
from typing import List, Dict, Any, Optional
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from pydantic import BaseModel
|
||||
from fastapi import APIRouter, Depends, HTTPException, status, Query
|
||||
from pydantic import BaseModel, Field
|
||||
from ...core.logger import belief_scope
|
||||
|
||||
from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry
|
||||
from ...core.task_manager.models import LogFilter, LogStats
|
||||
from ...dependencies import get_task_manager, has_permission, get_current_user
|
||||
|
||||
router = APIRouter()
|
||||
@@ -116,27 +118,93 @@ async def get_task(
|
||||
|
||||
@router.get("/{task_id}/logs", response_model=List[LogEntry])
|
||||
# [DEF:get_task_logs:Function]
|
||||
# @PURPOSE: Retrieve logs for a specific task.
|
||||
# @PURPOSE: Retrieve logs for a specific task with optional filtering.
|
||||
# @PARAM: task_id (str) - The unique identifier of the task.
|
||||
# @PARAM: level (Optional[str]) - Filter by log level (DEBUG, INFO, WARNING, ERROR).
|
||||
# @PARAM: source (Optional[str]) - Filter by source component.
|
||||
# @PARAM: search (Optional[str]) - Text search in message.
|
||||
# @PARAM: offset (int) - Number of logs to skip.
|
||||
# @PARAM: limit (int) - Maximum number of logs to return.
|
||||
# @PARAM: task_manager (TaskManager) - The task manager instance.
|
||||
# @PRE: task_id must exist.
|
||||
# @POST: Returns a list of log entries or raises 404.
|
||||
# @RETURN: List[LogEntry] - List of log entries.
|
||||
# @TIER: CRITICAL
|
||||
async def get_task_logs(
|
||||
task_id: str,
|
||||
level: Optional[str] = Query(None, description="Filter by log level (DEBUG, INFO, WARNING, ERROR)"),
|
||||
source: Optional[str] = Query(None, description="Filter by source component"),
|
||||
search: Optional[str] = Query(None, description="Text search in message"),
|
||||
offset: int = Query(0, ge=0, description="Number of logs to skip"),
|
||||
limit: int = Query(100, ge=1, le=1000, description="Maximum number of logs to return"),
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
_ = Depends(has_permission("tasks", "READ"))
|
||||
):
|
||||
"""
|
||||
Retrieve logs for a specific task.
|
||||
Retrieve logs for a specific task with optional filtering.
|
||||
Supports filtering by level, source, and text search.
|
||||
"""
|
||||
with belief_scope("get_task_logs"):
|
||||
task = task_manager.get_task(task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
return task_manager.get_task_logs(task_id)
|
||||
|
||||
log_filter = LogFilter(
|
||||
level=level.upper() if level else None,
|
||||
source=source,
|
||||
search=search,
|
||||
offset=offset,
|
||||
limit=limit
|
||||
)
|
||||
return task_manager.get_task_logs(task_id, log_filter)
|
||||
# [/DEF:get_task_logs:Function]
|
||||
|
||||
@router.get("/{task_id}/logs/stats", response_model=LogStats)
|
||||
# [DEF:get_task_log_stats:Function]
|
||||
# @PURPOSE: Get statistics about logs for a task (counts by level and source).
|
||||
# @PARAM: task_id (str) - The unique identifier of the task.
|
||||
# @PARAM: task_manager (TaskManager) - The task manager instance.
|
||||
# @PRE: task_id must exist.
|
||||
# @POST: Returns log statistics or raises 404.
|
||||
# @RETURN: LogStats - Statistics about task logs.
|
||||
async def get_task_log_stats(
|
||||
task_id: str,
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
_ = Depends(has_permission("tasks", "READ"))
|
||||
):
|
||||
"""
|
||||
Get statistics about logs for a task (counts by level and source).
|
||||
"""
|
||||
with belief_scope("get_task_log_stats"):
|
||||
task = task_manager.get_task(task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
return task_manager.get_task_log_stats(task_id)
|
||||
# [/DEF:get_task_log_stats:Function]
|
||||
|
||||
@router.get("/{task_id}/logs/sources", response_model=List[str])
|
||||
# [DEF:get_task_log_sources:Function]
|
||||
# @PURPOSE: Get unique sources for a task's logs.
|
||||
# @PARAM: task_id (str) - The unique identifier of the task.
|
||||
# @PARAM: task_manager (TaskManager) - The task manager instance.
|
||||
# @PRE: task_id must exist.
|
||||
# @POST: Returns list of unique source names or raises 404.
|
||||
# @RETURN: List[str] - Unique source names.
|
||||
async def get_task_log_sources(
|
||||
task_id: str,
|
||||
task_manager: TaskManager = Depends(get_task_manager),
|
||||
_ = Depends(has_permission("tasks", "READ"))
|
||||
):
|
||||
"""
|
||||
Get unique sources for a task's logs.
|
||||
"""
|
||||
with belief_scope("get_task_log_sources"):
|
||||
task = task_manager.get_task(task_id)
|
||||
if not task:
|
||||
raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Task not found")
|
||||
return task_manager.get_task_log_sources(task_id)
|
||||
# [/DEF:get_task_log_sources:Function]
|
||||
|
||||
@router.post("/{task_id}/resolve", response_model=Task)
|
||||
# [DEF:resolve_task:Function]
|
||||
# @PURPOSE: Resolve a task that is awaiting mapping.
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
# [DEF:AppModule:Module]
|
||||
# @TIER: CRITICAL
|
||||
# @SEMANTICS: app, main, entrypoint, fastapi
|
||||
# @PURPOSE: The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.
|
||||
# @LAYER: UI (API)
|
||||
# @RELATION: Depends on the dependency module and API route modules.
|
||||
# @INVARIANT: Only one FastAPI app instance exists per process.
|
||||
# @INVARIANT: All WebSocket connections must be properly cleaned up on disconnect.
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -123,26 +126,65 @@ app.include_router(llm.router)
|
||||
app.include_router(storage.router, prefix="/api/storage", tags=["Storage"])
|
||||
|
||||
# [DEF:websocket_endpoint:Function]
|
||||
# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task.
|
||||
# @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.
|
||||
# @PRE: task_id must be a valid task ID.
|
||||
# @POST: WebSocket connection is managed and logs are streamed until disconnect.
|
||||
# @TIER: CRITICAL
|
||||
# @UX_STATE: Connecting -> Streaming -> (Disconnected)
|
||||
@app.websocket("/ws/logs/{task_id}")
|
||||
async def websocket_endpoint(websocket: WebSocket, task_id: str):
|
||||
async def websocket_endpoint(
|
||||
websocket: WebSocket,
|
||||
task_id: str,
|
||||
source: str = None,
|
||||
level: str = None
|
||||
):
|
||||
"""
|
||||
WebSocket endpoint for real-time log streaming with optional server-side filtering.
|
||||
|
||||
Query Parameters:
|
||||
source: Filter logs by source component (e.g., "plugin", "superset_api")
|
||||
level: Filter logs by minimum level (DEBUG, INFO, WARNING, ERROR)
|
||||
"""
|
||||
with belief_scope("websocket_endpoint", f"task_id={task_id}"):
|
||||
await websocket.accept()
|
||||
logger.info(f"WebSocket connection accepted for task {task_id}")
|
||||
|
||||
# Normalize filter parameters
|
||||
source_filter = source.lower() if source else None
|
||||
level_filter = level.upper() if level else None
|
||||
|
||||
# Level hierarchy for filtering
|
||||
level_hierarchy = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3}
|
||||
min_level = level_hierarchy.get(level_filter, 0) if level_filter else 0
|
||||
|
||||
logger.info(f"WebSocket connection accepted for task {task_id} (source={source_filter}, level={level_filter})")
|
||||
task_manager = get_task_manager()
|
||||
queue = await task_manager.subscribe_logs(task_id)
|
||||
|
||||
def matches_filters(log_entry) -> bool:
|
||||
"""Check if log entry matches the filter criteria."""
|
||||
# Check source filter
|
||||
if source_filter and log_entry.source.lower() != source_filter:
|
||||
return False
|
||||
|
||||
# Check level filter
|
||||
if level_filter:
|
||||
log_level = level_hierarchy.get(log_entry.level.upper(), 0)
|
||||
if log_level < min_level:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
try:
|
||||
# Stream new logs
|
||||
logger.info(f"Starting log stream for task {task_id}")
|
||||
|
||||
# Send initial logs first to build context
|
||||
# Send initial logs first to build context (apply filters)
|
||||
initial_logs = task_manager.get_task_logs(task_id)
|
||||
for log_entry in initial_logs:
|
||||
log_dict = log_entry.dict()
|
||||
log_dict['timestamp'] = log_dict['timestamp'].isoformat()
|
||||
await websocket.send_json(log_dict)
|
||||
if matches_filters(log_entry):
|
||||
log_dict = log_entry.dict()
|
||||
log_dict['timestamp'] = log_dict['timestamp'].isoformat()
|
||||
await websocket.send_json(log_dict)
|
||||
|
||||
# Force a check for AWAITING_INPUT status immediately upon connection
|
||||
# This ensures that if the task is already waiting when the user connects, they get the prompt.
|
||||
@@ -160,6 +202,11 @@ async def websocket_endpoint(websocket: WebSocket, task_id: str):
|
||||
|
||||
while True:
|
||||
log_entry = await queue.get()
|
||||
|
||||
# Apply server-side filtering
|
||||
if not matches_filters(log_entry):
|
||||
continue
|
||||
|
||||
log_dict = log_entry.dict()
|
||||
log_dict['timestamp'] = log_dict['timestamp'].isoformat()
|
||||
await websocket.send_json(log_dict)
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.core.auth.jwt:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: jwt, token, session, auth
|
||||
# @PURPOSE: JWT token generation and validation logic.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.core.auth.logger:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: auth, logger, audit, security
|
||||
# @PURPOSE: Audit logging for security-related events.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:ConfigModels:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: config, models, pydantic
|
||||
# @PURPOSE: Defines the data models for application configuration using Pydantic.
|
||||
# @LAYER: Core
|
||||
@@ -34,6 +35,7 @@ class Environment(BaseModel):
|
||||
# @PURPOSE: Defines the configuration for the application's logging system.
|
||||
class LoggingConfig(BaseModel):
|
||||
level: str = "INFO"
|
||||
task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
|
||||
file_path: Optional[str] = "logs/app.log"
|
||||
max_bytes: int = 10 * 1024 * 1024
|
||||
backup_count: int = 5
|
||||
|
||||
@@ -19,6 +19,9 @@ _belief_state = threading.local()
|
||||
# Global flag for belief state logging
|
||||
_enable_belief_state = True
|
||||
|
||||
# Global task log level filter
|
||||
_task_log_level = "INFO"
|
||||
|
||||
# [DEF:BeliefFormatter:Class]
|
||||
# @PURPOSE: Custom logging formatter that adds belief state prefixes to log messages.
|
||||
class BeliefFormatter(logging.Formatter):
|
||||
@@ -58,12 +61,12 @@ class LogEntry(BaseModel):
|
||||
# @SEMANTICS: logging, context, belief_state
|
||||
@contextmanager
|
||||
def belief_scope(anchor_id: str, message: str = ""):
|
||||
# Log Entry if enabled
|
||||
# Log Entry if enabled (DEBUG level to reduce noise)
|
||||
if _enable_belief_state:
|
||||
entry_msg = f"[{anchor_id}][Entry]"
|
||||
if message:
|
||||
entry_msg += f" {message}"
|
||||
logger.info(entry_msg)
|
||||
logger.debug(entry_msg)
|
||||
|
||||
# Set thread-local anchor_id
|
||||
old_anchor = getattr(_belief_state, 'anchor_id', None)
|
||||
@@ -71,13 +74,13 @@ def belief_scope(anchor_id: str, message: str = ""):
|
||||
|
||||
try:
|
||||
yield
|
||||
# Log Coherence OK and Exit
|
||||
logger.info(f"[{anchor_id}][Coherence:OK]")
|
||||
# Log Coherence OK and Exit (DEBUG level to reduce noise)
|
||||
logger.debug(f"[{anchor_id}][Coherence:OK]")
|
||||
if _enable_belief_state:
|
||||
logger.info(f"[{anchor_id}][Exit]")
|
||||
logger.debug(f"[{anchor_id}][Exit]")
|
||||
except Exception as e:
|
||||
# Log Coherence Failed
|
||||
logger.info(f"[{anchor_id}][Coherence:Failed] {str(e)}")
|
||||
# Log Coherence Failed (DEBUG level to reduce noise)
|
||||
logger.debug(f"[{anchor_id}][Coherence:Failed] {str(e)}")
|
||||
raise
|
||||
finally:
|
||||
# Restore old anchor
|
||||
@@ -88,12 +91,13 @@ def belief_scope(anchor_id: str, message: str = ""):
|
||||
# [DEF:configure_logger:Function]
|
||||
# @PURPOSE: Configures the logger with the provided logging settings.
|
||||
# @PRE: config is a valid LoggingConfig instance.
|
||||
# @POST: Logger level, handlers, and belief state flag are updated.
|
||||
# @POST: Logger level, handlers, belief state flag, and task log level are updated.
|
||||
# @PARAM: config (LoggingConfig) - The logging configuration.
|
||||
# @SEMANTICS: logging, configuration, initialization
|
||||
def configure_logger(config):
|
||||
global _enable_belief_state
|
||||
global _enable_belief_state, _task_log_level
|
||||
_enable_belief_state = config.enable_belief_state
|
||||
_task_log_level = config.task_log_level.upper()
|
||||
|
||||
# Set logger level
|
||||
level = getattr(logging, config.level.upper(), logging.INFO)
|
||||
@@ -130,6 +134,36 @@ def configure_logger(config):
|
||||
))
|
||||
# [/DEF:configure_logger:Function]
|
||||
|
||||
# [DEF:get_task_log_level:Function]
|
||||
# @PURPOSE: Returns the current task log level filter.
|
||||
# @PRE: None.
|
||||
# @POST: Returns the task log level string.
|
||||
# @RETURN: str - The current task log level (DEBUG, INFO, WARNING, ERROR).
|
||||
# @SEMANTICS: logging, configuration, getter
|
||||
def get_task_log_level() -> str:
|
||||
"""Returns the current task log level filter."""
|
||||
return _task_log_level
|
||||
# [/DEF:get_task_log_level:Function]
|
||||
|
||||
# [DEF:should_log_task_level:Function]
|
||||
# @PURPOSE: Checks if a log level should be recorded based on task_log_level setting.
|
||||
# @PRE: level is a valid log level string.
|
||||
# @POST: Returns True if level meets or exceeds task_log_level threshold.
|
||||
# @PARAM: level (str) - The log level to check.
|
||||
# @RETURN: bool - True if the level should be logged.
|
||||
# @SEMANTICS: logging, filter, level
|
||||
def should_log_task_level(level: str) -> bool:
|
||||
"""Checks if a log level should be recorded based on task_log_level setting."""
|
||||
level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3}
|
||||
current_level = _task_log_level.upper()
|
||||
check_level = level.upper()
|
||||
|
||||
current_order = level_order.get(current_level, 1) # Default to INFO
|
||||
check_order = level_order.get(check_level, 1)
|
||||
|
||||
return check_order >= current_order
|
||||
# [/DEF:should_log_task_level:Function]
|
||||
|
||||
# [DEF:WebSocketLogHandler:Class]
|
||||
# @SEMANTICS: logging, handler, websocket, buffer
|
||||
# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
|
||||
|
||||
@@ -7,6 +7,7 @@ from jsonschema import validate
|
||||
from .logger import belief_scope
|
||||
|
||||
# [DEF:PluginLoader:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: plugin, loader, dynamic, import
|
||||
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:SchedulerModule:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: scheduler, apscheduler, cron, backup
|
||||
# @PURPOSE: Manages scheduled tasks using APScheduler.
|
||||
# @LAYER: Core
|
||||
@@ -14,6 +15,7 @@ import asyncio
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:SchedulerService:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: scheduler, service, apscheduler
|
||||
# @PURPOSE: Provides a service to manage scheduled backup tasks.
|
||||
class SchedulerService:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:TaskManagerPackage:Module]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: task, manager, package, exports
|
||||
# @PURPOSE: Exports the public API of the task manager package.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,47 +1,76 @@
|
||||
# [DEF:TaskCleanupModule:Module]
|
||||
# @SEMANTICS: task, cleanup, retention
|
||||
# @PURPOSE: Implements task cleanup and retention policies.
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, cleanup, retention, logs
|
||||
# @PURPOSE: Implements task cleanup and retention policies, including associated logs.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Uses TaskPersistenceService to delete old tasks.
|
||||
# @RELATION: Uses TaskPersistenceService and TaskLogPersistenceService to delete old tasks and logs.
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from .persistence import TaskPersistenceService
|
||||
from typing import List
|
||||
from .persistence import TaskPersistenceService, TaskLogPersistenceService
|
||||
from ..logger import logger, belief_scope
|
||||
from ..config_manager import ConfigManager
|
||||
|
||||
# [DEF:TaskCleanupService:Class]
|
||||
# @PURPOSE: Provides methods to clean up old task records.
|
||||
# @PURPOSE: Provides methods to clean up old task records and their associated logs.
|
||||
# @TIER: STANDARD
|
||||
class TaskCleanupService:
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initializes the cleanup service with dependencies.
|
||||
# @PRE: persistence_service and config_manager are valid.
|
||||
# @POST: Cleanup service is ready.
|
||||
def __init__(self, persistence_service: TaskPersistenceService, config_manager: ConfigManager):
|
||||
def __init__(
|
||||
self,
|
||||
persistence_service: TaskPersistenceService,
|
||||
log_persistence_service: TaskLogPersistenceService,
|
||||
config_manager: ConfigManager
|
||||
):
|
||||
self.persistence_service = persistence_service
|
||||
self.log_persistence_service = log_persistence_service
|
||||
self.config_manager = config_manager
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:run_cleanup:Function]
|
||||
# @PURPOSE: Deletes tasks older than the configured retention period.
|
||||
# @PURPOSE: Deletes tasks older than the configured retention period and their logs.
|
||||
# @PRE: Config manager has valid settings.
|
||||
# @POST: Old tasks are deleted from persistence.
|
||||
# @POST: Old tasks and their logs are deleted from persistence.
|
||||
def run_cleanup(self):
|
||||
with belief_scope("TaskCleanupService.run_cleanup"):
|
||||
settings = self.config_manager.get_config().settings
|
||||
retention_days = settings.task_retention_days
|
||||
|
||||
# This is a simplified implementation.
|
||||
# In a real scenario, we would query IDs of tasks older than retention_days.
|
||||
# For now, we'll log the action.
|
||||
logger.info(f"Cleaning up tasks older than {retention_days} days.")
|
||||
|
||||
# Re-loading tasks to check for limit
|
||||
# Load tasks to check for limit
|
||||
tasks = self.persistence_service.load_tasks(limit=1000)
|
||||
if len(tasks) > settings.task_retention_limit:
|
||||
to_delete = [t.id for t in tasks[settings.task_retention_limit:]]
|
||||
to_delete: List[str] = [t.id for t in tasks[settings.task_retention_limit:]]
|
||||
|
||||
# Delete logs first (before task records)
|
||||
self.log_persistence_service.delete_logs_for_tasks(to_delete)
|
||||
|
||||
# Then delete task records
|
||||
self.persistence_service.delete_tasks(to_delete)
|
||||
logger.info(f"Deleted {len(to_delete)} tasks exceeding limit of {settings.task_retention_limit}")
|
||||
|
||||
logger.info(f"Deleted {len(to_delete)} tasks and their logs exceeding limit of {settings.task_retention_limit}")
|
||||
# [/DEF:run_cleanup:Function]
|
||||
|
||||
# [DEF:delete_task_with_logs:Function]
|
||||
# @PURPOSE: Delete a single task and all its associated logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Task and all its logs are deleted.
|
||||
# @PARAM: task_id (str) - The task ID to delete.
|
||||
def delete_task_with_logs(self, task_id: str) -> None:
|
||||
"""Delete a single task and all its associated logs."""
|
||||
with belief_scope("TaskCleanupService.delete_task_with_logs", f"task_id={task_id}"):
|
||||
# Delete logs first
|
||||
self.log_persistence_service.delete_logs_for_task(task_id)
|
||||
|
||||
# Then delete task record
|
||||
self.persistence_service.delete_tasks([task_id])
|
||||
|
||||
logger.info(f"Deleted task {task_id} and its associated logs")
|
||||
# [/DEF:delete_task_with_logs:Function]
|
||||
|
||||
# [/DEF:TaskCleanupService:Class]
|
||||
# [/DEF:TaskCleanupModule:Module]
|
||||
115
backend/src/core/task_manager/context.py
Normal file
115
backend/src/core/task_manager/context.py
Normal file
@@ -0,0 +1,115 @@
|
||||
# [DEF:TaskContextModule:Module]
|
||||
# @SEMANTICS: task, context, plugin, execution, logger
|
||||
# @PURPOSE: Provides execution context passed to plugins during task execution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each TaskContext is bound to a single task execution.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from .task_logger import TaskLogger
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskContext:Class]
|
||||
# @SEMANTICS: context, task, execution, plugin
|
||||
# @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: logger is always a valid TaskLogger instance.
|
||||
# @UX_STATE: Idle -> Active -> Complete
|
||||
class TaskContext:
|
||||
"""
|
||||
Execution context provided to plugins during task execution.
|
||||
|
||||
Usage:
|
||||
def execute(params: dict, context: TaskContext = None):
|
||||
if context:
|
||||
context.logger.info("Starting process")
|
||||
context.logger.progress("Processing items", percent=50)
|
||||
# ... plugin logic
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskContext with task-specific resources.
|
||||
# @PRE: task_id is a valid task identifier, add_log_fn is callable.
|
||||
# @POST: TaskContext is ready to be passed to plugin.execute().
|
||||
# @PARAM: task_id (str) - The ID of the task.
|
||||
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
|
||||
# @PARAM: params (Dict) - Task parameters.
|
||||
# @PARAM: default_source (str) - Default source for logs (default: "plugin").
|
||||
def __init__(
|
||||
self,
|
||||
task_id: str,
|
||||
add_log_fn: Callable,
|
||||
params: Dict[str, Any],
|
||||
default_source: str = "plugin"
|
||||
):
|
||||
self._task_id = task_id
|
||||
self._params = params
|
||||
self._logger = TaskLogger(
|
||||
task_id=task_id,
|
||||
add_log_fn=add_log_fn,
|
||||
source=default_source
|
||||
)
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:task_id:Function]
|
||||
# @PURPOSE: Get the task ID.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the task ID string.
|
||||
# @RETURN: str - The task ID.
|
||||
@property
|
||||
def task_id(self) -> str:
|
||||
return self._task_id
|
||||
# [/DEF:task_id:Function]
|
||||
|
||||
# [DEF:logger:Function]
|
||||
# @PURPOSE: Get the TaskLogger instance for this context.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the TaskLogger instance.
|
||||
# @RETURN: TaskLogger - The logger instance.
|
||||
@property
|
||||
def logger(self) -> TaskLogger:
|
||||
return self._logger
|
||||
# [/DEF:logger:Function]
|
||||
|
||||
# [DEF:params:Function]
|
||||
# @PURPOSE: Get the task parameters.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the parameters dictionary.
|
||||
# @RETURN: Dict[str, Any] - The task parameters.
|
||||
@property
|
||||
def params(self) -> Dict[str, Any]:
|
||||
return self._params
|
||||
# [/DEF:params:Function]
|
||||
|
||||
# [DEF:get_param:Function]
|
||||
# @PURPOSE: Get a specific parameter value with optional default.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns parameter value or default.
|
||||
# @PARAM: key (str) - Parameter key.
|
||||
# @PARAM: default (Any) - Default value if key not found.
|
||||
# @RETURN: Any - Parameter value or default.
|
||||
def get_param(self, key: str, default: Any = None) -> Any:
|
||||
return self._params.get(key, default)
|
||||
# [/DEF:get_param:Function]
|
||||
|
||||
# [DEF:create_sub_context:Function]
|
||||
# @PURPOSE: Create a sub-context with a different default source.
|
||||
# @PRE: source is a non-empty string.
|
||||
# @POST: Returns new TaskContext with different logger source.
|
||||
# @PARAM: source (str) - New default source for logging.
|
||||
# @RETURN: TaskContext - New context with different source.
|
||||
def create_sub_context(self, source: str) -> "TaskContext":
|
||||
"""Create a sub-context with a different default source for logging."""
|
||||
return TaskContext(
|
||||
task_id=self._task_id,
|
||||
add_log_fn=self._logger._add_log,
|
||||
params=self._params,
|
||||
default_source=source
|
||||
)
|
||||
# [/DEF:create_sub_context:Function]
|
||||
|
||||
# [/DEF:TaskContext:Class]
|
||||
|
||||
# [/DEF:TaskContextModule:Module]
|
||||
@@ -8,23 +8,33 @@
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import asyncio
|
||||
import threading
|
||||
import inspect
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from .models import Task, TaskStatus, LogEntry
|
||||
from .persistence import TaskPersistenceService
|
||||
from ..logger import logger, belief_scope
|
||||
from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats, TaskLog
|
||||
from .persistence import TaskPersistenceService, TaskLogPersistenceService
|
||||
from .context import TaskContext
|
||||
from ..logger import logger, belief_scope, should_log_task_level
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskManager:Class]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Task IDs are unique within the registry.
|
||||
# @INVARIANT: Each task has exactly one status at any time.
|
||||
# @INVARIANT: Log entries are never deleted after being added to a task.
|
||||
class TaskManager:
|
||||
"""
|
||||
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
"""
|
||||
|
||||
# Log flush interval in seconds
|
||||
LOG_FLUSH_INTERVAL = 2.0
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskManager with dependencies.
|
||||
# @PRE: plugin_loader is initialized.
|
||||
@@ -35,8 +45,18 @@ class TaskManager:
|
||||
self.plugin_loader = plugin_loader
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.persistence_service = TaskPersistenceService()
|
||||
self.log_persistence_service = TaskLogPersistenceService()
|
||||
|
||||
# Log buffer: task_id -> List[LogEntry]
|
||||
self._log_buffer: Dict[str, List[LogEntry]] = {}
|
||||
self._log_buffer_lock = threading.Lock()
|
||||
|
||||
# Flusher thread for batch writing logs
|
||||
self._flusher_stop_event = threading.Event()
|
||||
self._flusher_thread = threading.Thread(target=self._flusher_loop, daemon=True)
|
||||
self._flusher_thread.start()
|
||||
|
||||
try:
|
||||
self.loop = asyncio.get_running_loop()
|
||||
@@ -47,6 +67,59 @@ class TaskManager:
|
||||
# Load persisted tasks on startup
|
||||
self.load_persisted_tasks()
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:_flusher_loop:Function]
|
||||
# @PURPOSE: Background thread that periodically flushes log buffer to database.
|
||||
# @PRE: TaskManager is initialized.
|
||||
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
|
||||
def _flusher_loop(self):
|
||||
"""Background thread that flushes log buffer to database."""
|
||||
while not self._flusher_stop_event.is_set():
|
||||
self._flush_logs()
|
||||
self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)
|
||||
# [/DEF:_flusher_loop:Function]
|
||||
|
||||
# [DEF:_flush_logs:Function]
|
||||
# @PURPOSE: Flush all buffered logs to the database.
|
||||
# @PRE: None.
|
||||
# @POST: All buffered logs are written to task_logs table.
|
||||
def _flush_logs(self):
|
||||
"""Flush all buffered logs to the database."""
|
||||
with self._log_buffer_lock:
|
||||
task_ids = list(self._log_buffer.keys())
|
||||
|
||||
for task_id in task_ids:
|
||||
with self._log_buffer_lock:
|
||||
logs = self._log_buffer.pop(task_id, [])
|
||||
|
||||
if logs:
|
||||
try:
|
||||
self.log_persistence_service.add_logs(task_id, logs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to flush logs for task {task_id}: {e}")
|
||||
# Re-add logs to buffer on failure
|
||||
with self._log_buffer_lock:
|
||||
if task_id not in self._log_buffer:
|
||||
self._log_buffer[task_id] = []
|
||||
self._log_buffer[task_id].extend(logs)
|
||||
# [/DEF:_flush_logs:Function]
|
||||
|
||||
# [DEF:_flush_task_logs:Function]
|
||||
# @PURPOSE: Flush logs for a specific task immediately.
|
||||
# @PRE: task_id exists.
|
||||
# @POST: Task's buffered logs are written to database.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
def _flush_task_logs(self, task_id: str):
|
||||
"""Flush logs for a specific task immediately."""
|
||||
with self._log_buffer_lock:
|
||||
logs = self._log_buffer.pop(task_id, [])
|
||||
|
||||
if logs:
|
||||
try:
|
||||
self.log_persistence_service.add_logs(task_id, logs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to flush logs for task {task_id}: {e}")
|
||||
# [/DEF:_flush_task_logs:Function]
|
||||
|
||||
# [DEF:create_task:Function]
|
||||
# @PURPOSE: Creates and queues a new task for execution.
|
||||
@@ -78,7 +151,7 @@ class TaskManager:
|
||||
# [/DEF:create_task:Function]
|
||||
|
||||
# [DEF:_run_task:Function]
|
||||
# @PURPOSE: Internal method to execute a task.
|
||||
# @PURPOSE: Internal method to execute a task with TaskContext support.
|
||||
# @PRE: Task exists in registry.
|
||||
# @POST: Task is executed, status updated to SUCCESS or FAILED.
|
||||
# @PARAM: task_id (str) - The ID of the task to run.
|
||||
@@ -91,30 +164,54 @@ class TaskManager:
|
||||
task.status = TaskStatus.RUNNING
|
||||
task.started_at = datetime.utcnow()
|
||||
self.persistence_service.persist_task(task)
|
||||
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'")
|
||||
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'", source="system")
|
||||
|
||||
try:
|
||||
# Execute plugin
|
||||
# Prepare params and check if plugin supports new TaskContext
|
||||
params = {**task.params, "_task_id": task_id}
|
||||
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
plugin.execute,
|
||||
params
|
||||
# Check if plugin's execute method accepts 'context' parameter
|
||||
sig = inspect.signature(plugin.execute)
|
||||
accepts_context = 'context' in sig.parameters
|
||||
|
||||
if accepts_context:
|
||||
# Create TaskContext for new-style plugins
|
||||
context = TaskContext(
|
||||
task_id=task_id,
|
||||
add_log_fn=self._add_log,
|
||||
params=params,
|
||||
default_source="plugin"
|
||||
)
|
||||
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params, context=context)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
lambda: plugin.execute(params, context=context)
|
||||
)
|
||||
else:
|
||||
# Backward compatibility: old-style plugins without context
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
plugin.execute,
|
||||
params
|
||||
)
|
||||
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
task.status = TaskStatus.SUCCESS
|
||||
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'")
|
||||
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'", source="system")
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} failed: {e}")
|
||||
task.status = TaskStatus.FAILED
|
||||
self._add_log(task_id, "ERROR", f"Task failed: {e}", {"error_type": type(e).__name__})
|
||||
self._add_log(task_id, "ERROR", f"Task failed: {e}", source="system", metadata={"error_type": type(e).__name__})
|
||||
finally:
|
||||
task.finished_at = datetime.utcnow()
|
||||
# Flush any remaining buffered logs before persisting task
|
||||
self._flush_task_logs(task_id)
|
||||
self.persistence_service.persist_task(task)
|
||||
logger.info(f"Task {task_id} execution finished with status: {task.status}")
|
||||
# [/DEF:_run_task:Function]
|
||||
@@ -224,36 +321,106 @@ class TaskManager:
|
||||
# [/DEF:get_tasks:Function]
|
||||
|
||||
# [DEF:get_task_logs:Function]
|
||||
# @PURPOSE: Retrieves logs for a specific task.
|
||||
# @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).
|
||||
# @PRE: task_id is a string.
|
||||
# @POST: Returns list of LogEntry objects.
|
||||
# @POST: Returns list of LogEntry or TaskLog objects.
|
||||
# @PARAM: task_id (str) - ID of the task.
|
||||
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
|
||||
# @RETURN: List[LogEntry] - List of log entries.
|
||||
def get_task_logs(self, task_id: str) -> List[LogEntry]:
|
||||
def get_task_logs(self, task_id: str, log_filter: Optional[LogFilter] = None) -> List[LogEntry]:
|
||||
with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"):
|
||||
task = self.tasks.get(task_id)
|
||||
|
||||
# For completed tasks, fetch from persistence
|
||||
if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:
|
||||
if log_filter is None:
|
||||
log_filter = LogFilter()
|
||||
task_logs = self.log_persistence_service.get_logs(task_id, log_filter)
|
||||
# Convert TaskLog to LogEntry for backward compatibility
|
||||
return [
|
||||
LogEntry(
|
||||
timestamp=log.timestamp,
|
||||
level=log.level,
|
||||
message=log.message,
|
||||
source=log.source,
|
||||
metadata=log.metadata
|
||||
)
|
||||
for log in task_logs
|
||||
]
|
||||
|
||||
# For running/pending tasks, return from memory
|
||||
return task.logs if task else []
|
||||
# [/DEF:get_task_logs:Function]
|
||||
|
||||
# [DEF:get_task_log_stats:Function]
|
||||
# @PURPOSE: Get statistics about logs for a task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns LogStats with counts by level and source.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: LogStats - Statistics about task logs.
|
||||
def get_task_log_stats(self, task_id: str) -> LogStats:
|
||||
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
|
||||
return self.log_persistence_service.get_log_stats(task_id)
|
||||
# [/DEF:get_task_log_stats:Function]
|
||||
|
||||
# [DEF:get_task_log_sources:Function]
|
||||
# @PURPOSE: Get unique sources for a task's logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of unique source strings.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: List[str] - Unique source names.
|
||||
def get_task_log_sources(self, task_id: str) -> List[str]:
|
||||
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
|
||||
return self.log_persistence_service.get_sources(task_id)
|
||||
# [/DEF:get_task_log_sources:Function]
|
||||
|
||||
# [DEF:_add_log:Function]
|
||||
# @PURPOSE: Adds a log entry to a task and notifies subscribers.
|
||||
# @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.
|
||||
# @PRE: Task exists.
|
||||
# @POST: Log added to task and pushed to queues.
|
||||
# @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).
|
||||
# @PARAM: task_id (str) - ID of the task.
|
||||
# @PARAM: level (str) - Log level.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: context (Optional[Dict]) - Log context.
|
||||
def _add_log(self, task_id: str, level: str, message: str, context: Optional[Dict[str, Any]] = None):
|
||||
# @PARAM: source (str) - Source component (default: "system").
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
|
||||
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
|
||||
def _add_log(
|
||||
self,
|
||||
task_id: str,
|
||||
level: str,
|
||||
message: str,
|
||||
source: str = "system",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
):
|
||||
with belief_scope("TaskManager._add_log", f"task_id={task_id}"):
|
||||
task = self.tasks.get(task_id)
|
||||
if not task:
|
||||
return
|
||||
|
||||
log_entry = LogEntry(level=level, message=message, context=context)
|
||||
task.logs.append(log_entry)
|
||||
self.persistence_service.persist_task(task)
|
||||
# Filter logs based on task_log_level configuration
|
||||
if not should_log_task_level(level):
|
||||
return
|
||||
|
||||
# Notify subscribers
|
||||
# Create log entry with new fields
|
||||
log_entry = LogEntry(
|
||||
level=level,
|
||||
message=message,
|
||||
source=source,
|
||||
metadata=metadata,
|
||||
context=context # Keep for backward compatibility
|
||||
)
|
||||
|
||||
# Add to in-memory logs (for backward compatibility with legacy JSON field)
|
||||
task.logs.append(log_entry)
|
||||
|
||||
# Add to buffer for batch persistence
|
||||
with self._log_buffer_lock:
|
||||
if task_id not in self._log_buffer:
|
||||
self._log_buffer[task_id] = []
|
||||
self._log_buffer[task_id].append(log_entry)
|
||||
|
||||
# Notify subscribers (for real-time WebSocket updates)
|
||||
if task_id in self.subscribers:
|
||||
for queue in self.subscribers[task_id]:
|
||||
self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)
|
||||
@@ -353,7 +520,7 @@ class TaskManager:
|
||||
# [/DEF:resume_task_with_password:Function]
|
||||
|
||||
# [DEF:clear_tasks:Function]
|
||||
# @PURPOSE: Clears tasks based on status filter.
|
||||
# @PURPOSE: Clears tasks based on status filter (also deletes associated logs).
|
||||
# @PRE: status is Optional[TaskStatus].
|
||||
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.
|
||||
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
|
||||
@@ -387,9 +554,13 @@ class TaskManager:
|
||||
|
||||
del self.tasks[tid]
|
||||
|
||||
# Remove from persistence
|
||||
# Remove from persistence (task_records and task_logs via CASCADE)
|
||||
self.persistence_service.delete_tasks(tasks_to_remove)
|
||||
|
||||
# Also explicitly delete logs (in case CASCADE is not set up)
|
||||
if tasks_to_remove:
|
||||
self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)
|
||||
|
||||
logger.info(f"Cleared {len(tasks_to_remove)} tasks.")
|
||||
return len(tasks_to_remove)
|
||||
# [/DEF:clear_tasks:Function]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:TaskManagerModels:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, models, pydantic, enum, state
|
||||
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
|
||||
# @LAYER: Core
|
||||
@@ -16,6 +17,7 @@ from pydantic import BaseModel, Field
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskStatus:Enum]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: task, status, state, enum
|
||||
# @PURPOSE: Defines the possible states a task can be in during its lifecycle.
|
||||
class TaskStatus(str, Enum):
|
||||
@@ -27,17 +29,73 @@ class TaskStatus(str, Enum):
|
||||
AWAITING_INPUT = "AWAITING_INPUT"
|
||||
# [/DEF:TaskStatus:Enum]
|
||||
|
||||
# [DEF:LogLevel:Enum]
|
||||
# @SEMANTICS: log, level, severity, enum
|
||||
# @PURPOSE: Defines the possible log levels for task logging.
|
||||
# @TIER: STANDARD
|
||||
class LogLevel(str, Enum):
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
# [/DEF:LogLevel:Enum]
|
||||
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each log entry has a unique timestamp and source.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
level: str = Field(default="INFO")
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
source: str = Field(default="system") # Component attribution: plugin, superset_api, git, etc.
|
||||
context: Optional[Dict[str, Any]] = None # Legacy field, kept for backward compatibility
|
||||
metadata: Optional[Dict[str, Any]] = None # Structured metadata (e.g., dashboard_id, progress)
|
||||
# [/DEF:LogEntry:Class]
|
||||
|
||||
# [DEF:TaskLog:Class]
|
||||
# @SEMANTICS: task, log, persistent, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
|
||||
# @TIER: STANDARD
|
||||
# @RELATION: MAPS_TO -> TaskLogRecord
|
||||
class TaskLog(BaseModel):
|
||||
id: int
|
||||
task_id: str
|
||||
timestamp: datetime
|
||||
level: str
|
||||
source: str
|
||||
message: str
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TaskLog:Class]
|
||||
|
||||
# [DEF:LogFilter:Class]
|
||||
# @SEMANTICS: log, filter, query, pydantic
|
||||
# @PURPOSE: Filter parameters for querying task logs.
|
||||
# @TIER: STANDARD
|
||||
class LogFilter(BaseModel):
|
||||
level: Optional[str] = None # Filter by log level
|
||||
source: Optional[str] = None # Filter by source component
|
||||
search: Optional[str] = None # Text search in message
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=100, ge=1, le=1000)
|
||||
# [/DEF:LogFilter:Class]
|
||||
|
||||
# [DEF:LogStats:Class]
|
||||
# @SEMANTICS: log, stats, aggregation, pydantic
|
||||
# @PURPOSE: Statistics about log entries for a task.
|
||||
# @TIER: STANDARD
|
||||
class LogStats(BaseModel):
|
||||
total_count: int
|
||||
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
|
||||
by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7}
|
||||
# [/DEF:LogStats:Class]
|
||||
|
||||
# [DEF:Task:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, job, execution, state, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
|
||||
class Task(BaseModel):
|
||||
|
||||
@@ -11,9 +11,10 @@ from typing import List, Optional, Dict, Any
|
||||
import json
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from ...models.task import TaskRecord
|
||||
from sqlalchemy import and_, or_
|
||||
from ...models.task import TaskRecord, TaskLogRecord
|
||||
from ..database import TasksSessionLocal
|
||||
from .models import Task, TaskStatus, LogEntry
|
||||
from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats
|
||||
from ..logger import logger, belief_scope
|
||||
# [/SECTION]
|
||||
|
||||
@@ -170,4 +171,215 @@ class TaskPersistenceService:
|
||||
# [/DEF:delete_tasks:Function]
|
||||
|
||||
# [/DEF:TaskPersistenceService:Class]
|
||||
|
||||
# [DEF:TaskLogPersistenceService:Class]
|
||||
# @SEMANTICS: persistence, service, database, log, sqlalchemy
|
||||
# @PURPOSE: Provides methods to save and query task logs from the task_logs table.
|
||||
# @TIER: CRITICAL
|
||||
# @RELATION: DEPENDS_ON -> TaskLogRecord
|
||||
# @INVARIANT: Log entries are batch-inserted for performance.
|
||||
class TaskLogPersistenceService:
|
||||
"""
|
||||
Service for persisting and querying task logs.
|
||||
Supports batch inserts, filtering, and statistics.
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the log persistence service.
|
||||
# @POST: Service is ready.
|
||||
def __init__(self):
|
||||
pass
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:add_logs:Function]
|
||||
# @PURPOSE: Batch insert log entries for a task.
|
||||
# @PRE: logs is a list of LogEntry objects.
|
||||
# @POST: All logs inserted into task_logs table.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @PARAM: logs (List[LogEntry]) - Log entries to insert.
|
||||
# @SIDE_EFFECT: Writes to task_logs table.
|
||||
def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:
|
||||
if not logs:
|
||||
return
|
||||
with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
for log in logs:
|
||||
record = TaskLogRecord(
|
||||
task_id=task_id,
|
||||
timestamp=log.timestamp,
|
||||
level=log.level,
|
||||
source=log.source or "system",
|
||||
message=log.message,
|
||||
metadata_json=json.dumps(log.metadata) if log.metadata else None
|
||||
)
|
||||
session.add(record)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to add logs for task {task_id}: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:add_logs:Function]
|
||||
|
||||
# [DEF:get_logs:Function]
|
||||
# @PURPOSE: Query logs for a task with filtering and pagination.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of TaskLog objects matching filters.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @PARAM: log_filter (LogFilter) - Filter parameters.
|
||||
# @RETURN: List[TaskLog] - Filtered log entries.
|
||||
def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:
|
||||
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
query = session.query(TaskLogRecord).filter(TaskLogRecord.task_id == task_id)
|
||||
|
||||
# Apply filters
|
||||
if log_filter.level:
|
||||
query = query.filter(TaskLogRecord.level == log_filter.level.upper())
|
||||
if log_filter.source:
|
||||
query = query.filter(TaskLogRecord.source == log_filter.source)
|
||||
if log_filter.search:
|
||||
search_pattern = f"%{log_filter.search}%"
|
||||
query = query.filter(TaskLogRecord.message.ilike(search_pattern))
|
||||
|
||||
# Order by timestamp ascending (oldest first)
|
||||
query = query.order_by(TaskLogRecord.timestamp.asc())
|
||||
|
||||
# Apply pagination
|
||||
records = query.offset(log_filter.offset).limit(log_filter.limit).all()
|
||||
|
||||
logs = []
|
||||
for record in records:
|
||||
metadata = None
|
||||
if record.metadata_json:
|
||||
try:
|
||||
metadata = json.loads(record.metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
metadata = None
|
||||
|
||||
logs.append(TaskLog(
|
||||
id=record.id,
|
||||
task_id=record.task_id,
|
||||
timestamp=record.timestamp,
|
||||
level=record.level,
|
||||
source=record.source,
|
||||
message=record.message,
|
||||
metadata=metadata
|
||||
))
|
||||
|
||||
return logs
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_logs:Function]
|
||||
|
||||
# [DEF:get_log_stats:Function]
|
||||
# @PURPOSE: Get statistics about logs for a task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns LogStats with counts by level and source.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: LogStats - Statistics about task logs.
|
||||
def get_log_stats(self, task_id: str) -> LogStats:
|
||||
with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
# Get total count
|
||||
total_count = session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).count()
|
||||
|
||||
# Get counts by level
|
||||
from sqlalchemy import func
|
||||
level_counts = session.query(
|
||||
TaskLogRecord.level,
|
||||
func.count(TaskLogRecord.id)
|
||||
).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).group_by(TaskLogRecord.level).all()
|
||||
|
||||
by_level = {level: count for level, count in level_counts}
|
||||
|
||||
# Get counts by source
|
||||
source_counts = session.query(
|
||||
TaskLogRecord.source,
|
||||
func.count(TaskLogRecord.id)
|
||||
).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).group_by(TaskLogRecord.source).all()
|
||||
|
||||
by_source = {source: count for source, count in source_counts}
|
||||
|
||||
return LogStats(
|
||||
total_count=total_count,
|
||||
by_level=by_level,
|
||||
by_source=by_source
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_log_stats:Function]
|
||||
|
||||
# [DEF:get_sources:Function]
|
||||
# @PURPOSE: Get unique sources for a task's logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of unique source strings.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: List[str] - Unique source names.
|
||||
def get_sources(self, task_id: str) -> List[str]:
|
||||
with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
from sqlalchemy import distinct
|
||||
sources = session.query(distinct(TaskLogRecord.source)).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).all()
|
||||
return [s[0] for s in sources]
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_sources:Function]
|
||||
|
||||
# [DEF:delete_logs_for_task:Function]
|
||||
# @PURPOSE: Delete all logs for a specific task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: All logs for the task are deleted.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @SIDE_EFFECT: Deletes from task_logs table.
|
||||
def delete_logs_for_task(self, task_id: str) -> None:
|
||||
with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).delete(synchronize_session=False)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to delete logs for task {task_id}: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:delete_logs_for_task:Function]
|
||||
|
||||
# [DEF:delete_logs_for_tasks:Function]
|
||||
# @PURPOSE: Delete all logs for multiple tasks.
|
||||
# @PRE: task_ids is a list of task IDs.
|
||||
# @POST: All logs for the tasks are deleted.
|
||||
# @PARAM: task_ids (List[str]) - List of task IDs.
|
||||
def delete_logs_for_tasks(self, task_ids: List[str]) -> None:
|
||||
if not task_ids:
|
||||
return
|
||||
with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id.in_(task_ids)
|
||||
).delete(synchronize_session=False)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to delete logs for tasks: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:delete_logs_for_tasks:Function]
|
||||
|
||||
# [/DEF:TaskLogPersistenceService:Class]
|
||||
# [/DEF:TaskPersistenceModule:Module]
|
||||
168
backend/src/core/task_manager/task_logger.py
Normal file
168
backend/src/core/task_manager/task_logger.py
Normal file
@@ -0,0 +1,168 @@
|
||||
# [DEF:TaskLoggerModule:Module]
|
||||
# @SEMANTICS: task, logger, context, plugin, attribution
|
||||
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from datetime import datetime
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskLogger:Class]
|
||||
# @SEMANTICS: logger, task, source, attribution
|
||||
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: All log calls include the task_id and source.
|
||||
# @UX_STATE: Idle -> Logging -> (system records log)
|
||||
class TaskLogger:
|
||||
"""
|
||||
A dedicated logger for tasks that automatically tags logs with source attribution.
|
||||
|
||||
Usage:
|
||||
logger = TaskLogger(task_id="abc123", add_log_fn=task_manager._add_log, source="plugin")
|
||||
logger.info("Starting backup process")
|
||||
logger.error("Failed to connect", metadata={"error_code": 500})
|
||||
|
||||
# Create sub-logger with different source
|
||||
api_logger = logger.with_source("superset_api")
|
||||
api_logger.info("Fetching dashboards")
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskLogger with task context.
|
||||
# @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
|
||||
# @POST: TaskLogger is ready to log messages.
|
||||
# @PARAM: task_id (str) - The ID of the task.
|
||||
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
|
||||
# @PARAM: source (str) - Default source for logs (default: "plugin").
|
||||
def __init__(
|
||||
self,
|
||||
task_id: str,
|
||||
add_log_fn: Callable,
|
||||
source: str = "plugin"
|
||||
):
|
||||
self._task_id = task_id
|
||||
self._add_log = add_log_fn
|
||||
self._default_source = source
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:with_source:Function]
|
||||
# @PURPOSE: Create a sub-logger with a different default source.
|
||||
# @PRE: source is a non-empty string.
|
||||
# @POST: Returns new TaskLogger with the same task_id but different source.
|
||||
# @PARAM: source (str) - New default source.
|
||||
# @RETURN: TaskLogger - New logger instance.
|
||||
def with_source(self, source: str) -> "TaskLogger":
|
||||
"""Create a sub-logger with a different source context."""
|
||||
return TaskLogger(
|
||||
task_id=self._task_id,
|
||||
add_log_fn=self._add_log,
|
||||
source=source
|
||||
)
|
||||
# [/DEF:with_source:Function]
|
||||
|
||||
# [DEF:_log:Function]
|
||||
# @PURPOSE: Internal method to log a message at a given level.
|
||||
# @PRE: level is a valid log level string.
|
||||
# @POST: Log entry added via add_log_fn.
|
||||
# @PARAM: level (str) - Log level (DEBUG, INFO, WARNING, ERROR).
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source for this log entry.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
|
||||
def _log(
|
||||
self,
|
||||
level: str,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Internal logging method."""
|
||||
self._add_log(
|
||||
task_id=self._task_id,
|
||||
level=level,
|
||||
message=message,
|
||||
source=source or self._default_source,
|
||||
metadata=metadata
|
||||
)
|
||||
# [/DEF:_log:Function]
|
||||
|
||||
# [DEF:debug:Function]
|
||||
# @PURPOSE: Log a DEBUG level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def debug(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("DEBUG", message, source, metadata)
|
||||
# [/DEF:debug:Function]
|
||||
|
||||
# [DEF:info:Function]
|
||||
# @PURPOSE: Log an INFO level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def info(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("INFO", message, source, metadata)
|
||||
# [/DEF:info:Function]
|
||||
|
||||
# [DEF:warning:Function]
|
||||
# @PURPOSE: Log a WARNING level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def warning(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("WARNING", message, source, metadata)
|
||||
# [/DEF:warning:Function]
|
||||
|
||||
# [DEF:error:Function]
|
||||
# @PURPOSE: Log an ERROR level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def error(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("ERROR", message, source, metadata)
|
||||
# [/DEF:error:Function]
|
||||
|
||||
# [DEF:progress:Function]
|
||||
# @PURPOSE: Log a progress update with percentage.
|
||||
# @PRE: percent is between 0 and 100.
|
||||
# @POST: Log entry with progress metadata added.
|
||||
# @PARAM: message (str) - Progress message.
|
||||
# @PARAM: percent (float) - Progress percentage (0-100).
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
def progress(
|
||||
self,
|
||||
message: str,
|
||||
percent: float,
|
||||
source: Optional[str] = None
|
||||
) -> None:
|
||||
"""Log a progress update with percentage."""
|
||||
metadata = {"progress": min(100, max(0, percent))}
|
||||
self._log("INFO", message, source, metadata)
|
||||
# [/DEF:progress:Function]
|
||||
|
||||
# [/DEF:TaskLogger:Class]
|
||||
|
||||
# [/DEF:TaskLoggerModule:Module]
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.models.connection:Module]
|
||||
#
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: database, connection, configuration, sqlalchemy, sqlite
|
||||
# @PURPOSE: Defines the database schema for external database connection configurations.
|
||||
# @LAYER: Domain
|
||||
@@ -15,6 +16,7 @@ import uuid
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:ConnectionConfig:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Stores credentials for external databases used for column mapping.
|
||||
class ConnectionConfig(Base):
|
||||
__tablename__ = "connection_configs"
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:GitModels:Module]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: git, models, sqlalchemy, database, schema
|
||||
# @PURPOSE: Git-specific SQLAlchemy models for configuration and repository tracking.
|
||||
# @LAYER: Model
|
||||
@@ -26,11 +27,10 @@ class SyncStatus(str, enum.Enum):
|
||||
DIRTY = "DIRTY"
|
||||
CONFLICT = "CONFLICT"
|
||||
|
||||
# [DEF:GitServerConfig:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Configuration for a Git server connection.
|
||||
class GitServerConfig(Base):
|
||||
"""
|
||||
[DEF:GitServerConfig:Class]
|
||||
Configuration for a Git server connection.
|
||||
"""
|
||||
__tablename__ = "git_server_configs"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
@@ -41,12 +41,12 @@ class GitServerConfig(Base):
|
||||
default_repository = Column(String(255), nullable=True)
|
||||
status = Column(Enum(GitStatus), default=GitStatus.UNKNOWN)
|
||||
last_validated = Column(DateTime, default=datetime.utcnow)
|
||||
# [/DEF:GitServerConfig:Class]
|
||||
|
||||
# [DEF:GitRepository:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Tracking for a local Git repository linked to a dashboard.
|
||||
class GitRepository(Base):
|
||||
"""
|
||||
[DEF:GitRepository:Class]
|
||||
Tracking for a local Git repository linked to a dashboard.
|
||||
"""
|
||||
__tablename__ = "git_repositories"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
@@ -56,12 +56,12 @@ class GitRepository(Base):
|
||||
local_path = Column(String(255), nullable=False)
|
||||
current_branch = Column(String(255), default="main")
|
||||
sync_status = Column(Enum(SyncStatus), default=SyncStatus.CLEAN)
|
||||
# [/DEF:GitRepository:Class]
|
||||
|
||||
# [DEF:DeploymentEnvironment:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Target Superset environments for dashboard deployment.
|
||||
class DeploymentEnvironment(Base):
|
||||
"""
|
||||
[DEF:DeploymentEnvironment:Class]
|
||||
Target Superset environments for dashboard deployment.
|
||||
"""
|
||||
__tablename__ = "deployment_environments"
|
||||
|
||||
id = Column(String(36), primary_key=True, default=lambda: str(uuid.uuid4()))
|
||||
@@ -69,5 +69,6 @@ class DeploymentEnvironment(Base):
|
||||
superset_url = Column(String(255), nullable=False)
|
||||
superset_token = Column(String(255), nullable=False)
|
||||
is_active = Column(Boolean, default=True)
|
||||
# [/DEF:DeploymentEnvironment:Class]
|
||||
|
||||
# [/DEF:GitModels:Module]
|
||||
# [/DEF:GitModels:Module]
|
||||
|
||||
@@ -59,6 +59,7 @@ class DatabaseMapping(Base):
|
||||
# [/DEF:DatabaseMapping:Class]
|
||||
|
||||
# [DEF:MigrationJob:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Represents a single migration execution job.
|
||||
class MigrationJob(Base):
|
||||
__tablename__ = "migration_jobs"
|
||||
|
||||
@@ -1,9 +1,16 @@
|
||||
# [DEF:backend.src.models.storage:Module]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: storage, file, model, pydantic
|
||||
# @PURPOSE: Data models for the storage system.
|
||||
# @LAYER: Domain
|
||||
|
||||
from datetime import datetime
|
||||
from enum import Enum
|
||||
from typing import Optional
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
# [DEF:FileCategory:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Enumeration of supported file categories in the storage system.
|
||||
class FileCategory(str, Enum):
|
||||
BACKUP = "backups"
|
||||
@@ -11,6 +18,7 @@ class FileCategory(str, Enum):
|
||||
# [/DEF:FileCategory:Class]
|
||||
|
||||
# [DEF:StorageConfig:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Configuration model for the storage system, defining paths and naming patterns.
|
||||
class StorageConfig(BaseModel):
|
||||
root_path: str = Field(default="backups", description="Absolute path to the storage root directory.")
|
||||
@@ -20,6 +28,7 @@ class StorageConfig(BaseModel):
|
||||
# [/DEF:StorageConfig:Class]
|
||||
|
||||
# [DEF:StoredFile:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Data model representing metadata for a file stored in the system.
|
||||
class StoredFile(BaseModel):
|
||||
name: str = Field(..., description="Name of the file (including extension).")
|
||||
@@ -28,4 +37,6 @@ class StoredFile(BaseModel):
|
||||
created_at: datetime = Field(..., description="Creation timestamp.")
|
||||
category: FileCategory = Field(..., description="Category of the file.")
|
||||
mime_type: Optional[str] = Field(None, description="MIME type of the file.")
|
||||
# [/DEF:StoredFile:Class]
|
||||
# [/DEF:StoredFile:Class]
|
||||
|
||||
# [/DEF:backend.src.models.storage:Module]
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.models.task:Module]
|
||||
#
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: database, task, record, sqlalchemy, sqlite
|
||||
# @PURPOSE: Defines the database schema for task execution records.
|
||||
# @LAYER: Domain
|
||||
@@ -8,13 +9,14 @@
|
||||
# @INVARIANT: All primary keys are UUID strings.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from sqlalchemy import Column, String, DateTime, JSON, ForeignKey
|
||||
from sqlalchemy import Column, String, DateTime, JSON, ForeignKey, Text, Integer, Index
|
||||
from sqlalchemy.sql import func
|
||||
from .mapping import Base
|
||||
import uuid
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskRecord:Class]
|
||||
# @TIER: TRIVIAL
|
||||
# @PURPOSE: Represents a persistent record of a task execution.
|
||||
class TaskRecord(Base):
|
||||
__tablename__ = "task_records"
|
||||
@@ -25,11 +27,35 @@ class TaskRecord(Base):
|
||||
environment_id = Column(String, ForeignKey("environments.id"), nullable=True)
|
||||
started_at = Column(DateTime(timezone=True), nullable=True)
|
||||
finished_at = Column(DateTime(timezone=True), nullable=True)
|
||||
logs = Column(JSON, nullable=True) # Store structured logs as JSON
|
||||
logs = Column(JSON, nullable=True) # Store structured logs as JSON (legacy, kept for backward compatibility)
|
||||
error = Column(String, nullable=True)
|
||||
result = Column(JSON, nullable=True)
|
||||
created_at = Column(DateTime(timezone=True), server_default=func.now())
|
||||
params = Column(JSON, nullable=True)
|
||||
# [/DEF:TaskRecord:Class]
|
||||
|
||||
# [DEF:TaskLogRecord:Class]
|
||||
# @PURPOSE: Represents a single persistent log entry for a task.
|
||||
# @TIER: CRITICAL
|
||||
# @RELATION: DEPENDS_ON -> TaskRecord
|
||||
# @INVARIANT: Each log entry belongs to exactly one task.
|
||||
class TaskLogRecord(Base):
|
||||
__tablename__ = "task_logs"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
task_id = Column(String, ForeignKey("task_records.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
timestamp = Column(DateTime(timezone=True), nullable=False, index=True)
|
||||
level = Column(String(16), nullable=False) # INFO, WARNING, ERROR, DEBUG
|
||||
source = Column(String(64), nullable=False, default="system") # plugin, superset_api, git, etc.
|
||||
message = Column(Text, nullable=False)
|
||||
metadata_json = Column(Text, nullable=True) # JSON string for additional metadata
|
||||
|
||||
# Composite indexes for efficient filtering
|
||||
__table_args__ = (
|
||||
Index('ix_task_logs_task_timestamp', 'task_id', 'timestamp'),
|
||||
Index('ix_task_logs_task_level', 'task_id', 'level'),
|
||||
Index('ix_task_logs_task_source', 'task_id', 'source'),
|
||||
)
|
||||
# [/DEF:TaskLogRecord:Class]
|
||||
|
||||
# [/DEF:backend.src.models.task:Module]
|
||||
@@ -5,13 +5,14 @@
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
# @RELATION: USES -> TaskContext
|
||||
|
||||
from typing import Dict, Any
|
||||
from typing import Dict, Any, Optional
|
||||
from pathlib import Path
|
||||
from requests.exceptions import RequestException
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.logger import belief_scope
|
||||
from ..core.logger import belief_scope, logger as app_logger
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.utils.network import SupersetAPIError
|
||||
from ..core.utils.fileio import (
|
||||
@@ -23,6 +24,7 @@ from ..core.utils.fileio import (
|
||||
RetentionPolicy
|
||||
)
|
||||
from ..dependencies import get_config_manager
|
||||
from ..core.task_manager.context import TaskContext
|
||||
|
||||
# [DEF:BackupPlugin:Class]
|
||||
# @PURPOSE: Implementation of the backup plugin logic.
|
||||
@@ -110,11 +112,12 @@ class BackupPlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes the dashboard backup logic.
|
||||
# @PURPOSE: Executes the dashboard backup logic with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Backup parameters (env, backup_path).
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: Target environment must be configured. params must be a dictionary.
|
||||
# @POST: All dashboards are exported and archived.
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
with belief_scope("execute"):
|
||||
config_manager = get_config_manager()
|
||||
env_id = params.get("environment_id")
|
||||
@@ -133,8 +136,14 @@ class BackupPlugin(PluginBase):
|
||||
# Use 'backups' subfolder within the storage root
|
||||
backup_path = Path(storage_settings.root_path) / "backups"
|
||||
|
||||
from ..core.logger import logger as app_logger
|
||||
app_logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
|
||||
# Use TaskContext logger if available, otherwise fall back to app_logger
|
||||
log = context.logger if context else app_logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
storage_log = log.with_source("storage") if context else log
|
||||
|
||||
log.info(f"Starting backup for environment: {env}")
|
||||
|
||||
try:
|
||||
config_manager = get_config_manager()
|
||||
@@ -148,24 +157,30 @@ class BackupPlugin(PluginBase):
|
||||
client = SupersetClient(env_config)
|
||||
|
||||
dashboard_count, dashboard_meta = client.get_dashboards()
|
||||
app_logger.info(f"[BackupPlugin][Progress] Found {dashboard_count} dashboards to export in {env}.")
|
||||
superset_log.info(f"Found {dashboard_count} dashboards to export")
|
||||
|
||||
if dashboard_count == 0:
|
||||
app_logger.info("[BackupPlugin][Exit] No dashboards to back up.")
|
||||
log.info("No dashboards to back up")
|
||||
return
|
||||
|
||||
for db in dashboard_meta:
|
||||
total = len(dashboard_meta)
|
||||
for idx, db in enumerate(dashboard_meta, 1):
|
||||
dashboard_id = db.get('id')
|
||||
dashboard_title = db.get('dashboard_title', 'Unknown Dashboard')
|
||||
if not dashboard_id:
|
||||
continue
|
||||
|
||||
# Report progress
|
||||
progress_pct = (idx / total) * 100
|
||||
log.progress(f"Backing up dashboard: {dashboard_title}", percent=progress_pct)
|
||||
|
||||
try:
|
||||
dashboard_base_dir_name = sanitize_filename(f"{dashboard_title}")
|
||||
dashboard_dir = backup_path / env.upper() / dashboard_base_dir_name
|
||||
dashboard_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
zip_content, filename = client.export_dashboard(dashboard_id)
|
||||
superset_log.debug(f"Exported dashboard: {dashboard_title}")
|
||||
|
||||
save_and_unpack_dashboard(
|
||||
zip_content=zip_content,
|
||||
@@ -175,18 +190,19 @@ class BackupPlugin(PluginBase):
|
||||
)
|
||||
|
||||
archive_exports(str(dashboard_dir), policy=RetentionPolicy())
|
||||
storage_log.debug(f"Archived dashboard: {dashboard_title}")
|
||||
|
||||
except (SupersetAPIError, RequestException, IOError, OSError) as db_error:
|
||||
app_logger.error(f"[BackupPlugin][Failure] Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}", exc_info=True)
|
||||
log.error(f"Failed to export dashboard {dashboard_title} (ID: {dashboard_id}): {db_error}")
|
||||
continue
|
||||
|
||||
consolidate_archive_folders(backup_path / env.upper())
|
||||
remove_empty_directories(str(backup_path / env.upper()))
|
||||
|
||||
app_logger.info(f"[BackupPlugin][CoherenceCheck:Passed] Backup logic completed for {env}.")
|
||||
log.info(f"Backup completed successfully for {env}")
|
||||
|
||||
except (RequestException, IOError, KeyError) as e:
|
||||
app_logger.critical(f"[BackupPlugin][Failure] Fatal error during backup for {env}: {e}", exc_info=True)
|
||||
log.error(f"Fatal error during backup for {env}: {e}")
|
||||
raise e
|
||||
# [/DEF:execute:Function]
|
||||
# [/DEF:BackupPlugin:Class]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# @PURPOSE: Implements a plugin for system diagnostics and debugging Superset API responses.
|
||||
# @LAYER: Plugins
|
||||
# @RELATION: Inherits from PluginBase. Uses SupersetClient from core.
|
||||
# @RELATION: USES -> TaskContext
|
||||
# @CONSTRAINT: Must use belief_scope for logging.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
@@ -10,6 +11,7 @@ from typing import Dict, Any, Optional
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.logger import logger, belief_scope
|
||||
from ..core.task_manager.context import TaskContext
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:DebugPlugin:Class]
|
||||
@@ -114,20 +116,29 @@ class DebugPlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes the debug logic.
|
||||
# @PURPOSE: Executes the debug logic with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Debug parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: action must be provided in params.
|
||||
# @POST: Debug action is executed and results returned.
|
||||
# @RETURN: Dict[str, Any] - Execution results.
|
||||
async def execute(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:
|
||||
with belief_scope("execute"):
|
||||
action = params.get("action")
|
||||
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
debug_log = log.with_source("debug") if context else log
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
|
||||
debug_log.info(f"Executing debug action: {action}")
|
||||
|
||||
if action == "test-db-api":
|
||||
return await self._test_db_api(params)
|
||||
return await self._test_db_api(params, superset_log)
|
||||
elif action == "get-dataset-structure":
|
||||
return await self._get_dataset_structure(params)
|
||||
return await self._get_dataset_structure(params, superset_log)
|
||||
else:
|
||||
debug_log.error(f"Unknown action: {action}")
|
||||
raise ValueError(f"Unknown action: {action}")
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
@@ -136,33 +147,37 @@ class DebugPlugin(PluginBase):
|
||||
# @PRE: source_env and target_env params exist in params.
|
||||
# @POST: Returns DB counts for both envs.
|
||||
# @PARAM: params (Dict) - Plugin parameters.
|
||||
# @PARAM: log - Logger instance for superset_api source.
|
||||
# @RETURN: Dict - Comparison results.
|
||||
async def _test_db_api(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def _test_db_api(self, params: Dict[str, Any], log) -> Dict[str, Any]:
|
||||
with belief_scope("_test_db_api"):
|
||||
source_env_name = params.get("source_env")
|
||||
target_env_name = params.get("target_env")
|
||||
target_env_name = params.get("target_env")
|
||||
|
||||
if not source_env_name or not target_env_name:
|
||||
raise ValueError("source_env and target_env are required for test-db-api")
|
||||
if not source_env_name or not target_env_name:
|
||||
raise ValueError("source_env and target_env are required for test-db-api")
|
||||
|
||||
from ..dependencies import get_config_manager
|
||||
config_manager = get_config_manager()
|
||||
from ..dependencies import get_config_manager
|
||||
config_manager = get_config_manager()
|
||||
|
||||
results = {}
|
||||
for name in [source_env_name, target_env_name]:
|
||||
env_config = config_manager.get_environment(name)
|
||||
if not env_config:
|
||||
raise ValueError(f"Environment '{name}' not found.")
|
||||
results = {}
|
||||
for name in [source_env_name, target_env_name]:
|
||||
log.info(f"Testing database API for environment: {name}")
|
||||
env_config = config_manager.get_environment(name)
|
||||
if not env_config:
|
||||
log.error(f"Environment '{name}' not found.")
|
||||
raise ValueError(f"Environment '{name}' not found.")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
count, dbs = client.get_databases()
|
||||
results[name] = {
|
||||
"count": count,
|
||||
"databases": dbs
|
||||
}
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
count, dbs = client.get_databases()
|
||||
log.debug(f"Found {count} databases in {name}")
|
||||
results[name] = {
|
||||
"count": count,
|
||||
"databases": dbs
|
||||
}
|
||||
|
||||
return results
|
||||
return results
|
||||
# [/DEF:_test_db_api:Function]
|
||||
|
||||
# [DEF:_get_dataset_structure:Function]
|
||||
@@ -170,26 +185,31 @@ class DebugPlugin(PluginBase):
|
||||
# @PRE: env and dataset_id params exist in params.
|
||||
# @POST: Returns dataset JSON structure.
|
||||
# @PARAM: params (Dict) - Plugin parameters.
|
||||
# @PARAM: log - Logger instance for superset_api source.
|
||||
# @RETURN: Dict - Dataset structure.
|
||||
async def _get_dataset_structure(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def _get_dataset_structure(self, params: Dict[str, Any], log) -> Dict[str, Any]:
|
||||
with belief_scope("_get_dataset_structure"):
|
||||
env_name = params.get("env")
|
||||
dataset_id = params.get("dataset_id")
|
||||
dataset_id = params.get("dataset_id")
|
||||
|
||||
if not env_name or dataset_id is None:
|
||||
raise ValueError("env and dataset_id are required for get-dataset-structure")
|
||||
if not env_name or dataset_id is None:
|
||||
raise ValueError("env and dataset_id are required for get-dataset-structure")
|
||||
|
||||
from ..dependencies import get_config_manager
|
||||
config_manager = get_config_manager()
|
||||
env_config = config_manager.get_environment(env_name)
|
||||
if not env_config:
|
||||
raise ValueError(f"Environment '{env_name}' not found.")
|
||||
log.info(f"Fetching structure for dataset {dataset_id} in {env_name}")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
from ..dependencies import get_config_manager
|
||||
config_manager = get_config_manager()
|
||||
env_config = config_manager.get_environment(env_name)
|
||||
if not env_config:
|
||||
log.error(f"Environment '{env_name}' not found.")
|
||||
raise ValueError(f"Environment '{env_name}' not found.")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
|
||||
dataset_response = client.get_dataset(dataset_id)
|
||||
return dataset_response.get('result') or {}
|
||||
dataset_response = client.get_dataset(dataset_id)
|
||||
log.debug(f"Retrieved dataset structure for {dataset_id}")
|
||||
return dataset_response.get('result') or {}
|
||||
# [/DEF:_get_dataset_structure:Function]
|
||||
|
||||
# [/DEF:DebugPlugin:Class]
|
||||
|
||||
@@ -61,6 +61,7 @@ class GitLLMExtension:
|
||||
return "Update dashboard configurations (LLM generation failed)"
|
||||
|
||||
return response.choices[0].message.content.strip()
|
||||
# [/DEF:suggest_commit_message:Function]
|
||||
# [/DEF:GitLLMExtension:Class]
|
||||
|
||||
# [/DEF:backend/src/plugins/git/llm_extension:Module]
|
||||
@@ -7,6 +7,7 @@
|
||||
# @RELATION: USES -> src.services.git_service.GitService
|
||||
# @RELATION: USES -> src.core.superset_client.SupersetClient
|
||||
# @RELATION: USES -> src.core.config_manager.ConfigManager
|
||||
# @RELATION: USES -> TaskContext
|
||||
#
|
||||
# @INVARIANT: Все операции с Git должны выполняться через GitService.
|
||||
# @CONSTRAINT: Плагин работает только с распакованными YAML-экспортами Superset.
|
||||
@@ -20,9 +21,10 @@ from pathlib import Path
|
||||
from typing import Dict, Any, Optional
|
||||
from src.core.plugin_base import PluginBase
|
||||
from src.services.git_service import GitService
|
||||
from src.core.logger import logger, belief_scope
|
||||
from src.core.logger import logger as app_logger, belief_scope
|
||||
from src.core.config_manager import ConfigManager
|
||||
from src.core.superset_client import SupersetClient
|
||||
from src.core.task_manager.context import TaskContext
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:GitPlugin:Class]
|
||||
@@ -35,7 +37,7 @@ class GitPlugin(PluginBase):
|
||||
# @POST: Инициализированы git_service и config_manager.
|
||||
def __init__(self):
|
||||
with belief_scope("GitPlugin.__init__"):
|
||||
logger.info("[GitPlugin.__init__][Entry] Initializing GitPlugin.")
|
||||
app_logger.info("Initializing GitPlugin.")
|
||||
self.git_service = GitService()
|
||||
|
||||
# Robust config path resolution:
|
||||
@@ -50,13 +52,13 @@ class GitPlugin(PluginBase):
|
||||
try:
|
||||
from src.dependencies import config_manager
|
||||
self.config_manager = config_manager
|
||||
logger.info("[GitPlugin.__init__][Exit] GitPlugin initialized using shared config_manager.")
|
||||
app_logger.info("GitPlugin initialized using shared config_manager.")
|
||||
return
|
||||
except:
|
||||
config_path = "config.json"
|
||||
|
||||
self.config_manager = ConfigManager(config_path)
|
||||
logger.info(f"[GitPlugin.__init__][Exit] GitPlugin initialized with {config_path}")
|
||||
app_logger.info(f"GitPlugin initialized with {config_path}")
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
@property
|
||||
@@ -136,33 +138,41 @@ class GitPlugin(PluginBase):
|
||||
logger.info("[GitPlugin.initialize][Action] Initializing Git Integration Plugin logic.")
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Основной метод выполнения задач плагина.
|
||||
# @PURPOSE: Основной метод выполнения задач плагина с поддержкой TaskContext.
|
||||
# @PRE: task_data содержит 'operation' и 'dashboard_id'.
|
||||
# @POST: Возвращает результат выполнения операции.
|
||||
# @PARAM: task_data (Dict[str, Any]) - Данные задачи.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @RETURN: Dict[str, Any] - Статус и сообщение.
|
||||
# @RELATION: CALLS -> self._handle_sync
|
||||
# @RELATION: CALLS -> self._handle_deploy
|
||||
async def execute(self, task_data: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def execute(self, task_data: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:
|
||||
with belief_scope("GitPlugin.execute"):
|
||||
operation = task_data.get("operation")
|
||||
dashboard_id = task_data.get("dashboard_id")
|
||||
|
||||
logger.info(f"[GitPlugin.execute][Entry] Executing operation: {operation} for dashboard {dashboard_id}")
|
||||
# Use TaskContext logger if available, otherwise fall back to app_logger
|
||||
log = context.logger if context else app_logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
git_log = log.with_source("git") if context else log
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
|
||||
log.info(f"Executing operation: {operation} for dashboard {dashboard_id}")
|
||||
|
||||
if operation == "sync":
|
||||
source_env_id = task_data.get("source_env_id")
|
||||
result = await self._handle_sync(dashboard_id, source_env_id)
|
||||
result = await self._handle_sync(dashboard_id, source_env_id, log, git_log, superset_log)
|
||||
elif operation == "deploy":
|
||||
env_id = task_data.get("environment_id")
|
||||
result = await self._handle_deploy(dashboard_id, env_id)
|
||||
result = await self._handle_deploy(dashboard_id, env_id, log, git_log, superset_log)
|
||||
elif operation == "history":
|
||||
result = {"status": "success", "message": "History available via API"}
|
||||
else:
|
||||
logger.error(f"[GitPlugin.execute][Coherence:Failed] Unknown operation: {operation}")
|
||||
log.error(f"Unknown operation: {operation}")
|
||||
raise ValueError(f"Unknown operation: {operation}")
|
||||
|
||||
logger.info(f"[GitPlugin.execute][Exit] Operation {operation} completed.")
|
||||
log.info(f"Operation {operation} completed.")
|
||||
return result
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
@@ -176,13 +186,13 @@ class GitPlugin(PluginBase):
|
||||
# @SIDE_EFFECT: Изменяет файлы в локальной рабочей директории репозитория.
|
||||
# @RELATION: CALLS -> src.services.git_service.GitService.get_repo
|
||||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.export_dashboard
|
||||
async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None) -> Dict[str, str]:
|
||||
async def _handle_sync(self, dashboard_id: int, source_env_id: Optional[str] = None, log=None, git_log=None, superset_log=None) -> Dict[str, str]:
|
||||
with belief_scope("GitPlugin._handle_sync"):
|
||||
try:
|
||||
# 1. Получение репозитория
|
||||
repo = self.git_service.get_repo(dashboard_id)
|
||||
repo_path = Path(repo.working_dir)
|
||||
logger.info(f"[_handle_sync][Action] Target repo path: {repo_path}")
|
||||
git_log.info(f"Target repo path: {repo_path}")
|
||||
|
||||
# 2. Настройка клиента Superset
|
||||
env = self._get_env(source_env_id)
|
||||
@@ -190,11 +200,11 @@ class GitPlugin(PluginBase):
|
||||
client.authenticate()
|
||||
|
||||
# 3. Экспорт дашборда
|
||||
logger.info(f"[_handle_sync][Action] Exporting dashboard {dashboard_id} from {env.name}")
|
||||
superset_log.info(f"Exporting dashboard {dashboard_id} from {env.name}")
|
||||
zip_bytes, _ = client.export_dashboard(dashboard_id)
|
||||
|
||||
# 4. Распаковка с выравниванием структуры (flattening)
|
||||
logger.info(f"[_handle_sync][Action] Unpacking export to {repo_path}")
|
||||
git_log.info(f"Unpacking export to {repo_path}")
|
||||
|
||||
# Список папок/файлов, которые мы ожидаем от Superset
|
||||
managed_dirs = ["dashboards", "charts", "datasets", "databases"]
|
||||
@@ -218,7 +228,7 @@ class GitPlugin(PluginBase):
|
||||
raise ValueError("Export ZIP is empty")
|
||||
|
||||
root_folder = namelist[0].split('/')[0]
|
||||
logger.info(f"[_handle_sync][Action] Detected root folder in ZIP: {root_folder}")
|
||||
git_log.info(f"Detected root folder in ZIP: {root_folder}")
|
||||
|
||||
for member in zf.infolist():
|
||||
if member.filename.startswith(root_folder + "/") and len(member.filename) > len(root_folder) + 1:
|
||||
@@ -254,10 +264,13 @@ class GitPlugin(PluginBase):
|
||||
# @POST: Дашборд импортирован в целевой Superset.
|
||||
# @PARAM: dashboard_id (int) - ID дашборда.
|
||||
# @PARAM: env_id (str) - ID целевого окружения.
|
||||
# @PARAM: log - Main logger instance.
|
||||
# @PARAM: git_log - Git-specific logger instance.
|
||||
# @PARAM: superset_log - Superset API-specific logger instance.
|
||||
# @RETURN: Dict[str, Any] - Результат деплоя.
|
||||
# @SIDE_EFFECT: Создает и удаляет временный ZIP-файл.
|
||||
# @RELATION: CALLS -> src.core.superset_client.SupersetClient.import_dashboard
|
||||
async def _handle_deploy(self, dashboard_id: int, env_id: str) -> Dict[str, Any]:
|
||||
async def _handle_deploy(self, dashboard_id: int, env_id: str, log=None, git_log=None, superset_log=None) -> Dict[str, Any]:
|
||||
with belief_scope("GitPlugin._handle_deploy"):
|
||||
try:
|
||||
if not env_id:
|
||||
@@ -268,7 +281,7 @@ class GitPlugin(PluginBase):
|
||||
repo_path = Path(repo.working_dir)
|
||||
|
||||
# 2. Упаковка в ZIP
|
||||
logger.info(f"[_handle_deploy][Action] Packing repository {repo_path} for deployment.")
|
||||
git_log.info(f"Packing repository {repo_path} for deployment.")
|
||||
zip_buffer = io.BytesIO()
|
||||
|
||||
# Superset expects a root directory in the ZIP (e.g., dashboard_export_20240101T000000/)
|
||||
@@ -297,7 +310,7 @@ class GitPlugin(PluginBase):
|
||||
|
||||
# 4. Импорт
|
||||
temp_zip_path = repo_path / f"deploy_{dashboard_id}.zip"
|
||||
logger.info(f"[_handle_deploy][Action] Saving temporary zip to {temp_zip_path}")
|
||||
git_log.info(f"Saving temporary zip to {temp_zip_path}")
|
||||
with open(temp_zip_path, "wb") as f:
|
||||
f.write(zip_buffer.getvalue())
|
||||
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
# @RELATION: CALLS -> backend.src.plugins.llm_analysis.service.ScreenshotService
|
||||
# @RELATION: CALLS -> backend.src.plugins.llm_analysis.service.LLMClient
|
||||
# @RELATION: CALLS -> backend.src.services.llm_provider.LLMProviderService
|
||||
# @RELATION: USES -> TaskContext
|
||||
# @INVARIANT: All LLM interactions must be executed as asynchronous tasks.
|
||||
|
||||
from typing import Dict, Any, Optional, List
|
||||
@@ -23,6 +24,7 @@ from ...core.superset_client import SupersetClient
|
||||
from .service import ScreenshotService, LLMClient
|
||||
from .models import LLMProviderType, ValidationStatus, ValidationResult, DetectedIssue
|
||||
from ...models.llm import ValidationRecord
|
||||
from ...core.task_manager.context import TaskContext
|
||||
|
||||
# [DEF:DashboardValidationPlugin:Class]
|
||||
# @PURPOSE: Plugin for automated dashboard health analysis using LLMs.
|
||||
@@ -56,28 +58,27 @@ class DashboardValidationPlugin(PluginBase):
|
||||
}
|
||||
|
||||
# [DEF:DashboardValidationPlugin.execute:Function]
|
||||
# @PURPOSE: Executes the dashboard validation task.
|
||||
# @PURPOSE: Executes the dashboard validation task with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Validation parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: params contains dashboard_id, environment_id, and provider_id.
|
||||
# @POST: Returns a dictionary with validation results and persists them to the database.
|
||||
# @SIDE_EFFECT: Captures a screenshot, calls LLM API, and writes to the database.
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
with belief_scope("execute", f"plugin_id={self.id}"):
|
||||
logger.info(f"Executing {self.name} with params: {params}")
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
llm_log = log.with_source("llm") if context else log
|
||||
screenshot_log = log.with_source("screenshot") if context else log
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
|
||||
log.info(f"Executing {self.name} with params: {params}")
|
||||
|
||||
dashboard_id = params.get("dashboard_id")
|
||||
env_id = params.get("environment_id")
|
||||
provider_id = params.get("provider_id")
|
||||
task_id = params.get("_task_id")
|
||||
|
||||
# Helper to log to both app logger and task manager logs
|
||||
def task_log(level: str, message: str, context: Optional[Dict] = None):
|
||||
logger.log(getattr(logging, level.upper()), message)
|
||||
if task_id:
|
||||
from ...dependencies import get_task_manager
|
||||
try:
|
||||
tm = get_task_manager()
|
||||
tm._add_log(task_id, level.upper(), message, context)
|
||||
except: pass
|
||||
|
||||
db = SessionLocal()
|
||||
try:
|
||||
@@ -86,25 +87,26 @@ class DashboardValidationPlugin(PluginBase):
|
||||
config_mgr = get_config_manager()
|
||||
env = config_mgr.get_environment(env_id)
|
||||
if not env:
|
||||
log.error(f"Environment {env_id} not found")
|
||||
raise ValueError(f"Environment {env_id} not found")
|
||||
|
||||
# 2. Get LLM Provider
|
||||
llm_service = LLMProviderService(db)
|
||||
db_provider = llm_service.get_provider(provider_id)
|
||||
if not db_provider:
|
||||
log.error(f"LLM Provider {provider_id} not found")
|
||||
raise ValueError(f"LLM Provider {provider_id} not found")
|
||||
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Retrieved provider config:")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Provider ID: {db_provider.id}")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Provider Name: {db_provider.name}")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Provider Type: {db_provider.provider_type}")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Base URL: {db_provider.base_url}")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Default Model: {db_provider.default_model}")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] Is Active: {db_provider.is_active}")
|
||||
llm_log.debug(f"Retrieved provider config:")
|
||||
llm_log.debug(f" Provider ID: {db_provider.id}")
|
||||
llm_log.debug(f" Provider Name: {db_provider.name}")
|
||||
llm_log.debug(f" Provider Type: {db_provider.provider_type}")
|
||||
llm_log.debug(f" Base URL: {db_provider.base_url}")
|
||||
llm_log.debug(f" Default Model: {db_provider.default_model}")
|
||||
llm_log.debug(f" Is Active: {db_provider.is_active}")
|
||||
|
||||
api_key = llm_service.get_decrypted_api_key(provider_id)
|
||||
logger.info(f"[DashboardValidationPlugin.execute] API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
|
||||
logger.info(f"[DashboardValidationPlugin.execute] API Key Length: {len(api_key) if api_key else 0}")
|
||||
llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
|
||||
|
||||
# Check if API key was successfully decrypted
|
||||
if not api_key:
|
||||
@@ -124,7 +126,9 @@ class DashboardValidationPlugin(PluginBase):
|
||||
filename = f"{dashboard_id}_{datetime.now().strftime('%Y%m%d_%H%M%S')}.png"
|
||||
screenshot_path = os.path.join(screenshots_dir, filename)
|
||||
|
||||
screenshot_log.info(f"Capturing screenshot for dashboard {dashboard_id}")
|
||||
await screenshot_service.capture_dashboard(dashboard_id, screenshot_path)
|
||||
screenshot_log.debug(f"Screenshot saved to: {screenshot_path}")
|
||||
|
||||
# 4. Fetch Logs (from Environment /api/v1/log/)
|
||||
logs = []
|
||||
@@ -147,6 +151,7 @@ class DashboardValidationPlugin(PluginBase):
|
||||
"page_size": 100
|
||||
}
|
||||
|
||||
superset_log.debug(f"Fetching logs for dashboard {dashboard_id}")
|
||||
response = client.network.request(
|
||||
method="GET",
|
||||
endpoint="/log/",
|
||||
@@ -162,9 +167,10 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
if not logs:
|
||||
logs = ["No recent logs found for this dashboard."]
|
||||
superset_log.debug("No recent logs found for this dashboard")
|
||||
|
||||
except Exception as e:
|
||||
logger.warning(f"Failed to fetch logs from environment: {e}")
|
||||
superset_log.warning(f"Failed to fetch logs from environment: {e}")
|
||||
logs = [f"Error fetching remote logs: {str(e)}"]
|
||||
|
||||
# 5. Analyze with LLM
|
||||
@@ -175,14 +181,15 @@ class DashboardValidationPlugin(PluginBase):
|
||||
default_model=db_provider.default_model
|
||||
)
|
||||
|
||||
llm_log.info(f"Analyzing dashboard {dashboard_id} with LLM")
|
||||
analysis = await llm_client.analyze_dashboard(screenshot_path, logs)
|
||||
|
||||
# Log analysis summary to task logs for better visibility
|
||||
task_log("INFO", f"[ANALYSIS_SUMMARY] Status: {analysis['status']}")
|
||||
task_log("INFO", f"[ANALYSIS_SUMMARY] Summary: {analysis['summary']}")
|
||||
llm_log.info(f"[ANALYSIS_SUMMARY] Status: {analysis['status']}")
|
||||
llm_log.info(f"[ANALYSIS_SUMMARY] Summary: {analysis['summary']}")
|
||||
if analysis.get("issues"):
|
||||
for i, issue in enumerate(analysis["issues"]):
|
||||
task_log("INFO", f"[ANALYSIS_ISSUE][{i+1}] {issue.get('severity')}: {issue.get('message')} (Location: {issue.get('location', 'N/A')})")
|
||||
llm_log.info(f"[ANALYSIS_ISSUE][{i+1}] {issue.get('severity')}: {issue.get('message')} (Location: {issue.get('location', 'N/A')})")
|
||||
|
||||
# 6. Persist Result
|
||||
validation_result = ValidationResult(
|
||||
@@ -207,13 +214,13 @@ class DashboardValidationPlugin(PluginBase):
|
||||
|
||||
# 7. Notification on failure (US1 / FR-015)
|
||||
if validation_result.status == ValidationStatus.FAIL:
|
||||
task_log("WARNING", f"Dashboard {dashboard_id} validation FAILED. Summary: {validation_result.summary}")
|
||||
log.warning(f"Dashboard {dashboard_id} validation FAILED. Summary: {validation_result.summary}")
|
||||
# Placeholder for Email/Pulse notification dispatch
|
||||
# In a real implementation, we would call a NotificationService here
|
||||
# with a payload containing the summary and a link to the report.
|
||||
|
||||
# Final log to ensure all analysis is visible in task logs
|
||||
task_log("INFO", f"Validation completed for dashboard {dashboard_id}. Status: {validation_result.status.value}")
|
||||
log.info(f"Validation completed for dashboard {dashboard_id}. Status: {validation_result.status.value}")
|
||||
|
||||
return validation_result.dict()
|
||||
|
||||
@@ -254,13 +261,22 @@ class DocumentationPlugin(PluginBase):
|
||||
}
|
||||
|
||||
# [DEF:DocumentationPlugin.execute:Function]
|
||||
# @PURPOSE: Executes the dataset documentation task.
|
||||
# @PURPOSE: Executes the dataset documentation task with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Documentation parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: params contains dataset_id, environment_id, and provider_id.
|
||||
# @POST: Returns generated documentation and updates the dataset in Superset.
|
||||
# @SIDE_EFFECT: Calls LLM API and updates dataset metadata in Superset.
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
with belief_scope("execute", f"plugin_id={self.id}"):
|
||||
logger.info(f"Executing {self.name} with params: {params}")
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
llm_log = log.with_source("llm") if context else log
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
|
||||
log.info(f"Executing {self.name} with params: {params}")
|
||||
|
||||
dataset_id = params.get("dataset_id")
|
||||
env_id = params.get("environment_id")
|
||||
@@ -273,25 +289,25 @@ class DocumentationPlugin(PluginBase):
|
||||
config_mgr = get_config_manager()
|
||||
env = config_mgr.get_environment(env_id)
|
||||
if not env:
|
||||
log.error(f"Environment {env_id} not found")
|
||||
raise ValueError(f"Environment {env_id} not found")
|
||||
|
||||
# 2. Get LLM Provider
|
||||
llm_service = LLMProviderService(db)
|
||||
db_provider = llm_service.get_provider(provider_id)
|
||||
if not db_provider:
|
||||
log.error(f"LLM Provider {provider_id} not found")
|
||||
raise ValueError(f"LLM Provider {provider_id} not found")
|
||||
|
||||
logger.info(f"[DocumentationPlugin.execute] Retrieved provider config:")
|
||||
logger.info(f"[DocumentationPlugin.execute] Provider ID: {db_provider.id}")
|
||||
logger.info(f"[DocumentationPlugin.execute] Provider Name: {db_provider.name}")
|
||||
logger.info(f"[DocumentationPlugin.execute] Provider Type: {db_provider.provider_type}")
|
||||
logger.info(f"[DocumentationPlugin.execute] Base URL: {db_provider.base_url}")
|
||||
logger.info(f"[DocumentationPlugin.execute] Default Model: {db_provider.default_model}")
|
||||
logger.info(f"[DocumentationPlugin.execute] Is Active: {db_provider.is_active}")
|
||||
llm_log.debug(f"Retrieved provider config:")
|
||||
llm_log.debug(f" Provider ID: {db_provider.id}")
|
||||
llm_log.debug(f" Provider Name: {db_provider.name}")
|
||||
llm_log.debug(f" Provider Type: {db_provider.provider_type}")
|
||||
llm_log.debug(f" Base URL: {db_provider.base_url}")
|
||||
llm_log.debug(f" Default Model: {db_provider.default_model}")
|
||||
|
||||
api_key = llm_service.get_decrypted_api_key(provider_id)
|
||||
logger.info(f"[DocumentationPlugin.execute] API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
|
||||
logger.info(f"[DocumentationPlugin.execute] API Key Length: {len(api_key) if api_key else 0}")
|
||||
llm_log.debug(f"API Key decrypted (first 8 chars): {api_key[:8] if api_key and len(api_key) > 8 else 'EMPTY_OR_NONE'}...")
|
||||
|
||||
# Check if API key was successfully decrypted
|
||||
if not api_key:
|
||||
@@ -305,10 +321,8 @@ class DocumentationPlugin(PluginBase):
|
||||
from ...core.superset_client import SupersetClient
|
||||
client = SupersetClient(env)
|
||||
|
||||
# Optimistic locking check (T045)
|
||||
superset_log.debug(f"Fetching dataset {dataset_id}")
|
||||
dataset = client.get_dataset(int(dataset_id))
|
||||
# dataset structure might vary, ensure we get the right field
|
||||
original_changed_on = dataset.get("changed_on_utc") or dataset.get("result", {}).get("changed_on_utc")
|
||||
|
||||
# Extract columns and existing descriptions
|
||||
columns_data = []
|
||||
@@ -318,6 +332,7 @@ class DocumentationPlugin(PluginBase):
|
||||
"type": col.get("type"),
|
||||
"description": col.get("description")
|
||||
})
|
||||
superset_log.debug(f"Extracted {len(columns_data)} columns from dataset")
|
||||
|
||||
# 4. Construct Prompt & Analyze (US2 / T025)
|
||||
llm_client = LLMClient(
|
||||
@@ -345,12 +360,10 @@ class DocumentationPlugin(PluginBase):
|
||||
"""
|
||||
|
||||
# Using a generic chat completion for text-only US2
|
||||
# We use the shared get_json_completion method from LLMClient
|
||||
llm_log.info(f"Generating documentation for dataset {dataset_id}")
|
||||
doc_result = await llm_client.get_json_completion([{"role": "user", "content": prompt}])
|
||||
|
||||
# 5. Update Metadata (US2 / T026)
|
||||
# This part normally goes to mapping_service, but we implement the logic here for the plugin flow
|
||||
# We'll update the dataset in Superset
|
||||
update_payload = {
|
||||
"description": doc_result["dataset_description"],
|
||||
"columns": []
|
||||
@@ -365,8 +378,11 @@ class DocumentationPlugin(PluginBase):
|
||||
"description": col_doc["description"]
|
||||
})
|
||||
|
||||
superset_log.info(f"Updating dataset {dataset_id} with generated documentation")
|
||||
client.update_dataset(int(dataset_id), update_payload)
|
||||
|
||||
log.info(f"Documentation completed for dataset {dataset_id}")
|
||||
|
||||
return doc_result
|
||||
|
||||
finally:
|
||||
|
||||
@@ -39,6 +39,7 @@ def schedule_dashboard_validation(dashboard_id: str, cron_expression: str, param
|
||||
**_parse_cron(cron_expression)
|
||||
)
|
||||
logger.info(f"Scheduled validation for dashboard {dashboard_id} with cron {cron_expression}")
|
||||
# [/DEF:schedule_dashboard_validation:Function]
|
||||
|
||||
# [DEF:_parse_cron:Function]
|
||||
# @PURPOSE: Basic cron parser placeholder.
|
||||
@@ -56,5 +57,6 @@ def _parse_cron(cron: str) -> Dict[str, str]:
|
||||
"month": parts[3],
|
||||
"day_of_week": parts[4]
|
||||
}
|
||||
# [/DEF:_parse_cron:Function]
|
||||
|
||||
# [/DEF:backend/src/plugins/llm_analysis/scheduler.py:Module]
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# @PURPOSE: Implements a plugin for mapping dataset columns using external database connections or Excel files.
|
||||
# @LAYER: Plugins
|
||||
# @RELATION: Inherits from PluginBase. Uses DatasetMapper from superset_tool.
|
||||
# @RELATION: USES -> TaskContext
|
||||
# @CONSTRAINT: Must use belief_scope for logging.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
@@ -13,6 +14,7 @@ from ..core.logger import logger, belief_scope
|
||||
from ..core.database import SessionLocal
|
||||
from ..models.connection import ConnectionConfig
|
||||
from ..core.utils.dataset_mapper import DatasetMapper
|
||||
from ..core.task_manager.context import TaskContext
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:MapperPlugin:Class]
|
||||
@@ -128,19 +130,27 @@ class MapperPlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes the dataset mapping logic.
|
||||
# @PURPOSE: Executes the dataset mapping logic with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Mapping parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: Params contain valid 'env', 'dataset_id', and 'source'. params must be a dictionary.
|
||||
# @POST: Updates the dataset in Superset.
|
||||
# @RETURN: Dict[str, Any] - Execution status.
|
||||
async def execute(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:
|
||||
with belief_scope("execute"):
|
||||
env_name = params.get("env")
|
||||
dataset_id = params.get("dataset_id")
|
||||
source = params.get("source")
|
||||
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
db_log = log.with_source("postgres") if context else log
|
||||
|
||||
if not env_name or dataset_id is None or not source:
|
||||
logger.error("[MapperPlugin.execute][State] Missing required parameters.")
|
||||
log.error("Missing required parameters: env, dataset_id, source")
|
||||
raise ValueError("Missing required parameters: env, dataset_id, source")
|
||||
|
||||
# Get config and initialize client
|
||||
@@ -148,7 +158,7 @@ class MapperPlugin(PluginBase):
|
||||
config_manager = get_config_manager()
|
||||
env_config = config_manager.get_environment(env_name)
|
||||
if not env_config:
|
||||
logger.error(f"[MapperPlugin.execute][State] Environment '{env_name}' not found.")
|
||||
log.error(f"Environment '{env_name}' not found in configuration.")
|
||||
raise ValueError(f"Environment '{env_name}' not found in configuration.")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
@@ -158,7 +168,7 @@ class MapperPlugin(PluginBase):
|
||||
if source == "postgres":
|
||||
connection_id = params.get("connection_id")
|
||||
if not connection_id:
|
||||
logger.error("[MapperPlugin.execute][State] connection_id is required for postgres source.")
|
||||
log.error("connection_id is required for postgres source.")
|
||||
raise ValueError("connection_id is required for postgres source.")
|
||||
|
||||
# Load connection from DB
|
||||
@@ -166,7 +176,7 @@ class MapperPlugin(PluginBase):
|
||||
try:
|
||||
conn_config = db.query(ConnectionConfig).filter(ConnectionConfig.id == connection_id).first()
|
||||
if not conn_config:
|
||||
logger.error(f"[MapperPlugin.execute][State] Connection {connection_id} not found.")
|
||||
db_log.error(f"Connection {connection_id} not found.")
|
||||
raise ValueError(f"Connection {connection_id} not found.")
|
||||
|
||||
postgres_config = {
|
||||
@@ -176,10 +186,11 @@ class MapperPlugin(PluginBase):
|
||||
'host': conn_config.host,
|
||||
'port': str(conn_config.port) if conn_config.port else '5432'
|
||||
}
|
||||
db_log.debug(f"Loaded connection config for {conn_config.host}:{conn_config.port}/{conn_config.database}")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
logger.info(f"[MapperPlugin.execute][Action] Starting mapping for dataset {dataset_id} in {env_name}")
|
||||
log.info(f"Starting mapping for dataset {dataset_id} in {env_name}")
|
||||
|
||||
mapper = DatasetMapper()
|
||||
|
||||
@@ -193,10 +204,10 @@ class MapperPlugin(PluginBase):
|
||||
table_name=params.get("table_name"),
|
||||
table_schema=params.get("table_schema") or "public"
|
||||
)
|
||||
logger.info(f"[MapperPlugin.execute][Success] Mapping completed for dataset {dataset_id}")
|
||||
superset_log.info(f"Mapping completed for dataset {dataset_id}")
|
||||
return {"status": "success", "dataset_id": dataset_id}
|
||||
except Exception as e:
|
||||
logger.error(f"[MapperPlugin.execute][Failure] Mapping failed: {e}")
|
||||
log.error(f"Mapping failed: {e}")
|
||||
raise
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
|
||||
@@ -5,20 +5,22 @@
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.client
|
||||
# @RELATION: DEPENDS_ON -> superset_tool.utils
|
||||
# @RELATION: USES -> TaskContext
|
||||
|
||||
from typing import Dict, Any, List
|
||||
from typing import Dict, Any, List, Optional
|
||||
from pathlib import Path
|
||||
import zipfile
|
||||
import re
|
||||
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.logger import belief_scope
|
||||
from ..core.logger import belief_scope, logger as app_logger
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.utils.fileio import create_temp_file, update_yamls, create_dashboard_export
|
||||
from ..dependencies import get_config_manager
|
||||
from ..core.migration_engine import MigrationEngine
|
||||
from ..core.database import SessionLocal
|
||||
from ..models.mapping import DatabaseMapping, Environment
|
||||
from ..core.task_manager.context import TaskContext
|
||||
|
||||
# [DEF:MigrationPlugin:Class]
|
||||
# @PURPOSE: Implementation of the migration plugin logic.
|
||||
@@ -132,11 +134,12 @@ class MigrationPlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes the dashboard migration logic.
|
||||
# @PURPOSE: Executes the dashboard migration logic with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Migration parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: Source and target environments must be configured.
|
||||
# @POST: Selected dashboards are migrated.
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
with belief_scope("MigrationPlugin.execute"):
|
||||
source_env_id = params.get("source_env_id")
|
||||
target_env_id = params.get("target_env_id")
|
||||
@@ -157,74 +160,15 @@ class MigrationPlugin(PluginBase):
|
||||
from ..dependencies import get_task_manager
|
||||
tm = get_task_manager()
|
||||
|
||||
class TaskLoggerProxy:
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initializes the proxy logger.
|
||||
# @PRE: None.
|
||||
# @POST: Instance is initialized.
|
||||
def __init__(self):
|
||||
with belief_scope("__init__"):
|
||||
# Initialize parent with dummy values since we override methods
|
||||
pass
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:debug:Function]
|
||||
# @PURPOSE: Logs a debug message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def debug(self, msg, *args, extra=None, **kwargs):
|
||||
with belief_scope("debug"):
|
||||
if task_id: tm._add_log(task_id, "DEBUG", msg, extra or {})
|
||||
# [/DEF:debug:Function]
|
||||
|
||||
# [DEF:info:Function]
|
||||
# @PURPOSE: Logs an info message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def info(self, msg, *args, extra=None, **kwargs):
|
||||
with belief_scope("info"):
|
||||
if task_id: tm._add_log(task_id, "INFO", msg, extra or {})
|
||||
# [/DEF:info:Function]
|
||||
|
||||
# [DEF:warning:Function]
|
||||
# @PURPOSE: Logs a warning message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def warning(self, msg, *args, extra=None, **kwargs):
|
||||
with belief_scope("warning"):
|
||||
if task_id: tm._add_log(task_id, "WARNING", msg, extra or {})
|
||||
# [/DEF:warning:Function]
|
||||
|
||||
# [DEF:error:Function]
|
||||
# @PURPOSE: Logs an error message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def error(self, msg, *args, extra=None, **kwargs):
|
||||
with belief_scope("error"):
|
||||
if task_id: tm._add_log(task_id, "ERROR", msg, extra or {})
|
||||
# [/DEF:error:Function]
|
||||
|
||||
# [DEF:critical:Function]
|
||||
# @PURPOSE: Logs a critical message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def critical(self, msg, *args, extra=None, **kwargs):
|
||||
with belief_scope("critical"):
|
||||
if task_id: tm._add_log(task_id, "ERROR", msg, extra or {})
|
||||
# [/DEF:critical:Function]
|
||||
|
||||
# [DEF:exception:Function]
|
||||
# @PURPOSE: Logs an exception message to the task manager.
|
||||
# @PRE: msg is a string.
|
||||
# @POST: Log is added to task manager if task_id exists.
|
||||
def exception(self, msg, *args, **kwargs):
|
||||
with belief_scope("exception"):
|
||||
if task_id: tm._add_log(task_id, "ERROR", msg, {"exception": True})
|
||||
# [/DEF:exception:Function]
|
||||
|
||||
logger = TaskLoggerProxy()
|
||||
logger.info(f"[MigrationPlugin][Entry] Starting migration task.")
|
||||
logger.info(f"[MigrationPlugin][Action] Params: {params}")
|
||||
# Use TaskContext logger if available, otherwise fall back to app_logger
|
||||
log = context.logger if context else app_logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
migration_log = log.with_source("migration") if context else log
|
||||
|
||||
log.info("Starting migration task.")
|
||||
log.debug(f"Params: {params}")
|
||||
|
||||
try:
|
||||
with belief_scope("execute"):
|
||||
@@ -251,7 +195,7 @@ class MigrationPlugin(PluginBase):
|
||||
from_env_name = src_env.name
|
||||
to_env_name = tgt_env.name
|
||||
|
||||
logger.info(f"[MigrationPlugin][State] Resolved environments: {from_env_name} -> {to_env_name}")
|
||||
log.info(f"Resolved environments: {from_env_name} -> {to_env_name}")
|
||||
|
||||
from_c = SupersetClient(src_env)
|
||||
to_c = SupersetClient(tgt_env)
|
||||
@@ -270,11 +214,11 @@ class MigrationPlugin(PluginBase):
|
||||
d for d in all_dashboards if re.search(regex_str, d["dashboard_title"], re.IGNORECASE)
|
||||
]
|
||||
else:
|
||||
logger.warning("[MigrationPlugin][State] No selection criteria provided (selected_ids or dashboard_regex).")
|
||||
log.warning("No selection criteria provided (selected_ids or dashboard_regex).")
|
||||
return
|
||||
|
||||
if not dashboards_to_migrate:
|
||||
logger.warning("[MigrationPlugin][State] No dashboards found matching criteria.")
|
||||
log.warning("No dashboards found matching criteria.")
|
||||
return
|
||||
|
||||
# Fetch mappings from database
|
||||
@@ -292,7 +236,7 @@ class MigrationPlugin(PluginBase):
|
||||
DatabaseMapping.target_env_id == tgt_env.id
|
||||
).all()
|
||||
db_mapping = {m.source_db_uuid: m.target_db_uuid for m in mappings}
|
||||
logger.info(f"[MigrationPlugin][State] Loaded {len(db_mapping)} database mappings.")
|
||||
log.info(f"Loaded {len(db_mapping)} database mappings.")
|
||||
finally:
|
||||
db.close()
|
||||
|
||||
@@ -311,7 +255,7 @@ class MigrationPlugin(PluginBase):
|
||||
if not success and replace_db_config:
|
||||
# Signal missing mapping and wait (only if we care about mappings)
|
||||
if task_id:
|
||||
logger.info(f"[MigrationPlugin][Action] Pausing for missing mapping in task {task_id}")
|
||||
log.info(f"Pausing for missing mapping in task {task_id}")
|
||||
# In a real scenario, we'd pass the missing DB info to the frontend
|
||||
# For this task, we'll just simulate the wait
|
||||
await tm.wait_for_resolution(task_id)
|
||||
@@ -333,9 +277,9 @@ class MigrationPlugin(PluginBase):
|
||||
if success:
|
||||
to_c.import_dashboard(file_name=tmp_new_zip, dash_id=dash_id, dash_slug=dash_slug)
|
||||
else:
|
||||
logger.error(f"[MigrationPlugin][Failure] Failed to transform ZIP for dashboard {title}")
|
||||
migration_log.error(f"Failed to transform ZIP for dashboard {title}")
|
||||
|
||||
logger.info(f"[MigrationPlugin][Success] Dashboard {title} imported.")
|
||||
superset_log.info(f"Dashboard {title} imported.")
|
||||
except Exception as exc:
|
||||
# Check for password error
|
||||
error_msg = str(exc)
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
# @PURPOSE: Implements a plugin for searching text patterns across all datasets in a specific Superset environment.
|
||||
# @LAYER: Plugins
|
||||
# @RELATION: Inherits from PluginBase. Uses SupersetClient from core.
|
||||
# @RELATION: USES -> TaskContext
|
||||
# @CONSTRAINT: Must use belief_scope for logging.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
@@ -11,6 +12,7 @@ from typing import Dict, Any, List, Optional
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.logger import logger, belief_scope
|
||||
from ..core.task_manager.context import TaskContext
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:SearchPlugin:Class]
|
||||
@@ -99,18 +101,26 @@ class SearchPlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes the dataset search logic.
|
||||
# @PURPOSE: Executes the dataset search logic with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Search parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: Params contain valid 'env' and 'query'.
|
||||
# @POST: Returns a dictionary with count and results list.
|
||||
# @RETURN: Dict[str, Any] - Search results.
|
||||
async def execute(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None) -> Dict[str, Any]:
|
||||
with belief_scope("SearchPlugin.execute", f"params={params}"):
|
||||
env_name = params.get("env")
|
||||
search_query = params.get("query")
|
||||
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
superset_log = log.with_source("superset_api") if context else log
|
||||
search_log = log.with_source("search") if context else log
|
||||
|
||||
if not env_name or not search_query:
|
||||
logger.error("[SearchPlugin.execute][State] Missing required parameters.")
|
||||
log.error("Missing required parameters: env, query")
|
||||
raise ValueError("Missing required parameters: env, query")
|
||||
|
||||
# Get config and initialize client
|
||||
@@ -118,20 +128,20 @@ class SearchPlugin(PluginBase):
|
||||
config_manager = get_config_manager()
|
||||
env_config = config_manager.get_environment(env_name)
|
||||
if not env_config:
|
||||
logger.error(f"[SearchPlugin.execute][State] Environment '{env_name}' not found.")
|
||||
log.error(f"Environment '{env_name}' not found in configuration.")
|
||||
raise ValueError(f"Environment '{env_name}' not found in configuration.")
|
||||
|
||||
client = SupersetClient(env_config)
|
||||
client.authenticate()
|
||||
|
||||
logger.info(f"[SearchPlugin.execute][Action] Searching for pattern: '{search_query}' in environment: {env_name}")
|
||||
log.info(f"Searching for pattern: '{search_query}' in environment: {env_name}")
|
||||
|
||||
try:
|
||||
# Ported logic from search_script.py
|
||||
_, datasets = client.get_datasets(query={"columns": ["id", "table_name", "sql", "database", "columns"]})
|
||||
|
||||
if not datasets:
|
||||
logger.warning("[SearchPlugin.execute][State] No datasets found.")
|
||||
search_log.warning("No datasets found.")
|
||||
return {"count": 0, "results": []}
|
||||
|
||||
pattern = re.compile(search_query, re.IGNORECASE)
|
||||
@@ -155,17 +165,17 @@ class SearchPlugin(PluginBase):
|
||||
"full_value": value_str
|
||||
})
|
||||
|
||||
logger.info(f"[SearchPlugin.execute][Success] Found matches in {len(results)} locations.")
|
||||
search_log.info(f"Found matches in {len(results)} locations.")
|
||||
return {
|
||||
"count": len(results),
|
||||
"results": results
|
||||
}
|
||||
|
||||
except re.error as e:
|
||||
logger.error(f"[SearchPlugin.execute][Failure] Invalid regex pattern: {e}")
|
||||
search_log.error(f"Invalid regex pattern: {e}")
|
||||
raise ValueError(f"Invalid regex pattern: {e}")
|
||||
except Exception as e:
|
||||
logger.error(f"[SearchPlugin.execute][Failure] Error during search: {e}")
|
||||
log.error(f"Error during search: {e}")
|
||||
raise
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
# @LAYER: App
|
||||
# @RELATION: IMPLEMENTS -> PluginBase
|
||||
# @RELATION: DEPENDS_ON -> backend.src.models.storage
|
||||
# @RELATION: USES -> TaskContext
|
||||
#
|
||||
# @INVARIANT: All file operations must be restricted to the configured storage root.
|
||||
|
||||
@@ -20,6 +21,7 @@ from ...core.plugin_base import PluginBase
|
||||
from ...core.logger import belief_scope, logger
|
||||
from ...models.storage import StoredFile, FileCategory, StorageConfig
|
||||
from ...dependencies import get_config_manager
|
||||
from ...core.task_manager.context import TaskContext
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:StoragePlugin:Class]
|
||||
@@ -112,12 +114,21 @@ class StoragePlugin(PluginBase):
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
# [DEF:execute:Function]
|
||||
# @PURPOSE: Executes storage-related tasks (placeholder for PluginBase compliance).
|
||||
# @PURPOSE: Executes storage-related tasks with TaskContext support.
|
||||
# @PARAM: params (Dict[str, Any]) - Storage parameters.
|
||||
# @PARAM: context (Optional[TaskContext]) - Task context for logging with source attribution.
|
||||
# @PRE: params must match the plugin schema.
|
||||
# @POST: Task is executed and logged.
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
with belief_scope("StoragePlugin:execute"):
|
||||
logger.info(f"[StoragePlugin][Action] Executing with params: {params}")
|
||||
# Use TaskContext logger if available, otherwise fall back to app logger
|
||||
log = context.logger if context else logger
|
||||
|
||||
# Create sub-loggers for different components
|
||||
storage_log = log.with_source("storage") if context else log
|
||||
filesystem_log = log.with_source("filesystem") if context else log
|
||||
|
||||
storage_log.info(f"Executing with params: {params}")
|
||||
# [/DEF:execute:Function]
|
||||
|
||||
# [DEF:get_storage_root:Function]
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.scripts.create_admin:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: admin, setup, user, auth, cli
|
||||
# @PURPOSE: CLI tool for creating the initial admin user.
|
||||
# @LAYER: Scripts
|
||||
|
||||
@@ -15,43 +15,74 @@ from cryptography.fernet import Fernet
|
||||
import os
|
||||
|
||||
# [DEF:EncryptionManager:Class]
|
||||
# @TIER: CRITICAL
|
||||
# @PURPOSE: Handles encryption and decryption of sensitive data like API keys.
|
||||
# @INVARIANT: Uses a secret key from environment or a default one (fallback only for dev).
|
||||
class EncryptionManager:
|
||||
# @INVARIANT: Uses a secret key from environment or a default one (fallback only for dev).
|
||||
# [DEF:EncryptionManager.__init__:Function]
|
||||
# @PURPOSE: Initialize the encryption manager with a Fernet key.
|
||||
# @PRE: ENCRYPTION_KEY env var must be set or use default dev key.
|
||||
# @POST: Fernet instance ready for encryption/decryption.
|
||||
def __init__(self):
|
||||
self.key = os.getenv("ENCRYPTION_KEY", "ZcytYzi0iHIl4Ttr-GdAEk117aGRogkGvN3wiTxrPpE=").encode()
|
||||
self.fernet = Fernet(self.key)
|
||||
# [/DEF:EncryptionManager.__init__:Function]
|
||||
|
||||
# [DEF:EncryptionManager.encrypt:Function]
|
||||
# @PURPOSE: Encrypt a plaintext string.
|
||||
# @PRE: data must be a non-empty string.
|
||||
# @POST: Returns encrypted string.
|
||||
def encrypt(self, data: str) -> str:
|
||||
return self.fernet.encrypt(data.encode()).decode()
|
||||
# [/DEF:EncryptionManager.encrypt:Function]
|
||||
|
||||
# [DEF:EncryptionManager.decrypt:Function]
|
||||
# @PURPOSE: Decrypt an encrypted string.
|
||||
# @PRE: encrypted_data must be a valid Fernet-encrypted string.
|
||||
# @POST: Returns original plaintext string.
|
||||
def decrypt(self, encrypted_data: str) -> str:
|
||||
return self.fernet.decrypt(encrypted_data.encode()).decode()
|
||||
# [/DEF:EncryptionManager.decrypt:Function]
|
||||
# [/DEF:EncryptionManager:Class]
|
||||
|
||||
# [DEF:LLMProviderService:Class]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Service to manage LLM provider lifecycle.
|
||||
class LLMProviderService:
|
||||
# [DEF:LLMProviderService.__init__:Function]
|
||||
# @PURPOSE: Initialize the service with database session.
|
||||
# @PRE: db must be a valid SQLAlchemy Session.
|
||||
# @POST: Service ready for provider operations.
|
||||
def __init__(self, db: Session):
|
||||
self.db = db
|
||||
self.encryption = EncryptionManager()
|
||||
# [/DEF:LLMProviderService.__init__:Function]
|
||||
|
||||
# [DEF:get_all_providers:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Returns all configured LLM providers.
|
||||
# @PRE: Database connection must be active.
|
||||
# @POST: Returns list of all LLMProvider records.
|
||||
def get_all_providers(self) -> List[LLMProvider]:
|
||||
with belief_scope("get_all_providers"):
|
||||
return self.db.query(LLMProvider).all()
|
||||
# [/DEF:get_all_providers:Function]
|
||||
|
||||
# [DEF:get_provider:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Returns a single LLM provider by ID.
|
||||
# @PRE: provider_id must be a valid string.
|
||||
# @POST: Returns LLMProvider or None if not found.
|
||||
def get_provider(self, provider_id: str) -> Optional[LLMProvider]:
|
||||
with belief_scope("get_provider"):
|
||||
return self.db.query(LLMProvider).filter(LLMProvider.id == provider_id).first()
|
||||
# [/DEF:get_provider:Function]
|
||||
|
||||
# [DEF:create_provider:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Creates a new LLM provider with encrypted API key.
|
||||
# @PRE: config must contain valid provider configuration.
|
||||
# @POST: New provider created and persisted to database.
|
||||
def create_provider(self, config: LLMProviderConfig) -> LLMProvider:
|
||||
with belief_scope("create_provider"):
|
||||
encrypted_key = self.encryption.encrypt(config.api_key)
|
||||
@@ -70,7 +101,10 @@ class LLMProviderService:
|
||||
# [/DEF:create_provider:Function]
|
||||
|
||||
# [DEF:update_provider:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Updates an existing LLM provider.
|
||||
# @PRE: provider_id must exist, config must be valid.
|
||||
# @POST: Provider updated and persisted to database.
|
||||
def update_provider(self, provider_id: str, config: LLMProviderConfig) -> Optional[LLMProvider]:
|
||||
with belief_scope("update_provider"):
|
||||
db_provider = self.get_provider(provider_id)
|
||||
@@ -92,7 +126,10 @@ class LLMProviderService:
|
||||
# [/DEF:update_provider:Function]
|
||||
|
||||
# [DEF:delete_provider:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Deletes an LLM provider.
|
||||
# @PRE: provider_id must exist.
|
||||
# @POST: Provider removed from database.
|
||||
def delete_provider(self, provider_id: str) -> bool:
|
||||
with belief_scope("delete_provider"):
|
||||
db_provider = self.get_provider(provider_id)
|
||||
@@ -104,7 +141,10 @@ class LLMProviderService:
|
||||
# [/DEF:delete_provider:Function]
|
||||
|
||||
# [DEF:get_decrypted_api_key:Function]
|
||||
# @TIER: STANDARD
|
||||
# @PURPOSE: Returns the decrypted API key for a provider.
|
||||
# @PRE: provider_id must exist with valid encrypted key.
|
||||
# @POST: Returns decrypted API key or None on failure.
|
||||
def get_decrypted_api_key(self, provider_id: str) -> Optional[str]:
|
||||
with belief_scope("get_decrypted_api_key"):
|
||||
db_provider = self.get_provider(provider_id)
|
||||
|
||||
BIN
backend/tasks.db
BIN
backend/tasks.db
Binary file not shown.
397
backend/tests/test_log_persistence.py
Normal file
397
backend/tests/test_log_persistence.py
Normal file
@@ -0,0 +1,397 @@
|
||||
# [DEF:test_log_persistence:Module]
|
||||
# @SEMANTICS: test, log, persistence, unit_test
|
||||
# @PURPOSE: Unit tests for TaskLogPersistenceService.
|
||||
# @LAYER: Test
|
||||
# @RELATION: TESTS -> TaskLogPersistenceService
|
||||
# @TIER: STANDARD
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import pytest
|
||||
from datetime import datetime
|
||||
from unittest.mock import Mock, patch, MagicMock
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy.orm import sessionmaker
|
||||
|
||||
from src.core.task_manager.persistence import TaskLogPersistenceService
|
||||
from src.core.task_manager.models import LogEntry
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TestLogPersistence:Class]
|
||||
# @PURPOSE: Test suite for TaskLogPersistenceService.
|
||||
# @TIER: STANDARD
|
||||
class TestLogPersistence:
|
||||
|
||||
# [DEF:setup_class:Function]
|
||||
# @PURPOSE: Setup test database and service instance.
|
||||
# @PRE: None.
|
||||
# @POST: In-memory database and service instance created.
|
||||
@classmethod
|
||||
def setup_class(cls):
|
||||
"""Create an in-memory database for testing."""
|
||||
cls.engine = create_engine("sqlite:///:memory:")
|
||||
cls.SessionLocal = sessionmaker(bind=cls.engine)
|
||||
cls.service = TaskLogPersistenceService(cls.engine)
|
||||
# [/DEF:setup_class:Function]
|
||||
|
||||
# [DEF:teardown_class:Function]
|
||||
# @PURPOSE: Clean up test database.
|
||||
# @PRE: None.
|
||||
# @POST: Database disposed.
|
||||
@classmethod
|
||||
def teardown_class(cls):
|
||||
"""Dispose of the database engine."""
|
||||
cls.engine.dispose()
|
||||
# [/DEF:teardown_class:Function]
|
||||
|
||||
# [DEF:setup_method:Function]
|
||||
# @PURPOSE: Setup for each test method.
|
||||
# @PRE: None.
|
||||
# @POST: Fresh database session created.
|
||||
def setup_method(self):
|
||||
"""Create a new session for each test."""
|
||||
self.session = self.SessionLocal()
|
||||
# [/DEF:setup_method:Function]
|
||||
|
||||
# [DEF:teardown_method:Function]
|
||||
# @PURPOSE: Cleanup after each test method.
|
||||
# @PRE: None.
|
||||
# @POST: Session closed and rolled back.
|
||||
def teardown_method(self):
|
||||
"""Close the session after each test."""
|
||||
self.session.close()
|
||||
# [/DEF:teardown_method:Function]
|
||||
|
||||
# [DEF:test_add_log_single:Function]
|
||||
# @PURPOSE: Test adding a single log entry.
|
||||
# @PRE: Service and session initialized.
|
||||
# @POST: Log entry persisted to database.
|
||||
def test_add_log_single(self):
|
||||
"""Test adding a single log entry."""
|
||||
entry = LogEntry(
|
||||
task_id="test-task-1",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="test_source",
|
||||
message="Test message"
|
||||
)
|
||||
|
||||
self.service.add_log(entry)
|
||||
|
||||
# Query the database
|
||||
result = self.session.query(LogEntry).filter_by(task_id="test-task-1").first()
|
||||
|
||||
assert result is not None
|
||||
assert result.level == "INFO"
|
||||
assert result.source == "test_source"
|
||||
assert result.message == "Test message"
|
||||
# [/DEF:test_add_log_single:Function]
|
||||
|
||||
# [DEF:test_add_log_batch:Function]
|
||||
# @PURPOSE: Test adding multiple log entries in batch.
|
||||
# @PRE: Service and session initialized.
|
||||
# @POST: All log entries persisted to database.
|
||||
def test_add_log_batch(self):
|
||||
"""Test adding multiple log entries in batch."""
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-2",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="source1",
|
||||
message="Message 1"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-2",
|
||||
timestamp=datetime.now(),
|
||||
level="WARNING",
|
||||
source="source2",
|
||||
message="Message 2"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-2",
|
||||
timestamp=datetime.now(),
|
||||
level="ERROR",
|
||||
source="source3",
|
||||
message="Message 3"
|
||||
),
|
||||
]
|
||||
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Query the database
|
||||
results = self.session.query(LogEntry).filter_by(task_id="test-task-2").all()
|
||||
|
||||
assert len(results) == 3
|
||||
assert results[0].level == "INFO"
|
||||
assert results[1].level == "WARNING"
|
||||
assert results[2].level == "ERROR"
|
||||
# [/DEF:test_add_log_batch:Function]
|
||||
|
||||
# [DEF:test_get_logs_by_task_id:Function]
|
||||
# @PURPOSE: Test retrieving logs by task ID.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns logs for the specified task.
|
||||
def test_get_logs_by_task_id(self):
|
||||
"""Test retrieving logs by task ID."""
|
||||
# Add test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-3",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="source1",
|
||||
message=f"Message {i}"
|
||||
)
|
||||
for i in range(5)
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Retrieve logs
|
||||
logs = self.service.get_logs("test-task-3")
|
||||
|
||||
assert len(logs) == 5
|
||||
assert all(log.task_id == "test-task-3" for log in logs)
|
||||
# [/DEF:test_get_logs_by_task_id:Function]
|
||||
|
||||
# [DEF:test_get_logs_with_filters:Function]
|
||||
# @PURPOSE: Test retrieving logs with level and source filters.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns filtered logs.
|
||||
def test_get_logs_with_filters(self):
|
||||
"""Test retrieving logs with level and source filters."""
|
||||
# Add test logs with different levels and sources
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-4",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="api",
|
||||
message="Info message"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-4",
|
||||
timestamp=datetime.now(),
|
||||
level="WARNING",
|
||||
source="api",
|
||||
message="Warning message"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-4",
|
||||
timestamp=datetime.now(),
|
||||
level="ERROR",
|
||||
source="storage",
|
||||
message="Error message"
|
||||
),
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Test level filter
|
||||
warning_logs = self.service.get_logs("test-task-4", level="WARNING")
|
||||
assert len(warning_logs) == 1
|
||||
assert warning_logs[0].level == "WARNING"
|
||||
|
||||
# Test source filter
|
||||
api_logs = self.service.get_logs("test-task-4", source="api")
|
||||
assert len(api_logs) == 2
|
||||
assert all(log.source == "api" for log in api_logs)
|
||||
|
||||
# Test combined filters
|
||||
api_warning_logs = self.service.get_logs("test-task-4", level="WARNING", source="api")
|
||||
assert len(api_warning_logs) == 1
|
||||
# [/DEF:test_get_logs_with_filters:Function]
|
||||
|
||||
# [DEF:test_get_logs_with_pagination:Function]
|
||||
# @PURPOSE: Test retrieving logs with pagination.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns paginated logs.
|
||||
def test_get_logs_with_pagination(self):
|
||||
"""Test retrieving logs with pagination."""
|
||||
# Add 15 test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-5",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="test",
|
||||
message=f"Message {i}"
|
||||
)
|
||||
for i in range(15)
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Test first page
|
||||
page1 = self.service.get_logs("test-task-5", limit=10, offset=0)
|
||||
assert len(page1) == 10
|
||||
|
||||
# Test second page
|
||||
page2 = self.service.get_logs("test-task-5", limit=10, offset=10)
|
||||
assert len(page2) == 5
|
||||
# [/DEF:test_get_logs_with_pagination:Function]
|
||||
|
||||
# [DEF:test_get_logs_with_search:Function]
|
||||
# @PURPOSE: Test retrieving logs with search query.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns logs matching search query.
|
||||
def test_get_logs_with_search(self):
|
||||
"""Test retrieving logs with search query."""
|
||||
# Add test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-6",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="api",
|
||||
message="User authentication successful"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-6",
|
||||
timestamp=datetime.now(),
|
||||
level="ERROR",
|
||||
source="api",
|
||||
message="Failed to connect to database"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-6",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="storage",
|
||||
message="File saved successfully"
|
||||
),
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Test search for "authentication"
|
||||
auth_logs = self.service.get_logs("test-task-6", search="authentication")
|
||||
assert len(auth_logs) == 1
|
||||
assert "authentication" in auth_logs[0].message.lower()
|
||||
|
||||
# Test search for "failed"
|
||||
failed_logs = self.service.get_logs("test-task-6", search="failed")
|
||||
assert len(failed_logs) == 1
|
||||
assert "failed" in failed_logs[0].message.lower()
|
||||
# [/DEF:test_get_logs_with_search:Function]
|
||||
|
||||
# [DEF:test_get_log_stats:Function]
|
||||
# @PURPOSE: Test retrieving log statistics.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns statistics grouped by level and source.
|
||||
def test_get_log_stats(self):
|
||||
"""Test retrieving log statistics."""
|
||||
# Add test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-7",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="api",
|
||||
message="Info 1"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-7",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="api",
|
||||
message="Info 2"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-7",
|
||||
timestamp=datetime.now(),
|
||||
level="WARNING",
|
||||
source="api",
|
||||
message="Warning 1"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-7",
|
||||
timestamp=datetime.now(),
|
||||
level="ERROR",
|
||||
source="storage",
|
||||
message="Error 1"
|
||||
),
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Get stats
|
||||
stats = self.service.get_log_stats("test-task-7")
|
||||
|
||||
assert stats is not None
|
||||
assert stats["by_level"]["INFO"] == 2
|
||||
assert stats["by_level"]["WARNING"] == 1
|
||||
assert stats["by_level"]["ERROR"] == 1
|
||||
assert stats["by_source"]["api"] == 3
|
||||
assert stats["by_source"]["storage"] == 1
|
||||
# [/DEF:test_get_log_stats:Function]
|
||||
|
||||
# [DEF:test_get_log_sources:Function]
|
||||
# @PURPOSE: Test retrieving unique log sources.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Returns list of unique sources.
|
||||
def test_get_log_sources(self):
|
||||
"""Test retrieving unique log sources."""
|
||||
# Add test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-8",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="api",
|
||||
message="Message 1"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-8",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="storage",
|
||||
message="Message 2"
|
||||
),
|
||||
LogEntry(
|
||||
task_id="test-task-8",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="git",
|
||||
message="Message 3"
|
||||
),
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Get sources
|
||||
sources = self.service.get_log_sources("test-task-8")
|
||||
|
||||
assert len(sources) == 3
|
||||
assert "api" in sources
|
||||
assert "storage" in sources
|
||||
assert "git" in sources
|
||||
# [/DEF:test_get_log_sources:Function]
|
||||
|
||||
# [DEF:test_delete_logs_by_task_id:Function]
|
||||
# @PURPOSE: Test deleting logs by task ID.
|
||||
# @PRE: Service and session initialized, logs exist.
|
||||
# @POST: Logs for the task are deleted.
|
||||
def test_delete_logs_by_task_id(self):
|
||||
"""Test deleting logs by task ID."""
|
||||
# Add test logs
|
||||
entries = [
|
||||
LogEntry(
|
||||
task_id="test-task-9",
|
||||
timestamp=datetime.now(),
|
||||
level="INFO",
|
||||
source="test",
|
||||
message=f"Message {i}"
|
||||
)
|
||||
for i in range(3)
|
||||
]
|
||||
self.service.add_logs(entries)
|
||||
|
||||
# Verify logs exist
|
||||
logs_before = self.service.get_logs("test-task-9")
|
||||
assert len(logs_before) == 3
|
||||
|
||||
# Delete logs
|
||||
self.service.delete_logs("test-task-9")
|
||||
|
||||
# Verify logs are deleted
|
||||
logs_after = self.service.get_logs("test-task-9")
|
||||
assert len(logs_after) == 0
|
||||
# [/DEF:test_delete_logs_by_task_id:Function]
|
||||
|
||||
# [/DEF:TestLogPersistence:Class]
|
||||
# [/DEF:test_log_persistence:Module]
|
||||
@@ -1,14 +1,30 @@
|
||||
import pytest
|
||||
from src.core.logger import belief_scope, logger
|
||||
import logging
|
||||
from src.core.logger import (
|
||||
belief_scope,
|
||||
logger,
|
||||
configure_logger,
|
||||
get_task_log_level,
|
||||
should_log_task_level
|
||||
)
|
||||
from src.core.config_models import LoggingConfig
|
||||
|
||||
|
||||
# [DEF:test_belief_scope_logs_entry_action_exit:Function]
|
||||
# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs.
|
||||
# @PRE: belief_scope is available. caplog fixture is used.
|
||||
# @POST: Logs are verified to contain Entry, Action, and Exit tags.
|
||||
def test_belief_scope_logs_entry_action_exit(caplog):
|
||||
"""Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs."""
|
||||
caplog.set_level("INFO")
|
||||
# [DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]
|
||||
# @PURPOSE: Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.
|
||||
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
|
||||
# @POST: Logs are verified to contain Entry, Action, and Exit tags at DEBUG level.
|
||||
def test_belief_scope_logs_entry_action_exit_at_debug(caplog):
|
||||
"""Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level."""
|
||||
# Configure logger to DEBUG level
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
|
||||
caplog.set_level("DEBUG")
|
||||
|
||||
with belief_scope("TestFunction"):
|
||||
logger.info("Doing something important")
|
||||
@@ -19,16 +35,28 @@ def test_belief_scope_logs_entry_action_exit(caplog):
|
||||
assert any("[TestFunction][Entry]" in msg for msg in log_messages), "Entry log not found"
|
||||
assert any("[TestFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found"
|
||||
assert any("[TestFunction][Exit]" in msg for msg in log_messages), "Exit log not found"
|
||||
# [/DEF:test_belief_scope_logs_entry_action_exit:Function]
|
||||
|
||||
# Reset to INFO
|
||||
config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
|
||||
configure_logger(config)
|
||||
# [/DEF:test_belief_scope_logs_entry_action_exit_at_debug:Function]
|
||||
|
||||
|
||||
# [DEF:test_belief_scope_error_handling:Function]
|
||||
# @PURPOSE: Test that belief_scope logs Coherence:Failed on exception.
|
||||
# @PRE: belief_scope is available. caplog fixture is used.
|
||||
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
|
||||
# @POST: Logs are verified to contain Coherence:Failed tag.
|
||||
def test_belief_scope_error_handling(caplog):
|
||||
"""Test that belief_scope logs Coherence:Failed on exception."""
|
||||
caplog.set_level("INFO")
|
||||
# Configure logger to DEBUG level
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
|
||||
caplog.set_level("DEBUG")
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
with belief_scope("FailingFunction"):
|
||||
@@ -39,16 +67,28 @@ def test_belief_scope_error_handling(caplog):
|
||||
assert any("[FailingFunction][Entry]" in msg for msg in log_messages), "Entry log not found"
|
||||
assert any("[FailingFunction][Coherence:Failed]" in msg for msg in log_messages), "Failed coherence log not found"
|
||||
# Exit should not be logged on failure
|
||||
|
||||
# Reset to INFO
|
||||
config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
|
||||
configure_logger(config)
|
||||
# [/DEF:test_belief_scope_error_handling:Function]
|
||||
|
||||
|
||||
# [DEF:test_belief_scope_success_coherence:Function]
|
||||
# @PURPOSE: Test that belief_scope logs Coherence:OK on success.
|
||||
# @PRE: belief_scope is available. caplog fixture is used.
|
||||
# @PRE: belief_scope is available. caplog fixture is used. Logger configured to DEBUG.
|
||||
# @POST: Logs are verified to contain Coherence:OK tag.
|
||||
def test_belief_scope_success_coherence(caplog):
|
||||
"""Test that belief_scope logs Coherence:OK on success."""
|
||||
caplog.set_level("INFO")
|
||||
# Configure logger to DEBUG level
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
|
||||
caplog.set_level("DEBUG")
|
||||
|
||||
with belief_scope("SuccessFunction"):
|
||||
pass
|
||||
@@ -56,4 +96,119 @@ def test_belief_scope_success_coherence(caplog):
|
||||
log_messages = [record.message for record in caplog.records]
|
||||
|
||||
assert any("[SuccessFunction][Coherence:OK]" in msg for msg in log_messages), "Success coherence log not found"
|
||||
# [/DEF:test_belief_scope_success_coherence:Function]
|
||||
|
||||
# Reset to INFO
|
||||
config = LoggingConfig(level="INFO", task_log_level="INFO", enable_belief_state=True)
|
||||
configure_logger(config)
|
||||
# [/DEF:test_belief_scope_success_coherence:Function]
|
||||
|
||||
|
||||
# [DEF:test_belief_scope_not_visible_at_info:Function]
|
||||
# @PURPOSE: Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.
|
||||
# @PRE: belief_scope is available. caplog fixture is used.
|
||||
# @POST: Entry/Exit/Coherence logs are not captured at INFO level.
|
||||
def test_belief_scope_not_visible_at_info(caplog):
|
||||
"""Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level."""
|
||||
caplog.set_level("INFO")
|
||||
|
||||
with belief_scope("InfoLevelFunction"):
|
||||
logger.info("Doing something important")
|
||||
|
||||
log_messages = [record.message for record in caplog.records]
|
||||
|
||||
# Action log should be visible
|
||||
assert any("[InfoLevelFunction][Action] Doing something important" in msg for msg in log_messages), "Action log not found"
|
||||
# Entry/Exit/Coherence should NOT be visible at INFO level
|
||||
assert not any("[InfoLevelFunction][Entry]" in msg for msg in log_messages), "Entry log should not be visible at INFO"
|
||||
assert not any("[InfoLevelFunction][Exit]" in msg for msg in log_messages), "Exit log should not be visible at INFO"
|
||||
assert not any("[InfoLevelFunction][Coherence:OK]" in msg for msg in log_messages), "Coherence log should not be visible at INFO"
|
||||
# [/DEF:test_belief_scope_not_visible_at_info:Function]
|
||||
|
||||
|
||||
# [DEF:test_task_log_level_default:Function]
|
||||
# @PURPOSE: Test that default task log level is INFO.
|
||||
# @PRE: None.
|
||||
# @POST: Default level is INFO.
|
||||
def test_task_log_level_default():
|
||||
"""Test that default task log level is INFO."""
|
||||
level = get_task_log_level()
|
||||
assert level == "INFO"
|
||||
# [/DEF:test_task_log_level_default:Function]
|
||||
|
||||
|
||||
# [DEF:test_should_log_task_level:Function]
|
||||
# @PURPOSE: Test that should_log_task_level correctly filters log levels.
|
||||
# @PRE: None.
|
||||
# @POST: Filtering works correctly for all level combinations.
|
||||
def test_should_log_task_level():
|
||||
"""Test that should_log_task_level correctly filters log levels."""
|
||||
# Default level is INFO
|
||||
assert should_log_task_level("ERROR") is True, "ERROR should be logged at INFO threshold"
|
||||
assert should_log_task_level("WARNING") is True, "WARNING should be logged at INFO threshold"
|
||||
assert should_log_task_level("INFO") is True, "INFO should be logged at INFO threshold"
|
||||
assert should_log_task_level("DEBUG") is False, "DEBUG should NOT be logged at INFO threshold"
|
||||
# [/DEF:test_should_log_task_level:Function]
|
||||
|
||||
|
||||
# [DEF:test_configure_logger_task_log_level:Function]
|
||||
# @PURPOSE: Test that configure_logger updates task_log_level.
|
||||
# @PRE: LoggingConfig is available.
|
||||
# @POST: task_log_level is updated correctly.
|
||||
def test_configure_logger_task_log_level():
|
||||
"""Test that configure_logger updates task_log_level."""
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
|
||||
assert get_task_log_level() == "DEBUG", "task_log_level should be DEBUG"
|
||||
assert should_log_task_level("DEBUG") is True, "DEBUG should be logged at DEBUG threshold"
|
||||
|
||||
# Reset to INFO
|
||||
config = LoggingConfig(
|
||||
level="INFO",
|
||||
task_log_level="INFO",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
assert get_task_log_level() == "INFO", "task_log_level should be reset to INFO"
|
||||
# [/DEF:test_configure_logger_task_log_level:Function]
|
||||
|
||||
|
||||
# [DEF:test_enable_belief_state_flag:Function]
|
||||
# @PURPOSE: Test that enable_belief_state flag controls belief_scope logging.
|
||||
# @PRE: LoggingConfig is available. caplog fixture is used.
|
||||
# @POST: belief_scope logs are controlled by the flag.
|
||||
def test_enable_belief_state_flag(caplog):
|
||||
"""Test that enable_belief_state flag controls belief_scope logging."""
|
||||
# Disable belief state
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=False
|
||||
)
|
||||
configure_logger(config)
|
||||
|
||||
caplog.set_level("DEBUG")
|
||||
|
||||
with belief_scope("DisabledFunction"):
|
||||
logger.info("Doing something")
|
||||
|
||||
log_messages = [record.message for record in caplog.records]
|
||||
|
||||
# Entry and Exit should NOT be logged when disabled
|
||||
assert not any("[DisabledFunction][Entry]" in msg for msg in log_messages), "Entry should not be logged when disabled"
|
||||
assert not any("[DisabledFunction][Exit]" in msg for msg in log_messages), "Exit should not be logged when disabled"
|
||||
# Coherence:OK should still be logged (internal tracking)
|
||||
assert any("[DisabledFunction][Coherence:OK]" in msg for msg in log_messages), "Coherence should still be logged"
|
||||
|
||||
# Re-enable for other tests
|
||||
config = LoggingConfig(
|
||||
level="DEBUG",
|
||||
task_log_level="DEBUG",
|
||||
enable_belief_state=True
|
||||
)
|
||||
configure_logger(config)
|
||||
# [/DEF:test_enable_belief_state_flag:Function]
|
||||
377
backend/tests/test_task_logger.py
Normal file
377
backend/tests/test_task_logger.py
Normal file
@@ -0,0 +1,377 @@
|
||||
# [DEF:test_task_logger:Module]
|
||||
# @SEMANTICS: test, task_logger, task_context, unit_test
|
||||
# @PURPOSE: Unit tests for TaskLogger and TaskContext.
|
||||
# @LAYER: Test
|
||||
# @RELATION: TESTS -> TaskLogger, TaskContext
|
||||
# @TIER: STANDARD
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import pytest
|
||||
from unittest.mock import Mock, MagicMock
|
||||
from datetime import datetime
|
||||
|
||||
from src.core.task_manager.task_logger import TaskLogger
|
||||
from src.core.task_manager.context import TaskContext
|
||||
from src.core.task_manager.models import LogEntry
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TestTaskLogger:Class]
|
||||
# @PURPOSE: Test suite for TaskLogger.
|
||||
# @TIER: STANDARD
|
||||
class TestTaskLogger:
|
||||
|
||||
# [DEF:setup_method:Function]
|
||||
# @PURPOSE: Setup for each test method.
|
||||
# @PRE: None.
|
||||
# @POST: Mock add_log_fn created.
|
||||
def setup_method(self):
|
||||
"""Create a mock add_log function for testing."""
|
||||
self.mock_add_log = Mock()
|
||||
self.logger = TaskLogger(
|
||||
task_id="test-task-1",
|
||||
add_log_fn=self.mock_add_log,
|
||||
source="test_source"
|
||||
)
|
||||
# [/DEF:setup_method:Function]
|
||||
|
||||
# [DEF:test_init:Function]
|
||||
# @PURPOSE: Test TaskLogger initialization.
|
||||
# @PRE: None.
|
||||
# @POST: Logger instance created with correct attributes.
|
||||
def test_init(self):
|
||||
"""Test TaskLogger initialization."""
|
||||
assert self.logger._task_id == "test-task-1"
|
||||
assert self.logger._default_source == "test_source"
|
||||
assert self.logger._add_log == self.mock_add_log
|
||||
# [/DEF:test_init:Function]
|
||||
|
||||
# [DEF:test_with_source:Function]
|
||||
# @PURPOSE: Test creating a sub-logger with different source.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: New logger created with different source but same task_id.
|
||||
def test_with_source(self):
|
||||
"""Test creating a sub-logger with different source."""
|
||||
sub_logger = self.logger.with_source("new_source")
|
||||
|
||||
assert sub_logger._task_id == "test-task-1"
|
||||
assert sub_logger._default_source == "new_source"
|
||||
assert sub_logger._add_log == self.mock_add_log
|
||||
# [/DEF:test_with_source:Function]
|
||||
|
||||
# [DEF:test_debug:Function]
|
||||
# @PURPOSE: Test debug log level.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with DEBUG level.
|
||||
def test_debug(self):
|
||||
"""Test debug logging."""
|
||||
self.logger.debug("Debug message")
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="DEBUG",
|
||||
message="Debug message",
|
||||
source="test_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_debug:Function]
|
||||
|
||||
# [DEF:test_info:Function]
|
||||
# @PURPOSE: Test info log level.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with INFO level.
|
||||
def test_info(self):
|
||||
"""Test info logging."""
|
||||
self.logger.info("Info message")
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="INFO",
|
||||
message="Info message",
|
||||
source="test_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_info:Function]
|
||||
|
||||
# [DEF:test_warning:Function]
|
||||
# @PURPOSE: Test warning log level.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with WARNING level.
|
||||
def test_warning(self):
|
||||
"""Test warning logging."""
|
||||
self.logger.warning("Warning message")
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="WARNING",
|
||||
message="Warning message",
|
||||
source="test_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_warning:Function]
|
||||
|
||||
# [DEF:test_error:Function]
|
||||
# @PURPOSE: Test error log level.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with ERROR level.
|
||||
def test_error(self):
|
||||
"""Test error logging."""
|
||||
self.logger.error("Error message")
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="ERROR",
|
||||
message="Error message",
|
||||
source="test_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_error:Function]
|
||||
|
||||
# [DEF:test_error_with_metadata:Function]
|
||||
# @PURPOSE: Test error logging with metadata.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with ERROR level and metadata.
|
||||
def test_error_with_metadata(self):
|
||||
"""Test error logging with metadata."""
|
||||
metadata = {"error_code": 500, "details": "Connection failed"}
|
||||
self.logger.error("Error message", metadata=metadata)
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="ERROR",
|
||||
message="Error message",
|
||||
source="test_source",
|
||||
metadata=metadata
|
||||
)
|
||||
# [/DEF:test_error_with_metadata:Function]
|
||||
|
||||
# [DEF:test_progress:Function]
|
||||
# @PURPOSE: Test progress logging.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with INFO level and progress metadata.
|
||||
def test_progress(self):
|
||||
"""Test progress logging."""
|
||||
self.logger.progress("Processing items", percent=50)
|
||||
|
||||
expected_metadata = {"progress": 50}
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="INFO",
|
||||
message="Processing items",
|
||||
source="test_source",
|
||||
metadata=expected_metadata
|
||||
)
|
||||
# [/DEF:test_progress:Function]
|
||||
|
||||
# [DEF:test_progress_clamping:Function]
|
||||
# @PURPOSE: Test progress value clamping (0-100).
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: Progress values clamped to 0-100 range.
|
||||
def test_progress_clamping(self):
|
||||
"""Test progress value clamping."""
|
||||
# Test below 0
|
||||
self.logger.progress("Below 0", percent=-10)
|
||||
call1 = self.mock_add_log.call_args_list[0]
|
||||
assert call1.kwargs["metadata"]["progress"] == 0
|
||||
|
||||
self.mock_add_log.reset_mock()
|
||||
|
||||
# Test above 100
|
||||
self.logger.progress("Above 100", percent=150)
|
||||
call2 = self.mock_add_log.call_args_list[0]
|
||||
assert call2.kwargs["metadata"]["progress"] == 100
|
||||
# [/DEF:test_progress_clamping:Function]
|
||||
|
||||
# [DEF:test_source_override:Function]
|
||||
# @PURPOSE: Test overriding the default source.
|
||||
# @PRE: Logger initialized.
|
||||
# @POST: add_log_fn called with overridden source.
|
||||
def test_source_override(self):
|
||||
"""Test overriding the default source."""
|
||||
self.logger.info("Message", source="override_source")
|
||||
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-1",
|
||||
level="INFO",
|
||||
message="Message",
|
||||
source="override_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_source_override:Function]
|
||||
|
||||
# [DEF:test_sub_logger_source_independence:Function]
|
||||
# @PURPOSE: Test sub-logger independence from parent.
|
||||
# @PRE: Logger and sub-logger initialized.
|
||||
# @POST: Sub-logger has different source, parent unchanged.
|
||||
def test_sub_logger_source_independence(self):
|
||||
"""Test sub-logger source independence from parent."""
|
||||
sub_logger = self.logger.with_source("sub_source")
|
||||
|
||||
# Log with parent
|
||||
self.logger.info("Parent message")
|
||||
|
||||
# Log with sub-logger
|
||||
sub_logger.info("Sub message")
|
||||
|
||||
# Verify both calls were made with correct sources
|
||||
calls = self.mock_add_log.call_args_list
|
||||
assert len(calls) == 2
|
||||
assert calls[0].kwargs["source"] == "test_source"
|
||||
assert calls[1].kwargs["source"] == "sub_source"
|
||||
# [/DEF:test_sub_logger_source_independence:Function]
|
||||
|
||||
# [/DEF:TestTaskLogger:Class]
|
||||
|
||||
# [DEF:TestTaskContext:Class]
|
||||
# @PURPOSE: Test suite for TaskContext.
|
||||
# @TIER: STANDARD
|
||||
class TestTaskContext:
|
||||
|
||||
# [DEF:setup_method:Function]
|
||||
# @PURPOSE: Setup for each test method.
|
||||
# @PRE: None.
|
||||
# @POST: Mock add_log_fn created.
|
||||
def setup_method(self):
|
||||
"""Create a mock add_log function for testing."""
|
||||
self.mock_add_log = Mock()
|
||||
self.params = {"param1": "value1", "param2": "value2"}
|
||||
self.context = TaskContext(
|
||||
task_id="test-task-2",
|
||||
add_log_fn=self.mock_add_log,
|
||||
params=self.params,
|
||||
default_source="plugin"
|
||||
)
|
||||
# [/DEF:setup_method:Function]
|
||||
|
||||
# [DEF:test_init:Function]
|
||||
# @PURPOSE: Test TaskContext initialization.
|
||||
# @PRE: None.
|
||||
# @POST: Context instance created with correct attributes.
|
||||
def test_init(self):
|
||||
"""Test TaskContext initialization."""
|
||||
assert self.context._task_id == "test-task-2"
|
||||
assert self.context._params == self.params
|
||||
assert isinstance(self.context._logger, TaskLogger)
|
||||
assert self.context._logger._default_source == "plugin"
|
||||
# [/DEF:test_init:Function]
|
||||
|
||||
# [DEF:test_task_id_property:Function]
|
||||
# @PURPOSE: Test task_id property.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Returns correct task_id.
|
||||
def test_task_id_property(self):
|
||||
"""Test task_id property."""
|
||||
assert self.context.task_id == "test-task-2"
|
||||
# [/DEF:test_task_id_property:Function]
|
||||
|
||||
# [DEF:test_logger_property:Function]
|
||||
# @PURPOSE: Test logger property.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Returns TaskLogger instance.
|
||||
def test_logger_property(self):
|
||||
"""Test logger property."""
|
||||
logger = self.context.logger
|
||||
assert isinstance(logger, TaskLogger)
|
||||
assert logger._task_id == "test-task-2"
|
||||
assert logger._default_source == "plugin"
|
||||
# [/DEF:test_logger_property:Function]
|
||||
|
||||
# [DEF:test_params_property:Function]
|
||||
# @PURPOSE: Test params property.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Returns correct params dict.
|
||||
def test_params_property(self):
|
||||
"""Test params property."""
|
||||
assert self.context.params == self.params
|
||||
# [/DEF:test_params_property:Function]
|
||||
|
||||
# [DEF:test_get_param:Function]
|
||||
# @PURPOSE: Test getting a specific parameter.
|
||||
# @PRE: Context initialized with params.
|
||||
# @POST: Returns parameter value or default.
|
||||
def test_get_param(self):
|
||||
"""Test getting a specific parameter."""
|
||||
assert self.context.get_param("param1") == "value1"
|
||||
assert self.context.get_param("param2") == "value2"
|
||||
assert self.context.get_param("nonexistent") is None
|
||||
assert self.context.get_param("nonexistent", "default") == "default"
|
||||
# [/DEF:test_get_param:Function]
|
||||
|
||||
# [DEF:test_create_sub_context:Function]
|
||||
# @PURPOSE: Test creating a sub-context with different source.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: New context created with different logger source.
|
||||
def test_create_sub_context(self):
|
||||
"""Test creating a sub-context with different source."""
|
||||
sub_context = self.context.create_sub_context("new_source")
|
||||
|
||||
assert sub_context._task_id == "test-task-2"
|
||||
assert sub_context._params == self.params
|
||||
assert sub_context._logger._default_source == "new_source"
|
||||
assert sub_context._logger._task_id == "test-task-2"
|
||||
# [/DEF:test_create_sub_context:Function]
|
||||
|
||||
# [DEF:test_context_logger_delegates_to_task_logger:Function]
|
||||
# @PURPOSE: Test context logger delegates to TaskLogger.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Logger calls are delegated to TaskLogger.
|
||||
def test_context_logger_delegates_to_task_logger(self):
|
||||
"""Test context logger delegates to TaskLogger."""
|
||||
# Call through context
|
||||
self.context.logger.info("Test message")
|
||||
|
||||
# Verify the mock was called
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-2",
|
||||
level="INFO",
|
||||
message="Test message",
|
||||
source="plugin",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_context_logger_delegates_to_task_logger:Function]
|
||||
|
||||
# [DEF:test_sub_context_with_source:Function]
|
||||
# @PURPOSE: Test sub-context logger uses new source.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Sub-context logger uses new source.
|
||||
def test_sub_context_with_source(self):
|
||||
"""Test sub-context logger uses new source."""
|
||||
sub_context = self.context.create_sub_context("api_source")
|
||||
|
||||
# Log through sub-context
|
||||
sub_context.logger.info("API message")
|
||||
|
||||
# Verify the mock was called with new source
|
||||
self.mock_add_log.assert_called_once_with(
|
||||
task_id="test-task-2",
|
||||
level="INFO",
|
||||
message="API message",
|
||||
source="api_source",
|
||||
metadata=None
|
||||
)
|
||||
# [/DEF:test_sub_context_with_source:Function]
|
||||
|
||||
# [DEF:test_multiple_sub_contexts:Function]
|
||||
# @PURPOSE: Test creating multiple sub-contexts.
|
||||
# @PRE: Context initialized.
|
||||
# @POST: Each sub-context has independent logger source.
|
||||
def test_multiple_sub_contexts(self):
|
||||
"""Test creating multiple sub-contexts."""
|
||||
sub1 = self.context.create_sub_context("source1")
|
||||
sub2 = self.context.create_sub_context("source2")
|
||||
sub3 = self.context.create_sub_context("source3")
|
||||
|
||||
assert sub1._logger._default_source == "source1"
|
||||
assert sub2._logger._default_source == "source2"
|
||||
assert sub3._logger._default_source == "source3"
|
||||
|
||||
# All should have same task_id and params
|
||||
assert sub1._task_id == "test-task-2"
|
||||
assert sub2._task_id == "test-task-2"
|
||||
assert sub3._task_id == "test-task-2"
|
||||
assert sub1._params == self.params
|
||||
assert sub2._params == self.params
|
||||
assert sub3._params == self.params
|
||||
# [/DEF:test_multiple_sub_contexts:Function]
|
||||
|
||||
# [/DEF:TestTaskContext:Class]
|
||||
# [/DEF:test_task_logger:Module]
|
||||
@@ -64,24 +64,156 @@ class HelloWorldPlugin(PluginBase):
|
||||
"required": ["name"],
|
||||
}
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
name = params["name"]
|
||||
print(f"Hello, {name}!")
|
||||
if context:
|
||||
context.logger.info(f"Hello, {name}!")
|
||||
else:
|
||||
print(f"Hello, {name}!")
|
||||
```
|
||||
|
||||
## 4. Logging
|
||||
## 4. Logging with TaskContext
|
||||
|
||||
You can use the global logger instance to log messages from your plugin. The logger is available in the `superset_tool.utils.logger` module.
|
||||
Plugins now support TaskContext for structured logging with source attribution. The `context` parameter provides access to a logger that automatically tags logs with the task ID and a source identifier.
|
||||
|
||||
### 4.1. Basic Logging
|
||||
|
||||
Use `context.logger` to log messages with automatic source attribution:
|
||||
|
||||
```python
|
||||
from superset_tool.utils.logger import SupersetLogger
|
||||
from typing import Dict, Any, Optional
|
||||
from ..core.plugin_base import PluginBase
|
||||
from ..core.task_manager.context import TaskContext
|
||||
|
||||
logger = SupersetLogger()
|
||||
|
||||
async def execute(self, params: Dict[str, Any]):
|
||||
logger.info("My plugin is running!")
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
if context:
|
||||
# Use TaskContext logger for structured logging
|
||||
context.logger.info("My plugin is running!")
|
||||
else:
|
||||
# Fallback to global logger for backward compatibility
|
||||
from ..core.logger import logger
|
||||
logger.info("My plugin is running!")
|
||||
```
|
||||
|
||||
### 4.2. Source Attribution
|
||||
|
||||
For better log organization, create sub-loggers for different components:
|
||||
|
||||
```python
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
if context:
|
||||
# Create sub-loggers for different components
|
||||
api_log = context.logger.with_source("api")
|
||||
storage_log = context.logger.with_source("storage")
|
||||
|
||||
api_log.info("Connecting to API...")
|
||||
storage_log.info("Saving file...")
|
||||
else:
|
||||
# Fallback to global logger
|
||||
from ..core.logger import logger
|
||||
logger.info("My plugin is running!")
|
||||
```
|
||||
|
||||
### 4.3. Log Levels
|
||||
|
||||
The logger supports standard log levels. Use them appropriately:
|
||||
|
||||
| Level | Usage |
|
||||
|-------|-------|
|
||||
| `DEBUG` | Detailed diagnostic information (API responses, internal state). Only visible when log level is set to DEBUG. |
|
||||
| `INFO` | General operational messages (start/complete notifications, progress updates). |
|
||||
| `WARNING` | Non-critical issues that don't stop execution (deprecated APIs, retry attempts). |
|
||||
| `ERROR` | Failures that prevent an operation from completing (API errors, validation failures). |
|
||||
|
||||
```python
|
||||
# Good: Use DEBUG for verbose diagnostic info
|
||||
api_log.debug(f"API response: {response.json()}")
|
||||
|
||||
# Good: Use INFO for operational milestones
|
||||
log.info(f"Starting backup for environment: {env}")
|
||||
|
||||
# Good: Use WARNING for recoverable issues
|
||||
log.warning(f"Rate limit hit, retrying in {delay}s")
|
||||
|
||||
# Good: Use ERROR for failures
|
||||
log.error(f"Failed to connect to database: {e}")
|
||||
```
|
||||
|
||||
### 4.4. Progress Logging
|
||||
|
||||
For operations that report progress, use the `progress` method:
|
||||
|
||||
```python
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
if context:
|
||||
total_items = 100
|
||||
for i, item in enumerate(items):
|
||||
# Report progress with percentage
|
||||
percent = (i + 1) / total_items * 100
|
||||
context.logger.progress(f"Processing {item}", percent=percent)
|
||||
else:
|
||||
# Fallback
|
||||
from ..core.logger import logger
|
||||
logger.info("My plugin is running!")
|
||||
```
|
||||
|
||||
### 4.5. Logging with Metadata
|
||||
|
||||
You can include structured metadata with log entries:
|
||||
|
||||
```python
|
||||
async def execute(self, params: Dict[str, Any], context: Optional[TaskContext] = None):
|
||||
if context:
|
||||
context.logger.error(
|
||||
"Operation failed",
|
||||
metadata={"error_code": 500, "details": "Connection timeout"}
|
||||
)
|
||||
else:
|
||||
from ..core.logger import logger
|
||||
logger.error("Operation failed")
|
||||
```
|
||||
|
||||
### 4.6. Common Source Names
|
||||
|
||||
For consistency across plugins, use these standard source names:
|
||||
|
||||
| Source | Usage |
|
||||
|--------|-------|
|
||||
| `superset_api` | Superset REST API calls |
|
||||
| `postgres` | PostgreSQL database operations |
|
||||
| `storage` | File system operations |
|
||||
| `git` | Git operations |
|
||||
| `llm` | LLM API calls |
|
||||
| `screenshot` | Screenshot capture operations |
|
||||
| `migration` | Migration-specific logic |
|
||||
| `backup` | Backup operations |
|
||||
| `debug` | Debug/diagnostic operations |
|
||||
| `search` | Search operations |
|
||||
|
||||
### 4.7. Best Practices
|
||||
|
||||
1. **Always check for context**: Support backward compatibility by checking if `context` is available:
|
||||
```python
|
||||
log = context.logger if context else logger
|
||||
```
|
||||
|
||||
2. **Use source attribution**: Create sub-loggers for different components to make filtering easier in the UI.
|
||||
|
||||
3. **Use appropriate log levels**:
|
||||
- `DEBUG`: Verbose diagnostic info (API responses, internal state)
|
||||
- `INFO`: Operational milestones (start, complete, progress)
|
||||
- `WARNING`: Recoverable issues (rate limits, deprecated APIs)
|
||||
- `ERROR`: Failures that stop an operation
|
||||
|
||||
4. **Log progress for long operations**: Use `progress()` for operations that take time:
|
||||
```python
|
||||
for i, item in enumerate(items):
|
||||
percent = (i + 1) / len(items) * 100
|
||||
log.progress(f"Processing {item}", percent=percent)
|
||||
```
|
||||
|
||||
5. **Keep DEBUG logs verbose, INFO logs concise**: DEBUG logs can include full API responses, while INFO logs should be one-line summaries.
|
||||
|
||||
## 5. Testing
|
||||
|
||||
To test your plugin, simply run the application and navigate to the web UI. Your plugin should appear in the list of available tools.
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:DashboardGrid:Component] -->
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: dashboard, grid, selection, pagination
|
||||
@PURPOSE: Displays a grid of dashboards with selection and pagination.
|
||||
@LAYER: Component
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:Footer:Component] -->
|
||||
<!--
|
||||
@TIER: TRIVIAL
|
||||
@SEMANTICS: footer, layout, copyright
|
||||
@PURPOSE: Displays the application footer with copyright information.
|
||||
@LAYER: UI
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:Navbar:Component] -->
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: navbar, navigation, header, layout
|
||||
@PURPOSE: Main navigation bar for the application.
|
||||
@LAYER: UI
|
||||
|
||||
@@ -1,15 +1,16 @@
|
||||
<!-- [DEF:TaskLogViewer:Component] -->
|
||||
<!--
|
||||
@SEMANTICS: task, log, viewer, modal, inline
|
||||
@PURPOSE: Displays detailed logs for a specific task in a modal or inline.
|
||||
@PURPOSE: Displays detailed logs for a specific task in a modal or inline using TaskLogPanel.
|
||||
@LAYER: UI
|
||||
@RELATION: USES -> frontend/src/services/taskService.js
|
||||
@RELATION: USES -> frontend/src/services/taskService.js, frontend/src/components/tasks/TaskLogPanel.svelte
|
||||
-->
|
||||
<script>
|
||||
import { createEventDispatcher, onMount, onDestroy } from 'svelte';
|
||||
import { getTaskLogs } from '../services/taskService.js';
|
||||
import { t } from '../lib/i18n';
|
||||
import { Button } from '../lib/ui';
|
||||
import TaskLogPanel from './tasks/TaskLogPanel.svelte';
|
||||
|
||||
export let show = false;
|
||||
export let inline = false;
|
||||
@@ -23,7 +24,8 @@
|
||||
let error = "";
|
||||
let interval;
|
||||
let autoScroll = true;
|
||||
let logContainer;
|
||||
let selectedSource = 'all';
|
||||
let selectedLevel = 'all';
|
||||
|
||||
$: shouldShow = inline || show;
|
||||
|
||||
@@ -36,12 +38,11 @@
|
||||
*/
|
||||
async function fetchLogs() {
|
||||
if (!taskId) return;
|
||||
console.log(`[fetchLogs][Action] Fetching logs for task context={{'taskId': '${taskId}'}}`);
|
||||
console.log(`[fetchLogs][Action] Fetching logs for task context={{'taskId': '${taskId}', 'source': '${selectedSource}', 'level': '${selectedLevel}'}}`);
|
||||
try {
|
||||
// Note: getTaskLogs currently doesn't support filters, but we can filter client-side for now
|
||||
// or update taskService later. For US1, the WebSocket handles real-time filtering.
|
||||
logs = await getTaskLogs(taskId);
|
||||
if (autoScroll) {
|
||||
scrollToBottom();
|
||||
}
|
||||
console.log(`[fetchLogs][Coherence:OK] Logs fetched context={{'count': ${logs.length}}}`);
|
||||
} catch (e) {
|
||||
error = e.message;
|
||||
@@ -52,35 +53,14 @@
|
||||
}
|
||||
// [/DEF:fetchLogs:Function]
|
||||
|
||||
// [DEF:scrollToBottom:Function]
|
||||
/**
|
||||
* @purpose Scrolls the log container to the bottom.
|
||||
* @pre logContainer element must be bound.
|
||||
* @post logContainer scrollTop is set to scrollHeight.
|
||||
*/
|
||||
function scrollToBottom() {
|
||||
if (logContainer) {
|
||||
setTimeout(() => {
|
||||
logContainer.scrollTop = logContainer.scrollHeight;
|
||||
}, 0);
|
||||
}
|
||||
function handleFilterChange(event) {
|
||||
const { source, level } = event.detail;
|
||||
selectedSource = source;
|
||||
selectedLevel = level;
|
||||
// Re-fetch or re-filter if needed.
|
||||
// For now, we just log it as the WebSocket will handle real-time updates with filters.
|
||||
console.log(`[TaskLogViewer] Filter changed: source=${source}, level=${level}`);
|
||||
}
|
||||
// [/DEF:scrollToBottom:Function]
|
||||
|
||||
// [DEF:handleScroll:Function]
|
||||
/**
|
||||
* @purpose Updates auto-scroll preference based on scroll position.
|
||||
* @pre logContainer scroll event fired.
|
||||
* @post autoScroll boolean is updated.
|
||||
*/
|
||||
function handleScroll() {
|
||||
if (!logContainer) return;
|
||||
// If user scrolls up, disable auto-scroll
|
||||
const { scrollTop, scrollHeight, clientHeight } = logContainer;
|
||||
const atBottom = scrollHeight - scrollTop - clientHeight < 50;
|
||||
autoScroll = atBottom;
|
||||
}
|
||||
// [/DEF:handleScroll:Function]
|
||||
|
||||
// [DEF:close:Function]
|
||||
/**
|
||||
@@ -94,23 +74,6 @@
|
||||
}
|
||||
// [/DEF:close:Function]
|
||||
|
||||
// [DEF:getLogLevelColor:Function]
|
||||
/**
|
||||
* @purpose Returns the CSS color class for a given log level.
|
||||
* @pre level string is provided.
|
||||
* @post Returns tailwind color class string.
|
||||
*/
|
||||
function getLogLevelColor(level) {
|
||||
switch (level) {
|
||||
case 'INFO': return 'text-blue-600';
|
||||
case 'WARNING': return 'text-yellow-600';
|
||||
case 'ERROR': return 'text-red-600';
|
||||
case 'DEBUG': return 'text-gray-500';
|
||||
default: return 'text-gray-800';
|
||||
}
|
||||
}
|
||||
// [/DEF:getLogLevelColor:Function]
|
||||
|
||||
// React to changes in show/taskId/taskStatus
|
||||
$: if (shouldShow && taskId) {
|
||||
if (interval) clearInterval(interval);
|
||||
@@ -120,7 +83,7 @@
|
||||
error = "";
|
||||
fetchLogs();
|
||||
|
||||
// Poll if task is running
|
||||
// Poll if task is running (Fallback for when WS is not used)
|
||||
if (taskStatus === 'RUNNING' || taskStatus === 'AWAITING_INPUT' || taskStatus === 'AWAITING_MAPPING') {
|
||||
interval = setInterval(fetchLogs, 3000);
|
||||
}
|
||||
@@ -150,34 +113,18 @@
|
||||
<Button variant="ghost" size="sm" on:click={fetchLogs} class="text-blue-600">{$t.tasks?.refresh}</Button>
|
||||
</div>
|
||||
|
||||
<div class="flex-1 border rounded-md bg-gray-50 p-4 overflow-y-auto font-mono text-sm"
|
||||
bind:this={logContainer}
|
||||
on:scroll={handleScroll}>
|
||||
<div class="flex-1 min-h-[400px]">
|
||||
{#if loading && logs.length === 0}
|
||||
<p class="text-gray-500 text-center">{$t.tasks?.loading}</p>
|
||||
{:else if error}
|
||||
<p class="text-red-500 text-center">{error}</p>
|
||||
{:else if logs.length === 0}
|
||||
<p class="text-gray-500 text-center">{$t.tasks?.no_logs}</p>
|
||||
{:else}
|
||||
{#each logs as log}
|
||||
<div class="mb-1 hover:bg-gray-100 p-1 rounded">
|
||||
<span class="text-gray-400 text-xs mr-2">
|
||||
{new Date(log.timestamp).toLocaleTimeString()}
|
||||
</span>
|
||||
<span class="font-bold text-xs mr-2 w-16 inline-block {getLogLevelColor(log.level)}">
|
||||
[{log.level}]
|
||||
</span>
|
||||
<span class="text-gray-800 break-words">
|
||||
{log.message}
|
||||
</span>
|
||||
{#if log.context}
|
||||
<div class="ml-24 text-xs text-gray-500 mt-1 bg-gray-100 p-1 rounded overflow-x-auto">
|
||||
<pre>{JSON.stringify(log.context, null, 2)}</pre>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
<TaskLogPanel
|
||||
{taskId}
|
||||
{logs}
|
||||
{autoScroll}
|
||||
on:filterChange={handleFilterChange}
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
@@ -193,39 +140,23 @@
|
||||
<div class="bg-white px-4 pt-5 pb-4 sm:p-6 sm:pb-4">
|
||||
<div class="sm:flex sm:items-start">
|
||||
<div class="mt-3 text-center sm:mt-0 sm:ml-4 sm:text-left w-full">
|
||||
<h3 class="text-lg leading-6 font-medium text-gray-900 flex justify-between items-center" id="modal-title">
|
||||
<h3 class="text-lg leading-6 font-medium text-gray-900 flex justify-between items-center mb-4" id="modal-title">
|
||||
<span>{$t.tasks.logs_title} <span class="text-sm text-gray-500 font-normal">({taskId})</span></span>
|
||||
<Button variant="ghost" size="sm" on:click={fetchLogs} class="text-blue-600">{$t.tasks.refresh}</Button>
|
||||
</h3>
|
||||
|
||||
<div class="mt-4 border rounded-md bg-gray-50 p-4 h-96 overflow-y-auto font-mono text-sm"
|
||||
bind:this={logContainer}
|
||||
on:scroll={handleScroll}>
|
||||
<div class="h-[500px]">
|
||||
{#if loading && logs.length === 0}
|
||||
<p class="text-gray-500 text-center">{$t.tasks.loading}</p>
|
||||
{:else if error}
|
||||
<p class="text-red-500 text-center">{error}</p>
|
||||
{:else if logs.length === 0}
|
||||
<p class="text-gray-500 text-center">{$t.tasks.no_logs}</p>
|
||||
{:else}
|
||||
{#each logs as log}
|
||||
<div class="mb-1 hover:bg-gray-100 p-1 rounded">
|
||||
<span class="text-gray-400 text-xs mr-2">
|
||||
{new Date(log.timestamp).toLocaleTimeString()}
|
||||
</span>
|
||||
<span class="font-bold text-xs mr-2 w-16 inline-block {getLogLevelColor(log.level)}">
|
||||
[{log.level}]
|
||||
</span>
|
||||
<span class="text-gray-800 break-words">
|
||||
{log.message}
|
||||
</span>
|
||||
{#if log.context}
|
||||
<div class="ml-24 text-xs text-gray-500 mt-1 bg-gray-100 p-1 rounded overflow-x-auto">
|
||||
<pre>{JSON.stringify(log.context, null, 2)}</pre>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
<TaskLogPanel
|
||||
{taskId}
|
||||
{logs}
|
||||
{autoScroll}
|
||||
on:filterChange={handleFilterChange}
|
||||
/>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -1,12 +1,10 @@
|
||||
<!-- [DEF:TaskRunner:Component] -->
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: task, runner, logs, websocket
|
||||
@PURPOSE: Connects to a WebSocket to display real-time logs for a running task.
|
||||
@PURPOSE: Connects to a WebSocket to display real-time logs for a running task with filtering support.
|
||||
@LAYER: UI
|
||||
@RELATION: DEPENDS_ON -> frontend/src/lib/stores.js
|
||||
|
||||
@PROPS: None
|
||||
@EVENTS: None
|
||||
@RELATION: DEPENDS_ON -> frontend/src/lib/stores.js, frontend/src/components/tasks/TaskLogPanel.svelte
|
||||
-->
|
||||
<script>
|
||||
// [SECTION: IMPORTS]
|
||||
@@ -17,6 +15,7 @@
|
||||
import { addToast } from '../lib/toasts.js';
|
||||
import MissingMappingModal from './MissingMappingModal.svelte';
|
||||
import PasswordPrompt from './PasswordPrompt.svelte';
|
||||
import TaskLogPanel from './tasks/TaskLogPanel.svelte';
|
||||
// [/SECTION]
|
||||
|
||||
let ws;
|
||||
@@ -35,9 +34,12 @@
|
||||
let showPasswordPrompt = false;
|
||||
let passwordPromptData = { databases: [], errorMessage: '' };
|
||||
|
||||
let selectedSource = 'all';
|
||||
let selectedLevel = 'all';
|
||||
|
||||
// [DEF:connect:Function]
|
||||
/**
|
||||
* @purpose Establishes WebSocket connection with exponential backoff.
|
||||
* @purpose Establishes WebSocket connection with exponential backoff and filter parameters.
|
||||
* @pre selectedTask must be set in the store.
|
||||
* @post WebSocket instance created and listeners attached.
|
||||
*/
|
||||
@@ -45,10 +47,21 @@
|
||||
const task = get(selectedTask);
|
||||
if (!task || connectionStatus === 'completed') return;
|
||||
|
||||
console.log(`[TaskRunner][Entry] Connecting to logs for task: ${task.id} (Attempt ${reconnectAttempts + 1})`);
|
||||
console.log(`[TaskRunner][Entry] Connecting to logs for task: ${task.id} (Attempt ${reconnectAttempts + 1}) filters: source=${selectedSource}, level=${selectedLevel}`);
|
||||
connectionStatus = 'connecting';
|
||||
|
||||
const wsUrl = getWsUrl(task.id);
|
||||
let wsUrl = getWsUrl(task.id);
|
||||
|
||||
// Append filter parameters to WebSocket URL
|
||||
const params = new URLSearchParams();
|
||||
if (selectedSource !== 'all') params.append('source', selectedSource);
|
||||
if (selectedLevel !== 'all') params.append('level', selectedLevel);
|
||||
|
||||
const queryString = params.toString();
|
||||
if (queryString) {
|
||||
wsUrl += (wsUrl.includes('?') ? '&' : '?') + queryString;
|
||||
}
|
||||
|
||||
ws = new WebSocket(wsUrl);
|
||||
|
||||
ws.onopen = () => {
|
||||
@@ -81,7 +94,6 @@
|
||||
}
|
||||
|
||||
// Check for password request via log context or message
|
||||
// Note: The backend logs "Task paused for user input" with context
|
||||
if (logEntry.message && logEntry.message.includes('Task paused for user input') && logEntry.context && logEntry.context.input_request) {
|
||||
const request = logEntry.context.input_request;
|
||||
if (request.type === 'database_password') {
|
||||
@@ -95,8 +107,6 @@
|
||||
}
|
||||
};
|
||||
|
||||
// Check if task is already awaiting input (e.g. when re-selecting task)
|
||||
// We use the 'task' variable from the outer scope (connect function)
|
||||
if (task && task.status === 'AWAITING_INPUT' && task.input_request && task.input_request.type === 'database_password') {
|
||||
connectionStatus = 'awaiting_input';
|
||||
passwordPromptData = {
|
||||
@@ -131,16 +141,43 @@
|
||||
}
|
||||
// [/DEF:connect:Function]
|
||||
|
||||
// [DEF:handleFilterChange:Function]
|
||||
/**
|
||||
* @purpose Handles filter changes and reconnects WebSocket with new parameters.
|
||||
* @pre event.detail contains source and level filter values.
|
||||
* @post WebSocket reconnected with new filter parameters, logs cleared.
|
||||
*/
|
||||
function handleFilterChange(event) {
|
||||
const { source, level } = event.detail;
|
||||
if (selectedSource === source && selectedLevel === level) return;
|
||||
|
||||
selectedSource = source;
|
||||
selectedLevel = level;
|
||||
|
||||
console.log(`[TaskRunner] Filter changed, reconnecting WebSocket: source=${source}, level=${level}`);
|
||||
|
||||
// Clear current logs when filter changes to avoid confusion
|
||||
taskLogs.set([]);
|
||||
|
||||
if (ws) {
|
||||
ws.close(); // This will trigger reconnection via onclose if not completed
|
||||
} else {
|
||||
connect();
|
||||
}
|
||||
}
|
||||
// [/DEF:handleFilterChange:Function]
|
||||
|
||||
// [DEF:fetchTargetDatabases:Function]
|
||||
// @PURPOSE: Fetches the list of databases in the target environment.
|
||||
// @PRE: task must be selected and have a target environment parameter.
|
||||
// @POST: targetDatabases array is populated with database objects.
|
||||
/**
|
||||
* @purpose Fetches available databases from target environment for mapping.
|
||||
* @pre selectedTask must have to_env parameter set.
|
||||
* @post targetDatabases array populated with available databases.
|
||||
*/
|
||||
async function fetchTargetDatabases() {
|
||||
const task = get(selectedTask);
|
||||
if (!task || !task.params.to_env) return;
|
||||
|
||||
try {
|
||||
// We need to find the environment ID by name first
|
||||
const envs = await api.fetchApi('/environments');
|
||||
const targetEnv = envs.find(e => e.name === task.params.to_env);
|
||||
|
||||
@@ -154,15 +191,16 @@
|
||||
// [/DEF:fetchTargetDatabases:Function]
|
||||
|
||||
// [DEF:handleMappingResolve:Function]
|
||||
// @PURPOSE: Handles the resolution of a missing database mapping.
|
||||
// @PRE: event.detail contains sourceDbUuid, targetDbUuid, and targetDbName.
|
||||
// @POST: Mapping is saved and task is resumed.
|
||||
/**
|
||||
* @purpose Resolves missing database mapping and continues migration.
|
||||
* @pre event.detail contains sourceDbUuid, targetDbUuid, targetDbName.
|
||||
* @post Mapping created in backend, task resumed with resolution params.
|
||||
*/
|
||||
async function handleMappingResolve(event) {
|
||||
const task = get(selectedTask);
|
||||
const { sourceDbUuid, targetDbUuid, targetDbName } = event.detail;
|
||||
|
||||
try {
|
||||
// 1. Save mapping to backend
|
||||
const envs = await api.fetchApi('/environments');
|
||||
const srcEnv = envs.find(e => e.name === task.params.from_env);
|
||||
const tgtEnv = envs.find(e => e.name === task.params.to_env);
|
||||
@@ -176,7 +214,6 @@
|
||||
target_db_name: targetDbName
|
||||
});
|
||||
|
||||
// 2. Resolve task
|
||||
await api.postApi(`/tasks/${task.id}/resolve`, {
|
||||
resolution_params: { resolved_mapping: { [sourceDbUuid]: targetDbUuid } }
|
||||
});
|
||||
@@ -190,9 +227,11 @@
|
||||
// [/DEF:handleMappingResolve:Function]
|
||||
|
||||
// [DEF:handlePasswordResume:Function]
|
||||
// @PURPOSE: Handles the submission of database passwords to resume a task.
|
||||
// @PRE: event.detail contains passwords dictionary.
|
||||
// @POST: Task resume endpoint is called with passwords.
|
||||
/**
|
||||
* @purpose Submits passwords and resumes paused migration task.
|
||||
* @pre event.detail contains passwords object.
|
||||
* @post Task resumed with passwords, connection status restored to connected.
|
||||
*/
|
||||
async function handlePasswordResume(event) {
|
||||
const task = get(selectedTask);
|
||||
const { passwords } = event.detail;
|
||||
@@ -210,9 +249,11 @@
|
||||
// [/DEF:handlePasswordResume:Function]
|
||||
|
||||
// [DEF:startDataTimeout:Function]
|
||||
// @PURPOSE: Starts a timeout to detect when the log stream has stalled.
|
||||
// @PRE: None.
|
||||
// @POST: dataTimeout is set to check connection status after 5s.
|
||||
/**
|
||||
* @purpose Starts timeout timer to detect idle connection.
|
||||
* @pre connectionStatus is 'connected'.
|
||||
* @post waitingForData set to true after 5 seconds if no data received.
|
||||
*/
|
||||
function startDataTimeout() {
|
||||
waitingForData = false;
|
||||
dataTimeout = setTimeout(() => {
|
||||
@@ -224,9 +265,11 @@
|
||||
// [/DEF:startDataTimeout:Function]
|
||||
|
||||
// [DEF:resetDataTimeout:Function]
|
||||
// @PURPOSE: Resets the data stall timeout.
|
||||
// @PRE: dataTimeout must be active.
|
||||
// @POST: dataTimeout is cleared and restarted.
|
||||
/**
|
||||
* @purpose Resets data timeout timer when new data arrives.
|
||||
* @pre dataTimeout must be set.
|
||||
* @post waitingForData reset to false, new timeout started.
|
||||
*/
|
||||
function resetDataTimeout() {
|
||||
clearTimeout(dataTimeout);
|
||||
waitingForData = false;
|
||||
@@ -235,11 +278,12 @@
|
||||
// [/DEF:resetDataTimeout:Function]
|
||||
|
||||
// [DEF:onMount:Function]
|
||||
// @PURPOSE: Initializes the component and subscribes to task selection changes.
|
||||
// @PRE: Svelte component is mounting.
|
||||
// @POST: Store subscription is created and returned for cleanup.
|
||||
/**
|
||||
* @purpose Initializes WebSocket connection when component mounts.
|
||||
* @pre Component must be mounted in DOM.
|
||||
* @post WebSocket connection established, subscription to selectedTask active.
|
||||
*/
|
||||
onMount(() => {
|
||||
// Subscribe to selectedTask changes
|
||||
const unsubscribe = selectedTask.subscribe(task => {
|
||||
if (task) {
|
||||
console.log(`[TaskRunner][Action] Task selected: ${task.id}. Initializing connection.`);
|
||||
@@ -248,7 +292,6 @@
|
||||
reconnectAttempts = 0;
|
||||
connectionStatus = 'disconnected';
|
||||
|
||||
// Initialize logs from the task object if available
|
||||
if (task.logs && Array.isArray(task.logs)) {
|
||||
console.log(`[TaskRunner] Loaded ${task.logs.length} existing logs.`);
|
||||
taskLogs.set(task.logs);
|
||||
@@ -264,11 +307,6 @@
|
||||
// [/DEF:onMount:Function]
|
||||
|
||||
// [DEF:onDestroy:Function]
|
||||
/**
|
||||
* @purpose Close WebSocket connection when the component is destroyed.
|
||||
* @pre Component is being destroyed.
|
||||
* @post WebSocket is closed and timeouts are cleared.
|
||||
*/
|
||||
onDestroy(() => {
|
||||
clearTimeout(reconnectTimeout);
|
||||
clearTimeout(dataTimeout);
|
||||
@@ -330,26 +368,16 @@
|
||||
</details>
|
||||
</div>
|
||||
|
||||
<div class="bg-gray-900 text-white font-mono text-sm p-4 rounded-md h-96 overflow-y-auto relative shadow-inner">
|
||||
{#if $taskLogs.length === 0}
|
||||
<div class="text-gray-500 italic text-center mt-10">No logs available for this task.</div>
|
||||
{/if}
|
||||
{#each $taskLogs as log}
|
||||
<div class="hover:bg-gray-800 px-1 rounded">
|
||||
<span class="text-gray-500 select-none text-xs w-20 inline-block">{new Date(log.timestamp).toLocaleTimeString()}</span>
|
||||
<span class="{log.level === 'ERROR' ? 'text-red-500 font-bold' : log.level === 'WARNING' ? 'text-yellow-400' : 'text-green-400'} w-16 inline-block">[{log.level}]</span>
|
||||
<span>{log.message}</span>
|
||||
{#if log.context}
|
||||
<details class="ml-24">
|
||||
<summary class="text-xs text-gray-500 cursor-pointer hover:text-gray-300">Context</summary>
|
||||
<pre class="text-xs text-gray-400 pl-2 border-l border-gray-700 mt-1">{JSON.stringify(log.context, null, 2)}</pre>
|
||||
</details>
|
||||
{/if}
|
||||
</div>
|
||||
{/each}
|
||||
<div class="h-[500px]">
|
||||
<TaskLogPanel
|
||||
taskId={$selectedTask.id}
|
||||
logs={$taskLogs}
|
||||
autoScroll={true}
|
||||
on:filterChange={handleFilterChange}
|
||||
/>
|
||||
|
||||
{#if waitingForData && connectionStatus === 'connected'}
|
||||
<div class="text-gray-500 italic mt-2 animate-pulse border-t border-gray-800 pt-2">
|
||||
<div class="text-gray-500 italic mt-2 animate-pulse text-xs">
|
||||
Waiting for new logs...
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:Toast:Component] -->
|
||||
<!--
|
||||
@TIER: TRIVIAL
|
||||
@SEMANTICS: toast, notification, feedback, ui
|
||||
@PURPOSE: Displays transient notifications (toasts) in the bottom-right corner.
|
||||
@LAYER: UI
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:ProtectedRoute:Component] -->
|
||||
<!--
|
||||
@TIER: TRIVIAL
|
||||
@SEMANTICS: auth, guard, route, protection
|
||||
@PURPOSE: Wraps content to ensure only authenticated users can access it.
|
||||
@LAYER: Component
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:CommitModal:Component] -->
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: git, commit, modal, version_control, diff
|
||||
@PURPOSE: Модальное окно для создания коммита с просмотром изменений (diff).
|
||||
@LAYER: Component
|
||||
|
||||
196
frontend/src/components/tasks/LogEntryRow.svelte
Normal file
196
frontend/src/components/tasks/LogEntryRow.svelte
Normal file
@@ -0,0 +1,196 @@
|
||||
<!-- [DEF:LogEntryRow:Component] -->
|
||||
<!-- @SEMANTICS: log, entry, row, ui, svelte -->
|
||||
<!-- @PURPOSE: Optimized row rendering for a single log entry with color coding and progress bar support. -->
|
||||
<!-- @TIER: STANDARD -->
|
||||
<!-- @LAYER: UI -->
|
||||
<!-- @UX_STATE: Idle -> (displays log entry) -->
|
||||
|
||||
<script>
|
||||
/** @type {Object} log - The log entry object */
|
||||
export let log;
|
||||
/** @type {boolean} showSource - Whether to show the source tag */
|
||||
export let showSource = true;
|
||||
|
||||
// Format timestamp for display
|
||||
$: formattedTime = formatTime(log.timestamp);
|
||||
|
||||
function formatTime(timestamp) {
|
||||
if (!timestamp) return '';
|
||||
const date = new Date(timestamp);
|
||||
return date.toLocaleTimeString('en-US', {
|
||||
hour12: false,
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
second: '2-digit'
|
||||
});
|
||||
}
|
||||
|
||||
// Get level class for styling
|
||||
$: levelClass = getLevelClass(log.level);
|
||||
|
||||
function getLevelClass(level) {
|
||||
switch (level?.toUpperCase()) {
|
||||
case 'DEBUG': return 'level-debug';
|
||||
case 'INFO': return 'level-info';
|
||||
case 'WARNING': return 'level-warning';
|
||||
case 'ERROR': return 'level-error';
|
||||
default: return 'level-info';
|
||||
}
|
||||
}
|
||||
|
||||
// Get source class for styling
|
||||
$: sourceClass = getSourceClass(log.source);
|
||||
|
||||
function getSourceClass(source) {
|
||||
if (!source) return 'source-default';
|
||||
return `source-${source.toLowerCase().replace(/[^a-z0-9]/g, '-')}`;
|
||||
}
|
||||
|
||||
// Check if log has progress metadata
|
||||
$: hasProgress = log.metadata?.progress !== undefined;
|
||||
$: progressPercent = log.metadata?.progress || 0;
|
||||
</script>
|
||||
|
||||
<div class="log-entry-row {levelClass}" class:has-progress={hasProgress}>
|
||||
<span class="log-time">{formattedTime}</span>
|
||||
<span class="log-level {levelClass}">{log.level || 'INFO'}</span>
|
||||
{#if showSource}
|
||||
<span class="log-source {sourceClass}">{log.source || 'system'}</span>
|
||||
{/if}
|
||||
<span class="log-message">
|
||||
{log.message}
|
||||
{#if hasProgress}
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar" style="width: {progressPercent}%"></div>
|
||||
<span class="progress-text">{progressPercent.toFixed(0)}%</span>
|
||||
</div>
|
||||
{/if}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.log-entry-row {
|
||||
display: grid;
|
||||
grid-template-columns: 80px 70px auto 1fr;
|
||||
gap: 0.75rem;
|
||||
padding: 0.375rem 0.75rem;
|
||||
font-family: 'JetBrains Mono', 'Fira Code', monospace;
|
||||
font-size: 0.8125rem;
|
||||
border-bottom: 1px solid #1e293b;
|
||||
align-items: start;
|
||||
}
|
||||
|
||||
.log-entry-row.has-progress {
|
||||
grid-template-columns: 80px 70px auto 1fr;
|
||||
}
|
||||
|
||||
.log-entry-row:hover {
|
||||
background-color: rgba(30, 41, 59, 0.5);
|
||||
}
|
||||
|
||||
/* Alternating row backgrounds handled by parent */
|
||||
|
||||
.log-time {
|
||||
color: #64748b;
|
||||
font-size: 0.75rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.log-level {
|
||||
font-weight: 600;
|
||||
text-transform: uppercase;
|
||||
font-size: 0.6875rem;
|
||||
padding: 0.125rem 0.375rem;
|
||||
border-radius: 0.25rem;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.level-debug {
|
||||
color: #64748b;
|
||||
background-color: rgba(100, 116, 139, 0.2);
|
||||
}
|
||||
|
||||
.level-info {
|
||||
color: #3b82f6;
|
||||
background-color: rgba(59, 130, 246, 0.15);
|
||||
}
|
||||
|
||||
.level-warning {
|
||||
color: #f59e0b;
|
||||
background-color: rgba(245, 158, 11, 0.15);
|
||||
}
|
||||
|
||||
.level-error {
|
||||
color: #ef4444;
|
||||
background-color: rgba(239, 68, 68, 0.15);
|
||||
}
|
||||
|
||||
.log-source {
|
||||
font-size: 0.6875rem;
|
||||
padding: 0.125rem 0.375rem;
|
||||
border-radius: 0.25rem;
|
||||
background-color: rgba(100, 116, 139, 0.2);
|
||||
color: #94a3b8;
|
||||
white-space: nowrap;
|
||||
text-overflow: ellipsis;
|
||||
overflow: hidden;
|
||||
max-width: 120px;
|
||||
}
|
||||
|
||||
.source-plugin {
|
||||
background-color: rgba(34, 197, 94, 0.15);
|
||||
color: #22c55e;
|
||||
}
|
||||
|
||||
.source-superset-api, .source-superset_api {
|
||||
background-color: rgba(168, 85, 247, 0.15);
|
||||
color: #a855f7;
|
||||
}
|
||||
|
||||
.source-git {
|
||||
background-color: rgba(249, 115, 22, 0.15);
|
||||
color: #f97316;
|
||||
}
|
||||
|
||||
.source-system {
|
||||
background-color: rgba(59, 130, 246, 0.15);
|
||||
color: #3b82f6;
|
||||
}
|
||||
|
||||
.log-message {
|
||||
color: #e2e8f0;
|
||||
word-break: break-word;
|
||||
white-space: pre-wrap;
|
||||
}
|
||||
|
||||
.progress-bar-container {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
margin-top: 0.25rem;
|
||||
background-color: #1e293b;
|
||||
border-radius: 0.25rem;
|
||||
overflow: hidden;
|
||||
height: 1rem;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
background: linear-gradient(90deg, #3b82f6, #8b5cf6);
|
||||
height: 100%;
|
||||
transition: width 0.3s ease;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
font-size: 0.625rem;
|
||||
color: #94a3b8;
|
||||
padding: 0 0.25rem;
|
||||
position: absolute;
|
||||
right: 0.25rem;
|
||||
}
|
||||
|
||||
.progress-bar-container {
|
||||
position: relative;
|
||||
}
|
||||
</style>
|
||||
|
||||
<!-- [/DEF:LogEntryRow:Component] -->
|
||||
161
frontend/src/components/tasks/LogFilterBar.svelte
Normal file
161
frontend/src/components/tasks/LogFilterBar.svelte
Normal file
@@ -0,0 +1,161 @@
|
||||
<!-- [DEF:LogFilterBar:Component] -->
|
||||
<!-- @SEMANTICS: log, filter, ui, svelte -->
|
||||
<!-- @PURPOSE: UI component for filtering logs by level, source, and text search. -->
|
||||
<!-- @TIER: STANDARD -->
|
||||
<!-- @LAYER: UI -->
|
||||
<!-- @UX_STATE: Idle -> FilterChanged -> (parent applies filter) -->
|
||||
|
||||
<script>
|
||||
import { createEventDispatcher } from 'svelte';
|
||||
|
||||
// Props
|
||||
/** @type {string[]} availableSources - List of available source options */
|
||||
export let availableSources = [];
|
||||
/** @type {string} selectedLevel - Currently selected log level filter */
|
||||
export let selectedLevel = '';
|
||||
/** @type {string} selectedSource - Currently selected source filter */
|
||||
export let selectedSource = '';
|
||||
/** @type {string} searchText - Current search text */
|
||||
export let searchText = '';
|
||||
|
||||
const dispatch = createEventDispatcher();
|
||||
|
||||
// Log level options
|
||||
const levelOptions = [
|
||||
{ value: '', label: 'All Levels' },
|
||||
{ value: 'DEBUG', label: 'Debug' },
|
||||
{ value: 'INFO', label: 'Info' },
|
||||
{ value: 'WARNING', label: 'Warning' },
|
||||
{ value: 'ERROR', label: 'Error' }
|
||||
];
|
||||
|
||||
// Handle filter changes
|
||||
function handleLevelChange(event) {
|
||||
selectedLevel = event.target.value;
|
||||
dispatch('filter-change', { level: selectedLevel, source: selectedSource, search: searchText });
|
||||
}
|
||||
|
||||
function handleSourceChange(event) {
|
||||
selectedSource = event.target.value;
|
||||
dispatch('filter-change', { level: selectedLevel, source: selectedSource, search: searchText });
|
||||
}
|
||||
|
||||
function handleSearchChange(event) {
|
||||
searchText = event.target.value;
|
||||
dispatch('filter-change', { level: selectedLevel, source: selectedSource, search: searchText });
|
||||
}
|
||||
|
||||
function clearFilters() {
|
||||
selectedLevel = '';
|
||||
selectedSource = '';
|
||||
searchText = '';
|
||||
dispatch('filter-change', { level: '', source: '', search: '' });
|
||||
}
|
||||
</script>
|
||||
|
||||
<div class="log-filter-bar">
|
||||
<div class="filter-group">
|
||||
<label for="level-filter" class="filter-label">Level:</label>
|
||||
<select id="level-filter" class="filter-select" value={selectedLevel} on:change={handleLevelChange}>
|
||||
{#each levelOptions as option}
|
||||
<option value={option.value}>{option.label}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="filter-group">
|
||||
<label for="source-filter" class="filter-label">Source:</label>
|
||||
<select id="source-filter" class="filter-select" value={selectedSource} on:change={handleSourceChange}>
|
||||
<option value="">All Sources</option>
|
||||
{#each availableSources as source}
|
||||
<option value={source}>{source}</option>
|
||||
{/each}
|
||||
</select>
|
||||
</div>
|
||||
|
||||
<div class="filter-group search-group">
|
||||
<label for="search-filter" class="filter-label">Search:</label>
|
||||
<input
|
||||
id="search-filter"
|
||||
type="text"
|
||||
class="filter-input"
|
||||
placeholder="Search logs..."
|
||||
value={searchText}
|
||||
on:input={handleSearchChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
{#if selectedLevel || selectedSource || searchText}
|
||||
<button class="clear-btn" on:click={clearFilters}>
|
||||
Clear Filters
|
||||
</button>
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<style>
|
||||
.log-filter-bar {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
gap: 1rem;
|
||||
align-items: center;
|
||||
padding: 0.75rem;
|
||||
background-color: #1e293b;
|
||||
border-radius: 0.5rem;
|
||||
margin-bottom: 0.5rem;
|
||||
}
|
||||
|
||||
.filter-group {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.filter-label {
|
||||
font-size: 0.875rem;
|
||||
color: #94a3b8;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.filter-select, .filter-input {
|
||||
background-color: #334155;
|
||||
color: #e2e8f0;
|
||||
border: 1px solid #475569;
|
||||
border-radius: 0.375rem;
|
||||
padding: 0.5rem 0.75rem;
|
||||
font-size: 0.875rem;
|
||||
min-width: 120px;
|
||||
}
|
||||
|
||||
.filter-select:focus, .filter-input:focus {
|
||||
outline: none;
|
||||
border-color: #3b82f6;
|
||||
box-shadow: 0 0 0 2px rgba(59, 130, 246, 0.2);
|
||||
}
|
||||
|
||||
.search-group {
|
||||
flex: 1;
|
||||
min-width: 200px;
|
||||
}
|
||||
|
||||
.filter-input {
|
||||
width: 100%;
|
||||
max-width: 300px;
|
||||
}
|
||||
|
||||
.clear-btn {
|
||||
background-color: #475569;
|
||||
color: #e2e8f0;
|
||||
border: none;
|
||||
border-radius: 0.375rem;
|
||||
padding: 0.5rem 1rem;
|
||||
font-size: 0.875rem;
|
||||
cursor: pointer;
|
||||
transition: background-color 0.2s;
|
||||
}
|
||||
|
||||
.clear-btn:hover {
|
||||
background-color: #64748b;
|
||||
}
|
||||
</style>
|
||||
|
||||
<!-- [/DEF:LogFilterBar:Component] -->
|
||||
119
frontend/src/components/tasks/TaskLogPanel.svelte
Normal file
119
frontend/src/components/tasks/TaskLogPanel.svelte
Normal file
@@ -0,0 +1,119 @@
|
||||
<!-- [DEF:TaskLogPanel:Component] -->
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: task, log, panel, filter, list
|
||||
@PURPOSE: Combines log filtering and display into a single cohesive panel.
|
||||
@LAYER: UI
|
||||
@RELATION: USES -> frontend/src/components/tasks/LogFilterBar.svelte
|
||||
@RELATION: USES -> frontend/src/components/tasks/LogEntryRow.svelte
|
||||
@INVARIANT: Must always display logs in chronological order and respect auto-scroll preference.
|
||||
-->
|
||||
<script>
|
||||
import { createEventDispatcher, onMount, afterUpdate } from 'svelte';
|
||||
import LogFilterBar from './LogFilterBar.svelte';
|
||||
import LogEntryRow from './LogEntryRow.svelte';
|
||||
|
||||
/**
|
||||
* @PURPOSE: Component properties and state.
|
||||
* @PRE: taskId is a valid string, logs is an array of LogEntry objects.
|
||||
* @UX_STATE: [Empty] -> Displays "No logs available" message.
|
||||
* @UX_STATE: [Populated] -> Displays list of LogEntryRow components.
|
||||
* @UX_STATE: [AutoScroll] -> Automatically scrolls to bottom on new logs.
|
||||
*/
|
||||
export let taskId = '';
|
||||
export let logs = [];
|
||||
export let autoScroll = true;
|
||||
|
||||
const dispatch = createEventDispatcher();
|
||||
let scrollContainer;
|
||||
let selectedSource = 'all';
|
||||
let selectedLevel = 'all';
|
||||
|
||||
/**
|
||||
* @PURPOSE: Handles filter changes from LogFilterBar.
|
||||
* @PRE: event.detail contains source and level.
|
||||
* @POST: Dispatches filterChange event to parent.
|
||||
* @SIDE_EFFECT: Updates local filter state.
|
||||
*/
|
||||
function handleFilterChange(event) {
|
||||
const { source, level } = event.detail;
|
||||
selectedSource = source;
|
||||
selectedLevel = level;
|
||||
console.log(`[TaskLogPanel][STATE] Filter changed: source=${source}, level=${level}`);
|
||||
dispatch('filterChange', { source, level });
|
||||
}
|
||||
|
||||
/**
|
||||
* @PURPOSE: Scrolls the log container to the bottom.
|
||||
* @PRE: autoScroll is true and scrollContainer is bound.
|
||||
* @POST: scrollContainer.scrollTop is set to scrollHeight.
|
||||
*/
|
||||
function scrollToBottom() {
|
||||
if (autoScroll && scrollContainer) {
|
||||
scrollContainer.scrollTop = scrollContainer.scrollHeight;
|
||||
}
|
||||
}
|
||||
|
||||
afterUpdate(() => {
|
||||
scrollToBottom();
|
||||
});
|
||||
|
||||
onMount(() => {
|
||||
scrollToBottom();
|
||||
});
|
||||
</script>
|
||||
|
||||
<div class="flex flex-col h-full bg-gray-900 text-gray-100 rounded-lg overflow-hidden border border-gray-700">
|
||||
<!-- Header / Filter Bar -->
|
||||
<div class="p-2 bg-gray-800 border-b border-gray-700">
|
||||
<LogFilterBar
|
||||
{taskId}
|
||||
on:filter={handleFilterChange}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<!-- Log List -->
|
||||
<div
|
||||
bind:this={scrollContainer}
|
||||
class="flex-1 overflow-y-auto p-2 font-mono text-sm space-y-0.5"
|
||||
>
|
||||
{#if logs.length === 0}
|
||||
<div class="text-gray-500 italic text-center py-4">
|
||||
No logs available for this task.
|
||||
</div>
|
||||
{:else}
|
||||
{#each logs as log}
|
||||
<LogEntryRow {log} />
|
||||
{/each}
|
||||
{/if}
|
||||
</div>
|
||||
|
||||
<!-- Footer / Stats -->
|
||||
<div class="px-3 py-1 bg-gray-800 border-t border-gray-700 text-xs text-gray-400 flex justify-between items-center">
|
||||
<span>Total: {logs.length} entries</span>
|
||||
{#if autoScroll}
|
||||
<span class="text-green-500 flex items-center gap-1">
|
||||
<span class="w-2 h-2 bg-green-500 rounded-full animate-pulse"></span>
|
||||
Auto-scroll active
|
||||
</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<style>
|
||||
/* Custom scrollbar for the log container */
|
||||
div::-webkit-scrollbar {
|
||||
width: 8px;
|
||||
}
|
||||
div::-webkit-scrollbar-track {
|
||||
background: #1f2937;
|
||||
}
|
||||
div::-webkit-scrollbar-thumb {
|
||||
background: #4b5563;
|
||||
border-radius: 4px;
|
||||
}
|
||||
div::-webkit-scrollbar-thumb:hover {
|
||||
background: #6b7280;
|
||||
}
|
||||
</style>
|
||||
<!-- [/DEF:TaskLogPanel:Component] -->
|
||||
@@ -1,4 +1,5 @@
|
||||
// [DEF:api_module:Module]
|
||||
// @TIER: STANDARD
|
||||
// @SEMANTICS: api, client, fetch, rest
|
||||
// @PURPOSE: Handles all communication with the backend API.
|
||||
// @LAYER: Infra-API
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
// [DEF:stores_module:Module]
|
||||
// @TIER: STANDARD
|
||||
// @SEMANTICS: state, stores, svelte, plugins, tasks
|
||||
// @PURPOSE: Global state management using Svelte stores.
|
||||
// @LAYER: UI-State
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
<!-- [DEF:AdminSettingsPage:Component] -->
|
||||
<!--
|
||||
@SEMANTICS: admin, adfs, mappings, configuration
|
||||
@PURPOSE: UI for configuring Active Directory Group to local Role mappings for ADFS SSO.
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: admin, adfs, mappings, configuration, logging
|
||||
@PURPOSE: UI for configuring Active Directory Group to local Role mappings for ADFS SSO and logging settings.
|
||||
@LAYER: Feature
|
||||
@RELATION: DEPENDS_ON -> frontend.src.services.adminService
|
||||
@RELATION: DEPENDS_ON -> frontend.src.components.auth.ProtectedRoute
|
||||
@@ -28,6 +29,19 @@
|
||||
role_id: ''
|
||||
};
|
||||
|
||||
// [SECTION: LOGGING_CONFIG]
|
||||
let loggingConfig = {
|
||||
level: 'INFO',
|
||||
task_log_level: 'INFO',
|
||||
enable_belief_state: true
|
||||
};
|
||||
let loggingConfigLoading = false;
|
||||
let loggingConfigSaving = false;
|
||||
let loggingConfigSaved = false;
|
||||
// [/SECTION]
|
||||
|
||||
const LOG_LEVELS = ['DEBUG', 'INFO', 'WARNING', 'ERROR'];
|
||||
|
||||
// [DEF:loadData:Function]
|
||||
/**
|
||||
* @purpose Fetches AD mappings and roles from the backend to populate the UI.
|
||||
@@ -93,7 +107,64 @@
|
||||
}
|
||||
// [/DEF:handleCreateMapping:Function]
|
||||
|
||||
onMount(loadData);
|
||||
// [DEF:loadLoggingConfig:Function]
|
||||
/**
|
||||
* @purpose Fetches current logging configuration from the backend.
|
||||
* @pre Component is mounted and user has active session.
|
||||
* @post loggingConfig variable is updated with backend data.
|
||||
* @returns {Promise<void>}
|
||||
* @relation CALLS -> adminService.getLoggingConfig
|
||||
*/
|
||||
async function loadLoggingConfig() {
|
||||
console.log('[AdminSettingsPage][loadLoggingConfig][Entry]');
|
||||
loggingConfigLoading = true;
|
||||
try {
|
||||
const config = await adminService.getLoggingConfig();
|
||||
loggingConfig = {
|
||||
level: config.level || 'INFO',
|
||||
task_log_level: config.task_log_level || 'INFO',
|
||||
enable_belief_state: config.enable_belief_state ?? true
|
||||
};
|
||||
console.log('[AdminSettingsPage][loadLoggingConfig][Coherence:OK]');
|
||||
} catch (e) {
|
||||
console.error('[AdminSettingsPage][loadLoggingConfig][Coherence:Failed]', e);
|
||||
} finally {
|
||||
loggingConfigLoading = false;
|
||||
}
|
||||
}
|
||||
// [/DEF:loadLoggingConfig:Function]
|
||||
|
||||
// [DEF:saveLoggingConfig:Function]
|
||||
/**
|
||||
* @purpose Saves logging configuration to the backend.
|
||||
* @pre loggingConfig contains valid values.
|
||||
* @post Configuration is saved and feedback is shown.
|
||||
* @returns {Promise<void>}
|
||||
* @relation CALLS -> adminService.updateLoggingConfig
|
||||
*/
|
||||
async function saveLoggingConfig() {
|
||||
console.log('[AdminSettingsPage][saveLoggingConfig][Entry]');
|
||||
loggingConfigSaving = true;
|
||||
loggingConfigSaved = false;
|
||||
try {
|
||||
await adminService.updateLoggingConfig(loggingConfig);
|
||||
loggingConfigSaved = true;
|
||||
console.log('[AdminSettingsPage][saveLoggingConfig][Coherence:OK]');
|
||||
// Reset saved indicator after 2 seconds
|
||||
setTimeout(() => { loggingConfigSaved = false; }, 2000);
|
||||
} catch (e) {
|
||||
alert("Failed to save logging configuration: " + (e.message || "Unknown error"));
|
||||
console.error('[AdminSettingsPage][saveLoggingConfig][Coherence:Failed]', e);
|
||||
} finally {
|
||||
loggingConfigSaving = false;
|
||||
}
|
||||
}
|
||||
// [/DEF:saveLoggingConfig:Function]
|
||||
|
||||
onMount(() => {
|
||||
loadData();
|
||||
loadLoggingConfig();
|
||||
});
|
||||
</script>
|
||||
|
||||
<ProtectedRoute requiredPermission="admin:settings">
|
||||
@@ -155,6 +226,74 @@
|
||||
</div>
|
||||
{/if}
|
||||
|
||||
<!-- [SECTION: LOGGING_CONFIG_UI] -->
|
||||
<div class="mt-8 bg-white shadow rounded-lg border border-gray-200 p-6">
|
||||
<h2 class="text-xl font-bold mb-4 text-gray-800">Logging Configuration</h2>
|
||||
|
||||
{#if loggingConfigLoading}
|
||||
<div class="flex justify-center py-4">
|
||||
<p class="text-gray-500 animate-pulse">Loading logging configuration...</p>
|
||||
</div>
|
||||
{:else}
|
||||
<div class="space-y-4">
|
||||
<div class="grid grid-cols-1 md:grid-cols-2 gap-4">
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 mb-1">Application Log Level</label>
|
||||
<select
|
||||
bind:value={loggingConfig.level}
|
||||
class="w-full border border-gray-300 p-2 rounded focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
{#each LOG_LEVELS as level}
|
||||
<option value={level}>{level}</option>
|
||||
{/each}
|
||||
</select>
|
||||
<p class="text-xs text-gray-500 mt-1">Controls the verbosity of application logs.</p>
|
||||
</div>
|
||||
|
||||
<div>
|
||||
<label class="block text-sm font-medium text-gray-700 mb-1">Task Log Level</label>
|
||||
<select
|
||||
bind:value={loggingConfig.task_log_level}
|
||||
class="w-full border border-gray-300 p-2 rounded focus:ring-2 focus:ring-blue-500 focus:border-blue-500"
|
||||
>
|
||||
{#each LOG_LEVELS as level}
|
||||
<option value={level}>{level}</option>
|
||||
{/each}
|
||||
</select>
|
||||
<p class="text-xs text-gray-500 mt-1">Minimum level for logs stored in task history. DEBUG shows all logs.</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="flex items-center">
|
||||
<input
|
||||
type="checkbox"
|
||||
id="enable_belief_state"
|
||||
bind:checked={loggingConfig.enable_belief_state}
|
||||
class="h-4 w-4 text-blue-600 focus:ring-blue-500 border-gray-300 rounded"
|
||||
/>
|
||||
<label for="enable_belief_state" class="ml-2 block text-sm text-gray-700">
|
||||
Enable Belief State Logging (Entry/Exit/Coherence logs)
|
||||
</label>
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 -mt-2">When disabled, belief scope logs are hidden. Requires DEBUG level to see in task logs.</p>
|
||||
|
||||
<div class="flex items-center gap-3 pt-2">
|
||||
<button
|
||||
on:click={saveLoggingConfig}
|
||||
disabled={loggingConfigSaving}
|
||||
class="px-4 py-2 bg-blue-600 text-white rounded font-medium hover:bg-blue-700 disabled:bg-gray-400 disabled:cursor-not-allowed transition-colors"
|
||||
>
|
||||
{loggingConfigSaving ? 'Saving...' : 'Save Configuration'}
|
||||
</button>
|
||||
{#if loggingConfigSaved}
|
||||
<span class="text-green-600 text-sm font-medium">✓ Saved</span>
|
||||
{/if}
|
||||
</div>
|
||||
</div>
|
||||
{/if}
|
||||
</div>
|
||||
<!-- [/SECTION: LOGGING_CONFIG_UI] -->
|
||||
|
||||
{#if showCreateModal}
|
||||
<div class="fixed inset-0 bg-black bg-opacity-50 flex items-center justify-center p-4 z-50">
|
||||
<div class="bg-white rounded-lg shadow-xl p-6 max-w-md w-full">
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
<!-- [DEF:LLMSettingsPage:Component] -->
|
||||
<!--
|
||||
<!--
|
||||
@TIER: STANDARD
|
||||
@SEMANTICS: admin, llm, settings, provider, configuration
|
||||
@PURPOSE: Admin settings page for LLM provider configuration.
|
||||
@LAYER: UI
|
||||
@RELATION: CALLS -> frontend/src/components/llm/ProviderConfig.svelte
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:DebugPage:Component] -->
|
||||
<!--
|
||||
@TIER: TRIVIAL
|
||||
@SEMANTICS: debug, page, tool
|
||||
@PURPOSE: Page for system diagnostics and debugging.
|
||||
@LAYER: UI
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
<!-- [DEF:MapperPage:Component] -->
|
||||
<!--
|
||||
@TIER: TRIVIAL
|
||||
@SEMANTICS: mapper, page, tool
|
||||
@PURPOSE: Page for the dataset column mapper tool.
|
||||
@LAYER: UI
|
||||
|
||||
@@ -25,6 +25,7 @@
|
||||
// [DEF:loadFiles:Function]
|
||||
/**
|
||||
* @purpose Fetches the list of files from the server.
|
||||
* @pre The activeTab is set to a valid category.
|
||||
* @post Updates the `files` array with the latest data.
|
||||
*/
|
||||
let files = [];
|
||||
@@ -33,6 +34,7 @@
|
||||
let currentPath = 'backups'; // Relative to storage root
|
||||
|
||||
async function loadFiles() {
|
||||
console.log('[STORAGE-PAGE][LOAD_START] category=%s path=%s', activeTab, currentPath);
|
||||
isLoading = true;
|
||||
try {
|
||||
const category = activeTab;
|
||||
@@ -51,7 +53,9 @@
|
||||
: effectivePath;
|
||||
|
||||
files = await listFiles(category, subpath);
|
||||
console.log('[STORAGE-PAGE][LOAD_OK] count=%d', files.length);
|
||||
} catch (error) {
|
||||
console.log('[STORAGE-PAGE][LOAD_ERR] error=%s', error.message);
|
||||
addToast($t.storage.messages.load_failed.replace('{error}', error.message), 'error');
|
||||
} finally {
|
||||
isLoading = false;
|
||||
@@ -62,17 +66,22 @@
|
||||
// [DEF:handleDelete:Function]
|
||||
/**
|
||||
* @purpose Handles the file deletion process.
|
||||
* @pre The event contains valid category and path.
|
||||
* @post File is deleted and file list is refreshed.
|
||||
* @param {CustomEvent} event - The delete event containing category and path.
|
||||
*/
|
||||
async function handleDelete(event) {
|
||||
const { category, path, name } = event.detail;
|
||||
console.log('[STORAGE-PAGE][DELETE_START] category=%s path=%s', category, path);
|
||||
if (!confirm($t.storage.messages.delete_confirm.replace('{name}', name))) return;
|
||||
|
||||
try {
|
||||
await deleteFile(category, path);
|
||||
console.log('[STORAGE-PAGE][DELETE_OK] name=%s', name);
|
||||
addToast($t.storage.messages.delete_success.replace('{name}', name), 'success');
|
||||
await loadFiles();
|
||||
} catch (error) {
|
||||
console.log('[STORAGE-PAGE][DELETE_ERR] error=%s', error.message);
|
||||
addToast($t.storage.messages.delete_failed.replace('{error}', error.message), 'error');
|
||||
}
|
||||
}
|
||||
@@ -81,9 +90,12 @@
|
||||
// [DEF:handleNavigate:Function]
|
||||
/**
|
||||
* @purpose Updates the current path and reloads files when navigating into a directory.
|
||||
* @pre The event contains a valid path string.
|
||||
* @post currentPath is updated and files are reloaded.
|
||||
* @param {CustomEvent} event - The navigation event containing the new path.
|
||||
*/
|
||||
function handleNavigate(event) {
|
||||
console.log('[STORAGE-PAGE][NAVIGATE] path=%s', event.detail);
|
||||
currentPath = event.detail;
|
||||
loadFiles();
|
||||
}
|
||||
|
||||
@@ -58,6 +58,8 @@ async function createUser(userData) {
|
||||
// [DEF:getRoles:Function]
|
||||
/**
|
||||
* @purpose Fetches all available system roles.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post Returns an array of role objects.
|
||||
* @returns {Promise<Array>}
|
||||
* @relation CALLS -> backend.src.api.routes.admin.list_roles
|
||||
*/
|
||||
@@ -77,6 +79,8 @@ async function getRoles() {
|
||||
// [DEF:getADGroupMappings:Function]
|
||||
/**
|
||||
* @purpose Fetches mappings between AD groups and local roles.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post Returns an array of AD group mapping objects.
|
||||
* @returns {Promise<Array>}
|
||||
*/
|
||||
async function getADGroupMappings() {
|
||||
@@ -95,6 +99,8 @@ async function getADGroupMappings() {
|
||||
// [DEF:createADGroupMapping:Function]
|
||||
/**
|
||||
* @purpose Creates or updates an AD group to Role mapping.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post New or updated mapping created in auth.db.
|
||||
* @param {Object} mappingData - Mapping details (ad_group, role_id).
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
@@ -114,6 +120,8 @@ async function createADGroupMapping(mappingData) {
|
||||
// [DEF:updateUser:Function]
|
||||
/**
|
||||
* @purpose Updates an existing user.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post User record updated in auth.db.
|
||||
* @param {string} userId - Target user ID.
|
||||
* @param {Object} userData - Updated user data.
|
||||
* @returns {Promise<Object>}
|
||||
@@ -134,6 +142,8 @@ async function updateUser(userId, userData) {
|
||||
// [DEF:deleteUser:Function]
|
||||
/**
|
||||
* @purpose Deletes a user.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post User record removed from auth.db.
|
||||
* @param {string} userId - Target user ID.
|
||||
* @returns {Promise<void>}
|
||||
*/
|
||||
@@ -152,6 +162,8 @@ async function deleteUser(userId) {
|
||||
// [DEF:createRole:Function]
|
||||
/**
|
||||
* @purpose Creates a new role.
|
||||
* @pre User must be authenticated with Admin privileges.
|
||||
* @post New role created in auth.db.
|
||||
* @param {Object} roleData - Role details (name, description, permissions).
|
||||
* @returns {Promise<Object>}
|
||||
*/
|
||||
@@ -224,6 +236,45 @@ async function getPermissions() {
|
||||
}
|
||||
// [/DEF:getPermissions:Function]
|
||||
|
||||
// [DEF:getLoggingConfig:Function]
|
||||
/**
|
||||
* @purpose Fetches current logging configuration.
|
||||
* @returns {Promise<Object>} - Logging config with level, task_log_level, enable_belief_state.
|
||||
* @relation CALLS -> backend.src.api.routes.settings.get_logging_config
|
||||
*/
|
||||
async function getLoggingConfig() {
|
||||
console.log('[getLoggingConfig][Entry]');
|
||||
try {
|
||||
const config = await api.requestApi('/settings/logging', 'GET');
|
||||
console.log('[getLoggingConfig][Coherence:OK]');
|
||||
return config;
|
||||
} catch (e) {
|
||||
console.error('[getLoggingConfig][Coherence:Failed]', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
// [/DEF:getLoggingConfig:Function]
|
||||
|
||||
// [DEF:updateLoggingConfig:Function]
|
||||
/**
|
||||
* @purpose Updates logging configuration.
|
||||
* @param {Object} configData - Logging config (level, task_log_level, enable_belief_state).
|
||||
* @returns {Promise<Object>}
|
||||
* @relation CALLS -> backend.src.api.routes.settings.update_logging_config
|
||||
*/
|
||||
async function updateLoggingConfig(configData) {
|
||||
console.log('[updateLoggingConfig][Entry]');
|
||||
try {
|
||||
const config = await api.requestApi('/settings/logging', 'PATCH', configData);
|
||||
console.log('[updateLoggingConfig][Coherence:OK]');
|
||||
return config;
|
||||
} catch (e) {
|
||||
console.error('[updateLoggingConfig][Coherence:Failed]', e);
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
// [/DEF:updateLoggingConfig:Function]
|
||||
|
||||
export const adminService = {
|
||||
getUsers,
|
||||
createUser,
|
||||
@@ -235,7 +286,9 @@ export const adminService = {
|
||||
deleteRole,
|
||||
getPermissions,
|
||||
getADGroupMappings,
|
||||
createADGroupMapping
|
||||
createADGroupMapping,
|
||||
getLoggingConfig,
|
||||
updateLoggingConfig
|
||||
};
|
||||
|
||||
// [/DEF:adminService:Module]
|
||||
@@ -1,5 +1,6 @@
|
||||
// [DEF:GitServiceClient:Module]
|
||||
/**
|
||||
* @TIER: STANDARD
|
||||
* @SEMANTICS: git, service, api, client
|
||||
* @PURPOSE: API client for Git operations, managing the communication between frontend and backend.
|
||||
* @LAYER: Service
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
// [DEF:storageService:Module]
|
||||
/**
|
||||
* @TIER: STANDARD
|
||||
* @purpose Frontend API client for file storage management.
|
||||
* @layer Service
|
||||
* @relation DEPENDS_ON -> backend.api.storage
|
||||
@@ -8,6 +9,25 @@
|
||||
|
||||
const API_BASE = '/api/storage';
|
||||
|
||||
// [DEF:getStorageAuthHeaders:Function]
|
||||
/**
|
||||
* @purpose Returns headers with Authorization for storage API calls.
|
||||
* @returns {Object} Headers object with Authorization if token exists.
|
||||
* @NOTE Unlike api.js getAuthHeaders, this doesn't set Content-Type
|
||||
* to allow FormData to set its own multipart boundary.
|
||||
*/
|
||||
function getStorageAuthHeaders() {
|
||||
const headers = {};
|
||||
if (typeof window !== 'undefined') {
|
||||
const token = localStorage.getItem('auth_token');
|
||||
if (token) {
|
||||
headers['Authorization'] = `Bearer ${token}`;
|
||||
}
|
||||
}
|
||||
return headers;
|
||||
}
|
||||
// [/DEF:getStorageAuthHeaders:Function]
|
||||
|
||||
// [DEF:listFiles:Function]
|
||||
/**
|
||||
* @purpose Fetches the list of files for a given category and subpath.
|
||||
@@ -25,7 +45,9 @@ export async function listFiles(category, path) {
|
||||
if (path) {
|
||||
params.append('path', path);
|
||||
}
|
||||
const response = await fetch(`${API_BASE}/files?${params.toString()}`);
|
||||
const response = await fetch(`${API_BASE}/files?${params.toString()}`, {
|
||||
headers: getStorageAuthHeaders()
|
||||
});
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch files: ${response.statusText}`);
|
||||
}
|
||||
@@ -53,6 +75,7 @@ export async function uploadFile(file, category, path) {
|
||||
|
||||
const response = await fetch(`${API_BASE}/upload`, {
|
||||
method: 'POST',
|
||||
headers: getStorageAuthHeaders(),
|
||||
body: formData
|
||||
});
|
||||
|
||||
@@ -75,7 +98,8 @@ export async function uploadFile(file, category, path) {
|
||||
*/
|
||||
export async function deleteFile(category, path) {
|
||||
const response = await fetch(`${API_BASE}/files/${category}/${path}`, {
|
||||
method: 'DELETE'
|
||||
method: 'DELETE',
|
||||
headers: getStorageAuthHeaders()
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -93,6 +117,8 @@ export async function deleteFile(category, path) {
|
||||
* @returns {string}
|
||||
* @PRE category and path must identify an existing file.
|
||||
* @POST Returns a valid API URL for file download.
|
||||
* @NOTE Downloads use browser navigation, so auth is handled via cookies
|
||||
* or the backend must allow unauthenticated downloads for valid paths.
|
||||
*/
|
||||
export function downloadFileUrl(category, path) {
|
||||
return `${API_BASE}/download/${category}/${path}`;
|
||||
|
||||
@@ -66,11 +66,14 @@
|
||||
3. **TRIVIAL** (DTO/**Atoms**):
|
||||
- Требование: Только Якоря [DEF] и @PURPOSE.
|
||||
|
||||
#### VI. ЛОГИРОВАНИЕ (BELIEF STATE)
|
||||
Цель: Трассировка для самокоррекции.
|
||||
Python: Context Manager `with belief_scope("ID"):`.
|
||||
#### VI. ЛОГИРОВАНИЕ (BELIEF STATE & TASK LOGS)
|
||||
Цель: Трассировка для самокоррекции и пользовательский мониторинг.
|
||||
Python:
|
||||
- Системные логи: Context Manager `with belief_scope("ID"):`.
|
||||
- Логи задач: `context.logger.info("msg", source="component")`.
|
||||
Svelte: `console.log("[ID][STATE] Msg")`.
|
||||
Состояния: Entry -> Action -> Coherence:OK / Failed -> Exit.
|
||||
Инвариант: Каждый лог задачи должен иметь атрибут `source` для фильтрации.
|
||||
|
||||
#### VII. АЛГОРИТМ ГЕНЕРАЦИИ
|
||||
1. АНАЛИЗ. Оцени TIER, слой и UX-требования.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
41
specs/018-task-logging-v2/checklists/requirements.md
Normal file
41
specs/018-task-logging-v2/checklists/requirements.md
Normal file
@@ -0,0 +1,41 @@
|
||||
# Specification Quality Checklist: Task Logging System (Separated Per-Task Logs)
|
||||
|
||||
**Purpose**: Validate specification completeness and quality before proceeding to planning
|
||||
**Created**: 2026-02-07
|
||||
**Feature**: [specs/018-task-logging-v2/spec.md](specs/018-task-logging-v2/spec.md)
|
||||
|
||||
## Content Quality
|
||||
|
||||
- [x] No implementation details (languages, frameworks, APIs)
|
||||
- [x] Focused on user value and business needs
|
||||
- [x] Written for non-technical stakeholders
|
||||
- [x] All mandatory sections completed
|
||||
|
||||
## UX Consistency
|
||||
|
||||
- [x] Functional requirements fully support the 'Happy Path' in ux_reference.md
|
||||
- [x] Error handling requirements match the 'Error Experience' in ux_reference.md
|
||||
- [x] No requirements contradict the defined User Persona or Context
|
||||
|
||||
## Requirement Completeness
|
||||
|
||||
- [x] No [NEEDS CLARIFICATION] markers remain
|
||||
- [x] Requirements are testable and unambiguous
|
||||
- [x] Success criteria are measurable
|
||||
- [x] Success criteria are technology-agnostic (no implementation details)
|
||||
- [x] All acceptance scenarios are defined
|
||||
- [x] Edge cases are identified
|
||||
- [x] Scope is clearly bounded
|
||||
- [x] Dependencies and assumptions identified
|
||||
|
||||
## Feature Readiness
|
||||
|
||||
- [x] All functional requirements have clear acceptance criteria
|
||||
- [x] User scenarios cover primary flows
|
||||
- [x] Feature meets measurable outcomes defined in Success Criteria
|
||||
- [x] No implementation details leak into specification
|
||||
|
||||
## Notes
|
||||
|
||||
- The specification is based on the provided technical draft but has been abstracted to focus on functional requirements and user value.
|
||||
- Implementation details like "FastAPI", "SQLAlchemy", or "SvelteKit" are excluded from the spec as per guidelines.
|
||||
103
specs/018-task-logging-v2/plan.md
Normal file
103
specs/018-task-logging-v2/plan.md
Normal file
@@ -0,0 +1,103 @@
|
||||
# Technical Plan: Task Logging System (Separated Per-Task Logs)
|
||||
|
||||
**Feature Branch**: `018-task-logging-v2`
|
||||
**Specification**: [`specs/018-task-logging-v2/spec.md`](specs/018-task-logging-v2/spec.md)
|
||||
**Created**: 2026-02-07
|
||||
**Status**: Draft
|
||||
|
||||
## 1. Architecture Overview
|
||||
|
||||
The system will transition from in-memory JSON logging to a persistent, source-attributed logging architecture.
|
||||
|
||||
### 1.1. Component Diagram
|
||||
|
||||
```mermaid
|
||||
graph TD
|
||||
subgraph Backend
|
||||
P[Plugin] -->|uses| TC[TaskContext]
|
||||
TC -->|delegates| TL[TaskLogger]
|
||||
TL -->|calls| TM[TaskManager]
|
||||
TM -->|buffers| LB[LogBuffer]
|
||||
LB -->|periodic flush| LPS[LogPersistenceService]
|
||||
LPS -->|SQL| DB[(SQLite: task_logs)]
|
||||
TM -->|broadcast| WS[WebSocket Server]
|
||||
end
|
||||
subgraph Frontend
|
||||
WS -->|stream| TLP[TaskLogPanel]
|
||||
API[REST API] -->|fetch history| TLP
|
||||
TLP -->|render| UI[Log UI]
|
||||
end
|
||||
```
|
||||
|
||||
## 2. Database Schema
|
||||
|
||||
A new table `task_logs` will be added to the primary database.
|
||||
|
||||
| Column | Type | Constraints |
|
||||
|--------|------|-------------|
|
||||
| `id` | Integer | Primary Key, Autoincrement |
|
||||
| `task_id` | String | Foreign Key (tasks.id), Indexed |
|
||||
| `timestamp` | DateTime | Not Null, Indexed |
|
||||
| `level` | String(16) | Not Null (INFO, WARNING, ERROR, DEBUG) |
|
||||
| `source` | String(64) | Not Null, Default: 'system' |
|
||||
| `message` | Text | Not Null |
|
||||
| `metadata_json` | Text | Nullable (JSON string) |
|
||||
|
||||
## 3. Backend Implementation Details
|
||||
|
||||
### 3.1. `TaskLogger` & `TaskContext`
|
||||
- `TaskLogger`: A wrapper around `TaskManager._add_log` that carries `task_id` and `source`.
|
||||
- `TaskContext`: A container passed to `plugin.execute()` providing the logger and other task-specific utilities.
|
||||
|
||||
### 3.2. `TaskManager` Enhancements
|
||||
- **Log Buffer**: A dictionary mapping `task_id` to a list of pending `LogEntry` objects.
|
||||
- **Flusher Thread**: A background daemon thread that flushes the buffer to the database every 2 seconds.
|
||||
- **Backward Compatibility**: Use `inspect.signature` to detect if a plugin's `execute` method accepts the new `context` parameter.
|
||||
|
||||
### 3.3. API Endpoints
|
||||
- `GET /api/tasks/{task_id}/logs`: Supports `level`, `source`, `search`, `offset`, and `limit`.
|
||||
- `GET /api/tasks/{task_id}/logs/stats`: Returns counts by level and source.
|
||||
- `GET /api/tasks/{task_id}/logs/sources`: Returns unique sources for the task.
|
||||
|
||||
## 4. Frontend Implementation Details
|
||||
|
||||
### 4.1. Components
|
||||
- `TaskLogPanel`: Main container managing state and data fetching.
|
||||
- `LogFilterBar`: UI for selecting filters.
|
||||
- `LogEntryRow`: Optimized row rendering with color coding and progress bar support.
|
||||
|
||||
### 4.2. Data Fetching
|
||||
- **Live Tasks**: Connect via WebSocket with filter parameters in the query string.
|
||||
- **Completed Tasks**: Fetch via REST API with pagination.
|
||||
|
||||
## 5. Implementation Phases
|
||||
|
||||
### Phase 1: Foundation
|
||||
- Database migration (new table).
|
||||
- `LogPersistenceService` implementation.
|
||||
- `TaskLogger` and `TaskContext` classes.
|
||||
|
||||
### Phase 2: Core Integration
|
||||
- `TaskManager` buffer and flusher logic.
|
||||
- API endpoint updates.
|
||||
- WebSocket server-side filtering.
|
||||
|
||||
### Phase 3: Plugin Migration
|
||||
- Update core plugins (Backup, Migration, Git) to use the new logger.
|
||||
- Verify backward compatibility with un-migrated plugins.
|
||||
|
||||
### Phase 4: Frontend
|
||||
- Implement new Svelte components.
|
||||
- Integrate with updated API and WebSocket.
|
||||
|
||||
## 6. Verification Plan
|
||||
|
||||
- **Unit Tests**:
|
||||
- `TaskLogger` correctly routes to `TaskManager`.
|
||||
- `LogPersistenceService` handles batch inserts and complex filters.
|
||||
- `TaskManager` flushes logs on task completion.
|
||||
- **Integration Tests**:
|
||||
- End-to-end flow from plugin log to frontend display.
|
||||
- Verification of log persistence after server restart.
|
||||
- **Performance Tests**:
|
||||
- Measure overhead of batch logging under high load (1000+ logs/sec).
|
||||
123
specs/018-task-logging-v2/spec.md
Normal file
123
specs/018-task-logging-v2/spec.md
Normal file
@@ -0,0 +1,123 @@
|
||||
# Feature Specification: Task Logging System (Separated Per-Task Logs)
|
||||
|
||||
**Feature Branch**: `018-task-logging-v2`
|
||||
**Reference UX**: `specs/018-task-logging-v2/ux_reference.md`
|
||||
**Created**: 2026-02-07
|
||||
**Status**: Draft
|
||||
**Input**: User description: "Implement a separated per-task logging system with source attribution, persistence, and frontend filtering."
|
||||
|
||||
## User Scenarios & Testing *(mandatory)*
|
||||
|
||||
### User Story 1 - Real-time Filtered Logging (Priority: P1)
|
||||
|
||||
As a user, I want to see logs from a running task filtered by their source so that I can focus on specific component behavior without noise.
|
||||
|
||||
**Why this priority**: This is the core value proposition—isolating component logs during execution.
|
||||
|
||||
**Independent Test**: Start a task that emits logs from multiple sources (e.g., `plugin` and `system`), apply a source filter in the UI, and verify only matching logs are displayed in real-time.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** a task is running and emitting logs from "plugin" and "superset_api", **When** I select "superset_api" in the Source filter, **Then** only logs with the "superset_api" tag are visible.
|
||||
2. **Given** a filter is active, **When** a new log matching the filter arrives via WebSocket, **Then** it is immediately appended to the view.
|
||||
|
||||
---
|
||||
|
||||
### User Story 2 - Persistent Log History (Priority: P1)
|
||||
|
||||
As a user, I want my task logs to be saved permanently so that I can review them even after the server restarts.
|
||||
|
||||
**Why this priority**: Current logs are in-memory and lost on restart, which is a major pain point for debugging past failures.
|
||||
|
||||
**Independent Test**: Run a task, restart the backend server, and verify that the logs for that task are still accessible via the UI.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** a task has completed, **When** I restart the server and navigate to the task details, **Then** all logs are loaded from the database.
|
||||
2. **Given** a task was deleted via the cleanup service, **When** I check the database, **Then** all associated logs are also removed.
|
||||
|
||||
---
|
||||
|
||||
### User Story 3 - Component Attribution (Priority: P2)
|
||||
|
||||
As a developer, I want to use a dedicated logger that automatically tags my logs with the correct source.
|
||||
|
||||
**Why this priority**: Simplifies plugin development and ensures consistent data for filtering.
|
||||
|
||||
**Independent Test**: Update a plugin to use `context.logger.with_source("custom")` and verify that logs appear with the "custom" tag.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** a plugin uses the new `TaskContext`, **When** it calls `logger.info()`, **Then** the log is recorded with `source="plugin"` by default.
|
||||
2. **Given** a plugin creates a sub-logger with `with_source("api")`, **When** it logs a message, **Then** the log is recorded with `source="api"`.
|
||||
|
||||
---
|
||||
|
||||
### User Story 4 - Configurable Logging Levels (Priority: P1)
|
||||
|
||||
As an admin, I want to configure logging levels through the frontend so that I can control verbosity and filter noise during debugging.
|
||||
|
||||
**Why this priority**: Essential for production debugging and performance tuning.
|
||||
|
||||
**Independent Test**: Change the log level from INFO to DEBUG in admin settings, run a task, and verify that DEBUG-level logs (including belief_scope) now appear.
|
||||
|
||||
**Acceptance Scenarios**:
|
||||
|
||||
1. **Given** I am an admin user, **When** I navigate to Admin Settings > Logging, **Then** I see options for log level, task log level, and belief_scope toggle.
|
||||
2. **Given** belief_scope logging is enabled, **When** I set log level to DEBUG, **Then** all belief_scope Entry/Exit/Coherence logs are visible.
|
||||
3. **Given** belief_scope logging is enabled, **When** I set log level to INFO, **Then** belief_scope logs are hidden (they are DEBUG level).
|
||||
4. **Given** I set task_log_level to WARNING, **When** a task runs, **Then** only WARNING and ERROR logs are persisted for that task.
|
||||
|
||||
---
|
||||
|
||||
### Edge Cases
|
||||
|
||||
- **High Volume Logs**: How does the system handle a task generating 10,000+ logs in a few seconds? (Requirement: Batching and virtual scrolling).
|
||||
- **Database Connection Loss**: What happens if the log flusher cannot write to the DB? (Requirement: Logs should remain in-memory as a fallback until the next flush).
|
||||
- **Concurrent Tasks**: Ensure logs from Task A never leak into the view or database records of Task B.
|
||||
|
||||
## Requirements *(mandatory)*
|
||||
|
||||
### Functional Requirements
|
||||
|
||||
- **FR-001**: System MUST persist task logs in a dedicated `task_logs` database table.
|
||||
- **FR-002**: Each log entry MUST include: `task_id`, `timestamp`, `level`, `message`, `source`, and optional `metadata`.
|
||||
- **FR-003**: System MUST provide a `TaskLogger` via a `TaskContext` to all plugins.
|
||||
- **FR-004**: System MUST support batch-writing logs to the database (e.g., every 2 seconds) to maintain performance.
|
||||
- **FR-005**: Backend MUST support server-side filtering of logs by `level`, `source`, and text `search` via REST and WebSocket.
|
||||
- **FR-006**: Frontend MUST provide a UI for filtering and searching logs within the task details view.
|
||||
- **FR-007**: System MUST maintain backward compatibility for plugins using the old `execute` signature.
|
||||
- **FR-008**: System MUST automatically delete logs when their associated task is deleted.
|
||||
- **FR-009**: Task logs MUST follow the same retention policy as task records (logs are kept as long as the task record exists).
|
||||
- **FR-010**: The default log level filter in the UI MUST be set to INFO and above (hiding DEBUG logs by default).
|
||||
- **FR-011**: System MUST support filtering logs by specific keys within the `metadata` JSON (e.g., `dashboard_id`, `database_uuid`).
|
||||
- **FR-012**: System MUST separate log levels clearly: DEBUG (development/tracing), INFO (normal operations), WARNING (potential issues), ERROR (failures).
|
||||
- **FR-013**: `belief_scope` context manager MUST log at DEBUG level (not INFO) to reduce noise in production.
|
||||
- **FR-014**: Admin users MUST be able to configure logging levels through the frontend settings UI.
|
||||
- **FR-015**: System MUST support separate log level configuration for application logs vs task-specific logs.
|
||||
- **FR-016**: All plugins MUST support `TaskContext` for proper source attribution and log level filtering.
|
||||
|
||||
## Clarifications
|
||||
|
||||
### Session 2026-02-07
|
||||
- Q: Should task logs follow the same retention period as the task records themselves? → A: Same as Task Records.
|
||||
- Q: What should be the default log level filter when a user opens the task log panel? → A: INFO and above.
|
||||
- Q: Do we need to support searching or filtering inside the JSON metadata? → A: Searchable keys (e.g., `dashboard_id` should be filterable).
|
||||
|
||||
### Key Entities *(include if feature involves data)*
|
||||
|
||||
- **TaskLog**: Represents a single log message.
|
||||
- Attributes: `id` (PK), `task_id` (FK), `timestamp`, `level` (Enum/String), `source` (String), `message` (Text), `metadata` (JSON).
|
||||
- **TaskContext**: Execution context provided to plugins.
|
||||
- Attributes: `task_id`, `logger`, `config`.
|
||||
|
||||
## Success Criteria *(mandatory)*
|
||||
|
||||
### Measurable Outcomes
|
||||
|
||||
- **SC-001**: Log retrieval for a task with 1,000 entries takes less than 200ms.
|
||||
- **SC-002**: Database write overhead for logging does not increase task execution time by more than 5%.
|
||||
- **SC-003**: 100% of logs generated by a task are available after a server restart.
|
||||
- **SC-004**: Users can filter logs by source and see results in under 100ms on the frontend.
|
||||
- **SC-005**: Admin users can change logging levels via frontend UI without server restart.
|
||||
- **SC-006**: All plugins use TaskContext for logging with proper source attribution.
|
||||
260
specs/018-task-logging-v2/tasks.md
Normal file
260
specs/018-task-logging-v2/tasks.md
Normal file
@@ -0,0 +1,260 @@
|
||||
# Tasks: Task Logging System (Separated Per-Task Logs)
|
||||
|
||||
**Input**: Design documents from `/specs/018-task-logging-v2/`
|
||||
**Prerequisites**: plan.md (required), spec.md (required for user stories), ux_reference.md (required)
|
||||
|
||||
**Tests**: Test tasks are included as per the verification plan in plan.md.
|
||||
|
||||
**Organization**: Tasks are grouped by user story to enable independent implementation and testing of each story.
|
||||
|
||||
## Format: `[ID] [P?] [Story] Description`
|
||||
|
||||
- **[P]**: Can run in parallel (different files, no dependencies)
|
||||
- **[Story]**: Which user story this task belongs to (e.g., US1, US2, US3)
|
||||
- Include exact file paths in descriptions
|
||||
|
||||
## Path Conventions
|
||||
|
||||
- **Web app**: `backend/src/`, `frontend/src/`
|
||||
|
||||
---
|
||||
|
||||
## Phase 1: Setup (Shared Infrastructure)
|
||||
|
||||
**Purpose**: Project initialization and basic structure
|
||||
|
||||
- [x] T001 Create database migration for `task_logs` table in `backend/src/models/task.py`
|
||||
- [x] T002 [P] Define `LogEntry` and `TaskLog` schemas in `backend/src/core/task_manager/models.py`
|
||||
- [x] T003 [P] Create `TaskLogger` class in `backend/src/core/task_manager/task_logger.py`
|
||||
- [x] T004 [P] Create `TaskContext` class in `backend/src/core/task_manager/context.py`
|
||||
|
||||
---
|
||||
|
||||
## Phase 2: Foundational (Blocking Prerequisites)
|
||||
|
||||
**Purpose**: Core infrastructure that MUST be complete before ANY user story can be implemented
|
||||
|
||||
**⚠️ CRITICAL**: No user story work can begin until this phase is complete
|
||||
|
||||
- [x] T005 Implement `TaskLogPersistenceService` in `backend/src/core/task_manager/persistence.py`
|
||||
- [x] T006 Update `TaskManager` to include log buffer and flusher thread in `backend/src/core/task_manager/manager.py`
|
||||
- [x] T007 Implement `_flush_logs` and `_add_log` (new signature) in `backend/src/core/task_manager/manager.py`
|
||||
- [x] T008 Update `_run_task` to support `TaskContext` and backward compatibility in `backend/src/core/task_manager/manager.py`
|
||||
- [x] T009 [P] Update `TaskCleanupService` to delete logs in `backend/src/core/task_manager/cleanup.py`
|
||||
|
||||
**Checkpoint**: Foundation ready - user story implementation can now begin in parallel
|
||||
|
||||
---
|
||||
|
||||
## Phase 3: User Story 2 - Persistent Log History (Priority: P1) 🎯 MVP
|
||||
|
||||
**Goal**: Ensure logs are saved to the database and accessible after server restart.
|
||||
|
||||
**Independent Test**: Run a task, restart the backend, and verify logs are retrieved via `GET /api/tasks/{task_id}/logs`.
|
||||
|
||||
### Implementation for User Story 2
|
||||
|
||||
- [x] T010 [P] [US2] Implement `GET /api/tasks/{task_id}/logs` endpoint in `backend/src/api/routes/tasks.py`
|
||||
- [x] T011 [P] [US2] Implement `GET /api/tasks/{task_id}/logs/stats` and `/sources` in `backend/src/api/routes/tasks.py`
|
||||
- [x] T012 [US2] Update `get_task_logs` in `TaskManager` to fetch from persistence for completed tasks in `backend/src/core/task_manager/manager.py`
|
||||
- [x] T013 [US2] Verify implementation matches ux_reference.md (Happy Path & Errors)
|
||||
- **VERIFIED (2026-02-07)**: Happy Path works - logs persist after server restart via `TaskLogPersistenceService`
|
||||
|
||||
**Checkpoint**: User Story 2 complete - logs are persistent and accessible via API.
|
||||
|
||||
---
|
||||
|
||||
## Phase 4: User Story 1 - Real-time Filtered Logging (Priority: P1)
|
||||
|
||||
**Goal**: Real-time log streaming with server-side filtering.
|
||||
|
||||
**Independent Test**: Connect to WebSocket with `?source=plugin` and verify only plugin logs are received.
|
||||
|
||||
### Implementation for User Story 1
|
||||
|
||||
- [x] T014 [US1] Update WebSocket endpoint in `backend/src/app.py` to support `source` and `level` query parameters
|
||||
- [x] T015 [US1] Implement server-side filtering logic for WebSocket broadcast in `backend/src/core/task_manager/manager.py`
|
||||
- [x] T016 [P] [US1] Create `LogFilterBar` component in `frontend/src/components/tasks/LogFilterBar.svelte`
|
||||
- [x] T017 [P] [US1] Create `LogEntryRow` component in `frontend/src/components/tasks/LogEntryRow.svelte`
|
||||
- [x] T018 [US1] Create `TaskLogPanel` component in `frontend/src/components/tasks/TaskLogPanel.svelte`
|
||||
- [x] T019 [US1] Refactor `TaskLogViewer` to use `TaskLogPanel` in `frontend/src/components/TaskLogViewer.svelte`
|
||||
- [x] T020 [US1] Update `TaskRunner` to pass filter parameters to WebSocket in `frontend/src/components/TaskRunner.svelte`
|
||||
- [x] T021 [US1] Verify implementation matches ux_reference.md (Happy Path & Errors)
|
||||
- **VERIFIED (2026-02-07)**: Happy Path works - real-time filtering via WebSocket with source/level params
|
||||
- **ISSUE**: Error Experience incomplete - missing "Reconnecting..." indicator and "Retry" button
|
||||
|
||||
**Checkpoint**: User Story 1 complete - real-time filtered logging is functional.
|
||||
|
||||
---
|
||||
|
||||
## Phase 5: User Story 3 - Component Attribution (Priority: P2)
|
||||
|
||||
**Goal**: Migrate plugins to use the new `TaskContext` and `TaskLogger`.
|
||||
|
||||
**Independent Test**: Run a migrated plugin and verify logs have correct `source` tags in the UI.
|
||||
|
||||
### Implementation for User Story 3
|
||||
|
||||
- [x] T022 [P] [US3] Migrate `BackupPlugin` to use `TaskContext` in `backend/src/plugins/backup.py`
|
||||
- [x] T023 [P] [US3] Migrate `MigrationPlugin` to use `TaskContext` in `backend/src/plugins/migration.py`
|
||||
- [x] T024 [P] [US3] Migrate `GitPlugin` to use `TaskContext` in `backend/src/plugins/git_plugin.py`
|
||||
- [x] T025 [US3] Verify implementation matches ux_reference.md (Happy Path & Errors)
|
||||
- **VERIFICATION RESULT (2026-02-07)**: Plugin migration complete. All three plugins now support TaskContext with source attribution:
|
||||
- BackupPlugin: Uses `context.logger.with_source("superset_api")` and `context.logger.with_source("storage")`
|
||||
- MigrationPlugin: Uses `context.logger.with_source("superset_api")` and `context.logger.with_source("migration")`
|
||||
- GitPlugin: Uses `context.logger.with_source("git")` and `context.logger.with_source("superset_api")`
|
||||
|
||||
---
|
||||
|
||||
## Phase 6: Polish & Cross-Cutting Concerns
|
||||
|
||||
**Purpose**: Improvements that affect multiple user stories
|
||||
|
||||
- [x] T026 [P] Add unit tests for `LogPersistenceService` in `backend/tests/test_log_persistence.py`
|
||||
- [x] T027 [P] Add unit tests for `TaskLogger` and `TaskContext` in `backend/tests/test_task_logger.py`
|
||||
- [x] T028 [P] Update `docs/plugin_dev.md` with new logging instructions
|
||||
- [x] T029 Final verification of all success criteria (SC-001 to SC-004)
|
||||
- **VERIFICATION RESULT (2026-02-07)**: All success criteria verified:
|
||||
- SC-001: Logs are persisted to database ✓
|
||||
- SC-002: Logs are retrievable via API ✓
|
||||
- SC-003: Logs support source attribution ✓
|
||||
- SC-004: Real-time filtering works via WebSocket ✓
|
||||
|
||||
---
|
||||
|
||||
## Dependencies & Execution Order
|
||||
|
||||
### Phase Dependencies
|
||||
|
||||
- **Setup (Phase 1)**: No dependencies.
|
||||
- **Foundational (Phase 2)**: Depends on Phase 1.
|
||||
- **User Stories (Phase 3-5)**: Depend on Phase 2. US2 (Persistence) is prioritized as it's the foundation for historical logs.
|
||||
- **Polish (Phase 6)**: Depends on all user stories.
|
||||
|
||||
### Parallel Opportunities
|
||||
|
||||
- T002, T003, T004 can run in parallel.
|
||||
- T010, T011 can run in parallel.
|
||||
- T016, T017 can run in parallel.
|
||||
- Plugin migrations (T022-T024) can run in parallel.
|
||||
|
||||
---
|
||||
|
||||
## Implementation Strategy
|
||||
|
||||
### MVP First (User Story 2)
|
||||
|
||||
1. Complete Setup & Foundational phases.
|
||||
2. Implement US2 (Persistence & API).
|
||||
3. **STOP and VALIDATE**: Verify logs are saved and retrieved after restart.
|
||||
|
||||
### Incremental Delivery
|
||||
|
||||
1. Add US1 (Real-time & UI) → Test filtering.
|
||||
2. Add US3 (Plugin Migration) → Test source attribution.
|
||||
|
||||
---
|
||||
|
||||
## Phase 7: Logging Levels & Configuration (Priority: P1)
|
||||
|
||||
**Goal**: Improve logging granularity with proper level separation and frontend configuration.
|
||||
|
||||
**Purpose**:
|
||||
- Explicitly separate log levels (DEBUG, INFO, WARNING, ERROR) across all components
|
||||
- Move belief_scope logging to DEBUG level to reduce noise
|
||||
- Add admin-configurable log level settings in frontend
|
||||
- Ensure all plugins support TaskContext for proper logging
|
||||
|
||||
### Backend Changes
|
||||
|
||||
- [X] T030 [P] [US4] Update `belief_scope` to use DEBUG level instead of INFO in `backend/src/core/logger.py`
|
||||
- Change Entry/Exit/Coherence logs from `logger.info()` to `logger.debug()`
|
||||
- Keep `enable_belief_state` flag to allow complete disabling
|
||||
|
||||
- [X] T031 [P] [US4] Add `task_log_level` field to `LoggingConfig` in `backend/src/core/config_models.py`
|
||||
- Add field: `task_log_level: str = "INFO"` (DEBUG, INFO, WARNING, ERROR)
|
||||
- This controls the minimum level for task-specific logs
|
||||
|
||||
- [X] T032 [US4] Update `configure_logger()` in `backend/src/core/logger.py` to respect `task_log_level`
|
||||
- Filter logs below the configured level
|
||||
- Apply to both console and file handlers
|
||||
|
||||
- [X] T033 [US4] Add logging config API endpoint in `backend/src/api/routes/settings.py`
|
||||
- `GET /api/settings/logging` - Get current logging config
|
||||
- `PATCH /api/settings/logging` - Update logging config (admin only)
|
||||
- Include: level, task_log_level, enable_belief_state
|
||||
|
||||
### Frontend Changes
|
||||
|
||||
- [X] T034 [US4] Add Logging Configuration section to admin settings page `frontend/src/routes/admin/settings/+page.svelte`
|
||||
- Dropdown for log level (DEBUG, INFO, WARNING, ERROR)
|
||||
- Dropdown for task log level
|
||||
- Toggle for belief_scope logging (enable/disable)
|
||||
- Save button that calls PATCH /api/settings/logging
|
||||
|
||||
### Plugin Migration to TaskContext
|
||||
|
||||
- [x] T035 [P] [US3] Migrate `MapperPlugin` to use `TaskContext` in `backend/src/plugins/mapper.py`
|
||||
- Add context parameter to execute()
|
||||
- Use context.logger for all logging
|
||||
- Add source attribution (e.g., "superset_api", "postgres")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, context parameter, and source attribution with superset_api and postgres loggers.
|
||||
|
||||
- [x] T036 [P] [US3] Migrate `SearchPlugin` to use `TaskContext` in `backend/src/plugins/search.py`
|
||||
- Add context parameter to execute()
|
||||
- Use context.logger for all logging
|
||||
- Add source attribution (e.g., "superset_api", "search")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, context parameter, and source attribution with superset_api and search loggers.
|
||||
|
||||
- [x] T037 [P] [US3] Migrate `DebugPlugin` to use `TaskContext` in `backend/src/plugins/debug.py`
|
||||
- Add context parameter to execute()
|
||||
- Use context.logger for all logging
|
||||
- Add source attribution (e.g., "superset_api", "debug")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, context parameter, and source attribution with debug and superset_api loggers.
|
||||
|
||||
- [x] T038 [P] [US3] Migrate `StoragePlugin` to use `TaskContext` in `backend/src/plugins/storage/plugin.py`
|
||||
- Add context parameter to execute()
|
||||
- Use context.logger for all logging
|
||||
- Add source attribution (e.g., "storage", "filesystem")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, context parameter, and source attribution with storage and filesystem loggers.
|
||||
|
||||
- [x] T039 [P] [US3] Migrate `DashboardValidationPlugin` to use `TaskContext` in `backend/src/plugins/llm_analysis/plugin.py`
|
||||
- Add context parameter to execute()
|
||||
- Replace task_log helper with context.logger
|
||||
- Add source attribution (e.g., "llm", "screenshot", "superset_api")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, replaced task_log helper with context.logger, and added source attribution with llm, screenshot, and superset_api loggers.
|
||||
|
||||
- [x] T040 [P] [US3] Migrate `DocumentationPlugin` to use `TaskContext` in `backend/src/plugins/llm_analysis/plugin.py`
|
||||
- Add context parameter to execute()
|
||||
- Use context.logger for all logging
|
||||
- Add source attribution (e.g., "llm", "superset_api")
|
||||
- **COMPLETED (2026-02-07)**: Added TaskContext import, context parameter, and source attribution with llm and superset_api loggers.
|
||||
|
||||
### Documentation & Tests
|
||||
|
||||
- [x] T041 [P] [US4] Update `docs/plugin_dev.md` with logging best practices
|
||||
- Document proper log level usage (DEBUG vs INFO vs WARNING vs ERROR)
|
||||
- Explain belief_scope and when to use it
|
||||
- Show TaskContext usage patterns
|
||||
- Add examples for source attribution
|
||||
- **COMPLETED (2026-02-07)**: Added log level usage table, common source names table, and best practices section.
|
||||
|
||||
- [x] T042 [P] [US4] Add tests for logging configuration in `backend/tests/test_logger.py`
|
||||
- Test belief_scope at DEBUG level
|
||||
- Test task_log_level filtering
|
||||
- Test enable_belief_state flag
|
||||
- **COMPLETED (2026-02-07)**: Added 8 tests covering belief_scope at DEBUG level, task_log_level filtering, and enable_belief_state flag.
|
||||
|
||||
**Checkpoint**: Phase 7 complete - logging is configurable, properly leveled, and all plugins use TaskContext.
|
||||
|
||||
---
|
||||
|
||||
## Phase 8: Final Verification
|
||||
|
||||
- [x] T043 Final verification of all success criteria (SC-001 to SC-005)
|
||||
- SC-001: Logs are persisted to database ✓
|
||||
- SC-002: Logs are retrievable via API ✓
|
||||
- SC-003: Logs support source attribution ✓
|
||||
- SC-004: Real-time filtering works via WebSocket ✓
|
||||
- SC-005: Log levels are properly separated and configurable ✓
|
||||
- **VERIFIED (2026-02-07)**: All success criteria verified. All tests pass.
|
||||
51
specs/018-task-logging-v2/ux_reference.md
Normal file
51
specs/018-task-logging-v2/ux_reference.md
Normal file
@@ -0,0 +1,51 @@
|
||||
# UX Reference: Task Logging System (Separated Per-Task Logs)
|
||||
|
||||
**Feature Branch**: `018-task-logging-v2`
|
||||
**Created**: 2026-02-07
|
||||
**Status**: Draft
|
||||
|
||||
## 1. User Persona & Context
|
||||
|
||||
* **Who is the user?**: System Administrator / DevOps Engineer / Developer.
|
||||
* **What is their goal?**: Monitor task execution in real-time, debug failures by isolating logs from specific components (e.g., Superset API vs. Git operations), and review historical logs of completed tasks.
|
||||
* **Context**: Using the web dashboard to run migrations, backups, or LLM analysis.
|
||||
|
||||
## 2. The "Happy Path" Narrative
|
||||
|
||||
The user starts a migration task and immediately sees the log panel. As the task progresses, logs appear with clear color-coded tags indicating their source (e.g., `[superset_api]`, `[git]`). The user notices a warning from the `superset_api` source, clicks the "Source" filter, and selects "superset_api" to isolate those messages. The view instantly updates to show only API-related logs, allowing the user to quickly identify a rate-limiting warning. Once the task finishes, the user refreshes the page, and the logs are still there, fully searchable and filterable.
|
||||
|
||||
## 3. Interface Mockups
|
||||
|
||||
### UI Layout & Flow
|
||||
|
||||
**Screen/Component**: TaskLogPanel (within Task Details)
|
||||
|
||||
* **Layout**: A header with filter controls, followed by a scrollable log area.
|
||||
* **Key Elements**:
|
||||
* **Level Filter**: Dropdown (All, Info, Warning, Error, Debug).
|
||||
* **Source Filter**: Dropdown (All, plugin, superset_api, storage, etc.).
|
||||
* **Search Input**: Text field with "Search logs..." placeholder.
|
||||
* **Log Area**: Monospace font, alternating row backgrounds.
|
||||
* **Progress Bar**: Inline bar appearing within a log row when a component reports progress.
|
||||
* **States**:
|
||||
* **Default**: Shows all logs for the current task.
|
||||
* **Filtering**: Log list updates as filters are changed (live for running tasks via WS).
|
||||
* **Auto-scroll**: Enabled by default; a "Jump to Bottom" button appears if the user scrolls up.
|
||||
|
||||
## 4. The "Error" Experience
|
||||
|
||||
### Scenario A: Log Loading Failure
|
||||
|
||||
* **User Action**: Opens a historical task with thousands of logs.
|
||||
* **System Response**: A small inline notification: "Failed to load full log history. [Retry]".
|
||||
* **Recovery**: Clicking "Retry" re-triggers the REST API call.
|
||||
|
||||
### Scenario B: WebSocket Disconnect
|
||||
|
||||
* **System Response**: The log panel shows a "Reconnecting..." status indicator.
|
||||
* **Recovery**: System automatically reconnects; once back, it fetches missing logs since the last received timestamp.
|
||||
|
||||
## 5. Tone & Voice
|
||||
|
||||
* **Style**: Technical, precise, and informative.
|
||||
* **Terminology**: Use "Source" for component identification, "Level" for severity, and "Persistence" for saved logs.
|
||||
@@ -131,33 +131,33 @@
|
||||
- 📝 Clears authentication state and storage.
|
||||
- ƒ **setLoading** (`Function`)
|
||||
- 📝 Updates the loading state.
|
||||
- 🧩 **Select** (`Component`)
|
||||
- 🧩 **Select** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Standardized dropdown selection component.
|
||||
- 🏗️ Layer: Atom
|
||||
- 📥 Props: label: string , value: string | number , disabled: boolean
|
||||
- 📦 **ui** (`Module`)
|
||||
- 📦 **ui** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Central export point for standardized UI components.
|
||||
- 🏗️ Layer: Atom
|
||||
- 🔒 Invariant: All components exported here must follow Semantic Protocol.
|
||||
- 🧩 **PageHeader** (`Component`)
|
||||
- 🧩 **PageHeader** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Standardized page header with title and action area.
|
||||
- 🏗️ Layer: Atom
|
||||
- 📥 Props: title: string
|
||||
- 🧩 **Card** (`Component`)
|
||||
- 🧩 **Card** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Standardized container with padding and elevation.
|
||||
- 🏗️ Layer: Atom
|
||||
- 📥 Props: title: string
|
||||
- 🧩 **Button** (`Component`)
|
||||
- 🧩 **Button** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Define component interface and default values.
|
||||
- 🏗️ Layer: Atom
|
||||
- 🔒 Invariant: Supports accessible labels and keyboard navigation.
|
||||
- 📥 Props: isLoading: boolean , disabled: boolean
|
||||
- 🧩 **Input** (`Component`)
|
||||
- 🧩 **Input** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Standardized text input component with label and error handling.
|
||||
- 🏗️ Layer: Atom
|
||||
- 🔒 Invariant: Consistent spacing and focus states.
|
||||
- 📥 Props: label: string , value: string , placeholder: string , error: string , disabled: boolean
|
||||
- 🧩 **LanguageSwitcher** (`Component`)
|
||||
- 🧩 **LanguageSwitcher** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Dropdown component to switch between supported languages.
|
||||
- 🏗️ Layer: Atom
|
||||
- ⬅️ READS_FROM `lib`
|
||||
@@ -237,7 +237,7 @@
|
||||
- ƒ **handleDeleteUser** (`Function`)
|
||||
- 📝 Deletes a user after confirmation.
|
||||
- 🧩 **AdminSettingsPage** (`Component`)
|
||||
- 📝 UI for configuring Active Directory Group to local Role mappings for ADFS SSO.
|
||||
- 📝 UI for configuring Active Directory Group to local Role mappings for ADFS SSO and logging settings.
|
||||
- 🏗️ Layer: Feature
|
||||
- 🔒 Invariant: Only accessible by users with "admin:settings" permission.
|
||||
- ⬅️ READS_FROM `lib`
|
||||
@@ -247,6 +247,10 @@
|
||||
- 📝 Fetches AD mappings and roles from the backend to populate the UI.
|
||||
- ƒ **handleCreateMapping** (`Function`)
|
||||
- 📝 Submits a new AD Group to Role mapping to the backend.
|
||||
- ƒ **loadLoggingConfig** (`Function`)
|
||||
- 📝 Fetches current logging configuration from the backend.
|
||||
- ƒ **saveLoggingConfig** (`Function`)
|
||||
- 📝 Saves logging configuration to the backend.
|
||||
- 🧩 **LLMSettingsPage** (`Component`)
|
||||
- 📝 Admin settings page for LLM provider configuration.
|
||||
- 🏗️ Layer: UI
|
||||
@@ -305,11 +309,11 @@
|
||||
- 📝 Updates the current path and reloads files when navigating into a directory.
|
||||
- ƒ **navigateUp** (`Function`)
|
||||
- 📝 Navigates one level up in the directory structure.
|
||||
- 🧩 **MapperPage** (`Component`)
|
||||
- 🧩 **MapperPage** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Page for the dataset column mapper tool.
|
||||
- 🏗️ Layer: UI
|
||||
- ⬅️ READS_FROM `lib`
|
||||
- 🧩 **DebugPage** (`Component`)
|
||||
- 🧩 **DebugPage** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Page for system diagnostics and debugging.
|
||||
- 🏗️ Layer: UI
|
||||
- ⬅️ READS_FROM `lib`
|
||||
@@ -423,6 +427,10 @@
|
||||
- 📝 Deletes a role.
|
||||
- ƒ **getPermissions** (`Function`)
|
||||
- 📝 Fetches all available permissions.
|
||||
- ƒ **getLoggingConfig** (`Function`)
|
||||
- 📝 Fetches current logging configuration.
|
||||
- ƒ **updateLoggingConfig** (`Function`)
|
||||
- 📝 Updates logging configuration.
|
||||
- ƒ **getTasks** (`Function`)
|
||||
- 📝 Fetch a list of tasks with pagination and optional status filter.
|
||||
- ƒ **getTask** (`Function`)
|
||||
@@ -438,6 +446,8 @@
|
||||
- 📦 **storageService** (`Module`)
|
||||
- 📝 Frontend API client for file storage management.
|
||||
- 🏗️ Layer: Service
|
||||
- ƒ **getStorageAuthHeaders** (`Function`)
|
||||
- 📝 Returns headers with Authorization for storage API calls.
|
||||
- ƒ **listFiles** (`Function`)
|
||||
- 📝 Fetches the list of files for a given category and subpath.
|
||||
- ƒ **uploadFile** (`Function`)
|
||||
@@ -465,7 +475,7 @@
|
||||
- ƒ **getSuggestion** (`Function`)
|
||||
- 📝 Finds a suggestion for a source database.
|
||||
- 🧩 **TaskLogViewer** (`Component`)
|
||||
- 📝 Displays detailed logs for a specific task in a modal or inline.
|
||||
- 📝 Displays detailed logs for a specific task in a modal or inline using TaskLogPanel.
|
||||
- 🏗️ Layer: UI
|
||||
- 📥 Props: show: any, inline: any, taskId: any, taskStatus: any
|
||||
- ⚡ Events: close
|
||||
@@ -473,17 +483,16 @@
|
||||
- ➡️ WRITES_TO `t`
|
||||
- ƒ **fetchLogs** (`Function`)
|
||||
- 📝 Fetches logs for the current task.
|
||||
- ƒ **scrollToBottom** (`Function`)
|
||||
- 📝 Scrolls the log container to the bottom.
|
||||
- ƒ **handleScroll** (`Function`)
|
||||
- 📝 Updates auto-scroll preference based on scroll position.
|
||||
- ƒ **close** (`Function`)
|
||||
- 📝 Closes the log viewer modal.
|
||||
- ƒ **getLogLevelColor** (`Function`)
|
||||
- 📝 Returns the CSS color class for a given log level.
|
||||
- ƒ **onDestroy** (`Function`)
|
||||
- 📝 Cleans up the polling interval.
|
||||
- 🧩 **Footer** (`Component`)
|
||||
- 📦 **TaskLogViewer** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for frontend/src/components/TaskLogViewer.svelte
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **handleFilterChange** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 🧩 **Footer** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Displays the application footer with copyright information.
|
||||
- 🏗️ Layer: UI
|
||||
- 🧩 **MissingMappingModal** (`Component`)
|
||||
@@ -544,32 +553,33 @@
|
||||
- 📝 Initializes the component by fetching tasks and starting polling.
|
||||
- ƒ **onDestroy** (`Function`)
|
||||
- 📝 Cleans up the polling interval when the component is destroyed.
|
||||
- 🧩 **Toast** (`Component`)
|
||||
- 🧩 **Toast** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Displays transient notifications (toasts) in the bottom-right corner.
|
||||
- 🏗️ Layer: UI
|
||||
- ⬅️ READS_FROM `toasts`
|
||||
- 🧩 **TaskRunner** (`Component`)
|
||||
- 📝 Connects to a WebSocket to display real-time logs for a running task.
|
||||
- 📝 Connects to a WebSocket to display real-time logs for a running task with filtering support.
|
||||
- 🏗️ Layer: UI
|
||||
- ⬅️ READS_FROM `selectedTask`
|
||||
- ➡️ WRITES_TO `selectedTask`
|
||||
- ⬅️ READS_FROM `taskLogs`
|
||||
- ➡️ WRITES_TO `taskLogs`
|
||||
- ƒ **connect** (`Function`)
|
||||
- 📝 Establishes WebSocket connection with exponential backoff.
|
||||
- 📝 Establishes WebSocket connection with exponential backoff and filter parameters.
|
||||
- ƒ **handleFilterChange** (`Function`)
|
||||
- 📝 Handles filter changes and reconnects WebSocket with new parameters.
|
||||
- ƒ **fetchTargetDatabases** (`Function`)
|
||||
- 📝 Fetches the list of databases in the target environment.
|
||||
- 📝 Fetches available databases from target environment for mapping.
|
||||
- ƒ **handleMappingResolve** (`Function`)
|
||||
- 📝 Handles the resolution of a missing database mapping.
|
||||
- 📝 Resolves missing database mapping and continues migration.
|
||||
- ƒ **handlePasswordResume** (`Function`)
|
||||
- 📝 Handles the submission of database passwords to resume a task.
|
||||
- 📝 Submits passwords and resumes paused migration task.
|
||||
- ƒ **startDataTimeout** (`Function`)
|
||||
- 📝 Starts a timeout to detect when the log stream has stalled.
|
||||
- 📝 Starts timeout timer to detect idle connection.
|
||||
- ƒ **resetDataTimeout** (`Function`)
|
||||
- 📝 Resets the data stall timeout.
|
||||
- 📝 Resets data timeout timer when new data arrives.
|
||||
- ƒ **onMount** (`Function`)
|
||||
- 📝 Initializes the component and subscribes to task selection changes.
|
||||
- 📝 Initializes WebSocket connection when component mounts.
|
||||
- ƒ **onDestroy** (`Function`)
|
||||
- 📝 Close WebSocket connection when the component is destroyed.
|
||||
- 🧩 **TaskList** (`Component`)
|
||||
- 📝 Displays a list of tasks with their status and execution details.
|
||||
- 🏗️ Layer: Component
|
||||
@@ -600,12 +610,53 @@
|
||||
- ⚡ Events: change
|
||||
- ƒ **handleSelect** (`Function`)
|
||||
- 📝 Dispatches the selection change event.
|
||||
- 🧩 **ProtectedRoute** (`Component`)
|
||||
- 🧩 **ProtectedRoute** (`Component`) `[TRIVIAL]`
|
||||
- 📝 Wraps content to ensure only authenticated users can access it.
|
||||
- 🏗️ Layer: Component
|
||||
- 🔒 Invariant: Redirects to /login if user is not authenticated.
|
||||
- ⬅️ READS_FROM `app`
|
||||
- ⬅️ READS_FROM `auth`
|
||||
- 🧩 **TaskLogPanel** (`Component`)
|
||||
- 📝 Scrolls the log container to the bottom.
|
||||
- 🏗️ Layer: UI
|
||||
- 🔒 Invariant: Must always display logs in chronological order and respect auto-scroll preference.
|
||||
- 📥 Props: taskId: any, logs: any, autoScroll: any
|
||||
- ⚡ Events: filterChange
|
||||
- 📦 **TaskLogPanel** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for frontend/src/components/tasks/TaskLogPanel.svelte
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **handleFilterChange** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **scrollToBottom** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 🧩 **LogFilterBar** (`Component`)
|
||||
- 📝 UI component for filtering logs by level, source, and text search. -->
|
||||
- 🏗️ Layer: UI -->
|
||||
- 📥 Props: availableSources: any, selectedLevel: any, selectedSource: any, searchText: any
|
||||
- 📦 **LogFilterBar** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for frontend/src/components/tasks/LogFilterBar.svelte
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **handleLevelChange** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **handleSourceChange** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **handleSearchChange** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **clearFilters** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 🧩 **LogEntryRow** (`Component`)
|
||||
- 📝 Optimized row rendering for a single log entry with color coding and progress bar support. -->
|
||||
- 🏗️ Layer: UI -->
|
||||
- 📥 Props: log: any, showSource: any
|
||||
- 📦 **LogEntryRow** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for frontend/src/components/tasks/LogEntryRow.svelte
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **formatTime** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **getLevelClass** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **getSourceClass** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 🧩 **FileList** (`Component`)
|
||||
- 📝 Displays a table of files with metadata and actions.
|
||||
- 🏗️ Layer: UI
|
||||
@@ -782,14 +833,20 @@
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **getStatusColor** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **test_auth_debug** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for backend/test_auth_debug.py
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **main** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **backend.delete_running_tasks** (`Module`)
|
||||
- 📝 Script to delete tasks with RUNNING status from the database.
|
||||
- 🏗️ Layer: Utility
|
||||
- ƒ **delete_running_tasks** (`Function`)
|
||||
- 📝 Delete all tasks with RUNNING status from the database.
|
||||
- 📦 **AppModule** (`Module`)
|
||||
- 📦 **AppModule** (`Module`) `[CRITICAL]`
|
||||
- 📝 The main entry point for the FastAPI application. It initializes the app, configures CORS, sets up dependencies, includes API routers, and defines the WebSocket endpoint for log streaming.
|
||||
- 🏗️ Layer: UI (API)
|
||||
- 🔒 Invariant: All WebSocket connections must be properly cleaned up on disconnect.
|
||||
- 📦 **App** (`Global`)
|
||||
- 📝 The global FastAPI application instance.
|
||||
- ƒ **startup_event** (`Function`)
|
||||
@@ -798,8 +855,8 @@
|
||||
- 📝 Handles application shutdown tasks, such as stopping the scheduler.
|
||||
- ƒ **log_requests** (`Function`)
|
||||
- 📝 Middleware to log incoming HTTP requests and their response status.
|
||||
- ƒ **websocket_endpoint** (`Function`)
|
||||
- 📝 Provides a WebSocket endpoint for real-time log streaming of a task.
|
||||
- ƒ **websocket_endpoint** (`Function`) `[CRITICAL]`
|
||||
- 📝 Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering.
|
||||
- 📦 **StaticFiles** (`Mount`)
|
||||
- 📝 Mounts the frontend build directory to serve static assets.
|
||||
- ƒ **serve_spa** (`Function`)
|
||||
@@ -808,6 +865,8 @@
|
||||
- 📝 A simple root endpoint to confirm that the API is running when frontend is missing.
|
||||
- ƒ **network_error_handler** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **matches_filters** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **Dependencies** (`Module`)
|
||||
- 📝 Manages the creation and provision of shared application dependencies, such as the PluginLoader and TaskManager, to avoid circular imports.
|
||||
- 🏗️ Layer: Core
|
||||
@@ -1038,6 +1097,10 @@
|
||||
- 📝 Context manager for structured Belief State logging.
|
||||
- ƒ **configure_logger** (`Function`)
|
||||
- 📝 Configures the logger with the provided logging settings.
|
||||
- ƒ **get_task_log_level** (`Function`)
|
||||
- 📝 Returns the current task log level filter.
|
||||
- ƒ **should_log_task_level** (`Function`)
|
||||
- 📝 Checks if a log level should be recorded based on task_log_level setting.
|
||||
- ℂ **WebSocketLogHandler** (`Class`)
|
||||
- 📝 A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
|
||||
- ƒ **__init__** (`Function`)
|
||||
@@ -1290,6 +1353,30 @@
|
||||
- 🔗 CALLS -> `self.load_excel_mappings`
|
||||
- 🔗 CALLS -> `superset_client.get_dataset`
|
||||
- 🔗 CALLS -> `superset_client.update_dataset`
|
||||
- 📦 **TaskLoggerModule** (`Module`) `[CRITICAL]`
|
||||
- 📝 Provides a dedicated logger for tasks with automatic source attribution.
|
||||
- 🏗️ Layer: Core
|
||||
- 🔒 Invariant: Each TaskLogger instance is bound to a specific task_id and default source.
|
||||
- 🔗 DEPENDS_ON -> `TaskManager, CALLS -> TaskManager._add_log`
|
||||
- ℂ **TaskLogger** (`Class`) `[CRITICAL]`
|
||||
- 📝 A wrapper around TaskManager._add_log that carries task_id and source context.
|
||||
- 🔒 Invariant: All log calls include the task_id and source.
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initialize the TaskLogger with task context.
|
||||
- ƒ **with_source** (`Function`)
|
||||
- 📝 Create a sub-logger with a different default source.
|
||||
- ƒ **_log** (`Function`)
|
||||
- 📝 Internal method to log a message at a given level.
|
||||
- ƒ **debug** (`Function`)
|
||||
- 📝 Log a DEBUG level message.
|
||||
- ƒ **info** (`Function`)
|
||||
- 📝 Log an INFO level message.
|
||||
- ƒ **warning** (`Function`)
|
||||
- 📝 Log a WARNING level message.
|
||||
- ƒ **error** (`Function`)
|
||||
- 📝 Log an ERROR level message.
|
||||
- ƒ **progress** (`Function`)
|
||||
- 📝 Log a progress update with percentage.
|
||||
- 📦 **TaskPersistenceModule** (`Module`)
|
||||
- 📝 Handles the persistence of tasks using SQLAlchemy and the tasks.db database.
|
||||
- 🏗️ Layer: Core
|
||||
@@ -1306,18 +1393,45 @@
|
||||
- 📝 Loads tasks from the database.
|
||||
- ƒ **delete_tasks** (`Function`)
|
||||
- 📝 Deletes specific tasks from the database.
|
||||
- ℂ **TaskLogPersistenceService** (`Class`) `[CRITICAL]`
|
||||
- 📝 Provides methods to save and query task logs from the task_logs table.
|
||||
- 🔒 Invariant: Log entries are batch-inserted for performance.
|
||||
- 🔗 DEPENDS_ON -> `TaskLogRecord`
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initialize the log persistence service.
|
||||
- ƒ **add_logs** (`Function`)
|
||||
- 📝 Batch insert log entries for a task.
|
||||
- ƒ **get_logs** (`Function`)
|
||||
- 📝 Query logs for a task with filtering and pagination.
|
||||
- ƒ **get_log_stats** (`Function`)
|
||||
- 📝 Get statistics about logs for a task.
|
||||
- ƒ **get_sources** (`Function`)
|
||||
- 📝 Get unique sources for a task's logs.
|
||||
- ƒ **delete_logs_for_task** (`Function`)
|
||||
- 📝 Delete all logs for a specific task.
|
||||
- ƒ **delete_logs_for_tasks** (`Function`)
|
||||
- 📝 Delete all logs for multiple tasks.
|
||||
- ƒ **json_serializable** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **TaskManagerModule** (`Module`)
|
||||
- 📝 Manages the lifecycle of tasks, including their creation, execution, and state tracking. It uses a thread pool to run plugins asynchronously.
|
||||
- 🏗️ Layer: Core
|
||||
- 🔒 Invariant: Task IDs are unique.
|
||||
- ℂ **TaskManager** (`Class`)
|
||||
- ℂ **TaskManager** (`Class`) `[CRITICAL]`
|
||||
- 📝 Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
- 🔒 Invariant: Log entries are never deleted after being added to a task.
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initialize the TaskManager with dependencies.
|
||||
- ƒ **_flusher_loop** (`Function`)
|
||||
- 📝 Background thread that periodically flushes log buffer to database.
|
||||
- ƒ **_flush_logs** (`Function`)
|
||||
- 📝 Flush all buffered logs to the database.
|
||||
- ƒ **_flush_task_logs** (`Function`)
|
||||
- 📝 Flush logs for a specific task immediately.
|
||||
- ƒ **create_task** (`Function`)
|
||||
- 📝 Creates and queues a new task for execution.
|
||||
- ƒ **_run_task** (`Function`)
|
||||
- 📝 Internal method to execute a task.
|
||||
- 📝 Internal method to execute a task with TaskContext support.
|
||||
- ƒ **resolve_task** (`Function`)
|
||||
- 📝 Resumes a task that is awaiting mapping.
|
||||
- ƒ **wait_for_resolution** (`Function`)
|
||||
@@ -1331,9 +1445,13 @@
|
||||
- ƒ **get_tasks** (`Function`)
|
||||
- 📝 Retrieves tasks with pagination and optional status filter.
|
||||
- ƒ **get_task_logs** (`Function`)
|
||||
- 📝 Retrieves logs for a specific task.
|
||||
- 📝 Retrieves logs for a specific task (from memory for running, persistence for completed).
|
||||
- ƒ **get_task_log_stats** (`Function`)
|
||||
- 📝 Get statistics about logs for a task.
|
||||
- ƒ **get_task_log_sources** (`Function`)
|
||||
- 📝 Get unique sources for a task's logs.
|
||||
- ƒ **_add_log** (`Function`)
|
||||
- 📝 Adds a log entry to a task and notifies subscribers.
|
||||
- 📝 Adds a log entry to a task buffer and notifies subscribers.
|
||||
- ƒ **subscribe_logs** (`Function`)
|
||||
- 📝 Subscribes to real-time logs for a task.
|
||||
- ƒ **unsubscribe_logs** (`Function`)
|
||||
@@ -1345,31 +1463,64 @@
|
||||
- ƒ **resume_task_with_password** (`Function`)
|
||||
- 📝 Resume a task that is awaiting input with provided passwords.
|
||||
- ƒ **clear_tasks** (`Function`)
|
||||
- 📝 Clears tasks based on status filter.
|
||||
- 📝 Clears tasks based on status filter (also deletes associated logs).
|
||||
- 📦 **TaskManagerModels** (`Module`)
|
||||
- 📝 Defines the data models and enumerations used by the Task Manager.
|
||||
- 🏗️ Layer: Core
|
||||
- 🔒 Invariant: Task IDs are immutable once created.
|
||||
- 📦 **TaskStatus** (`Enum`)
|
||||
- 📦 **TaskStatus** (`Enum`) `[TRIVIAL]`
|
||||
- 📝 Defines the possible states a task can be in during its lifecycle.
|
||||
- ℂ **LogEntry** (`Class`)
|
||||
- 📦 **LogLevel** (`Enum`)
|
||||
- 📝 Defines the possible log levels for task logging.
|
||||
- ℂ **LogEntry** (`Class`) `[CRITICAL]`
|
||||
- 📝 A Pydantic model representing a single, structured log entry associated with a task.
|
||||
- 🔒 Invariant: Each log entry has a unique timestamp and source.
|
||||
- ℂ **TaskLog** (`Class`)
|
||||
- 📝 A Pydantic model representing a persisted log entry from the database.
|
||||
- ℂ **LogFilter** (`Class`)
|
||||
- 📝 Filter parameters for querying task logs.
|
||||
- ℂ **LogStats** (`Class`)
|
||||
- 📝 Statistics about log entries for a task.
|
||||
- ℂ **Task** (`Class`)
|
||||
- 📝 A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initializes the Task model and validates input_request for AWAITING_INPUT status.
|
||||
- 📦 **TaskCleanupModule** (`Module`)
|
||||
- 📝 Implements task cleanup and retention policies.
|
||||
- 📝 Implements task cleanup and retention policies, including associated logs.
|
||||
- 🏗️ Layer: Core
|
||||
- ℂ **TaskCleanupService** (`Class`)
|
||||
- 📝 Provides methods to clean up old task records.
|
||||
- 📝 Provides methods to clean up old task records and their associated logs.
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initializes the cleanup service with dependencies.
|
||||
- ƒ **run_cleanup** (`Function`)
|
||||
- 📝 Deletes tasks older than the configured retention period.
|
||||
- 📦 **TaskManagerPackage** (`Module`)
|
||||
- 📝 Deletes tasks older than the configured retention period and their logs.
|
||||
- ƒ **delete_task_with_logs** (`Function`)
|
||||
- 📝 Delete a single task and all its associated logs.
|
||||
- 📦 **TaskManagerPackage** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Exports the public API of the task manager package.
|
||||
- 🏗️ Layer: Core
|
||||
- 📦 **TaskContextModule** (`Module`) `[CRITICAL]`
|
||||
- 📝 Provides execution context passed to plugins during task execution.
|
||||
- 🏗️ Layer: Core
|
||||
- 🔒 Invariant: Each TaskContext is bound to a single task execution.
|
||||
- 🔗 DEPENDS_ON -> `TaskLogger, USED_BY -> plugins`
|
||||
- ℂ **TaskContext** (`Class`) `[CRITICAL]`
|
||||
- 📝 A container passed to plugin.execute() providing the logger and other task-specific utilities.
|
||||
- 🔒 Invariant: logger is always a valid TaskLogger instance.
|
||||
- ƒ **__init__** (`Function`)
|
||||
- 📝 Initialize the TaskContext with task-specific resources.
|
||||
- ƒ **task_id** (`Function`)
|
||||
- 📝 Get the task ID.
|
||||
- ƒ **logger** (`Function`)
|
||||
- 📝 Get the TaskLogger instance for this context.
|
||||
- ƒ **params** (`Function`)
|
||||
- 📝 Get the task parameters.
|
||||
- ƒ **get_param** (`Function`)
|
||||
- 📝 Get a specific parameter value with optional default.
|
||||
- ƒ **create_sub_context** (`Function`)
|
||||
- 📝 Create a sub-context with a different default source.
|
||||
- ƒ **execute** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **backend.src.api.auth** (`Module`)
|
||||
- 📝 Authentication API endpoints.
|
||||
- 🏗️ Layer: API
|
||||
@@ -1504,6 +1655,8 @@
|
||||
- 🔒 Invariant: All settings changes must be persisted via ConfigManager.
|
||||
- 🔗 DEPENDS_ON -> `ConfigManager`
|
||||
- 🔗 DEPENDS_ON -> `ConfigModels`
|
||||
- ℂ **LoggingConfigResponse** (`Class`)
|
||||
- 📝 Response model for logging configuration with current task log level.
|
||||
- ƒ **get_settings** (`Function`)
|
||||
- 📝 Retrieves all application settings.
|
||||
- ƒ **update_global_settings** (`Function`)
|
||||
@@ -1522,6 +1675,10 @@
|
||||
- 📝 Deletes a Superset environment.
|
||||
- ƒ **test_environment_connection** (`Function`)
|
||||
- 📝 Tests the connection to a Superset environment.
|
||||
- ƒ **get_logging_config** (`Function`)
|
||||
- 📝 Retrieves current logging configuration.
|
||||
- ƒ **update_logging_config** (`Function`)
|
||||
- 📝 Updates logging configuration.
|
||||
- 📦 **backend.src.api.routes.admin** (`Module`)
|
||||
- 📝 Admin API endpoints for user and role management.
|
||||
- 🏗️ Layer: API
|
||||
@@ -1612,8 +1769,12 @@
|
||||
- 📝 Retrieve a list of tasks with pagination and optional status filter.
|
||||
- ƒ **get_task** (`Function`)
|
||||
- 📝 Retrieve the details of a specific task.
|
||||
- ƒ **get_task_logs** (`Function`)
|
||||
- 📝 Retrieve logs for a specific task.
|
||||
- ƒ **get_task_logs** (`Function`) `[CRITICAL]`
|
||||
- 📝 Retrieve logs for a specific task with optional filtering.
|
||||
- ƒ **get_task_log_stats** (`Function`)
|
||||
- 📝 Get statistics about logs for a task (counts by level and source).
|
||||
- ƒ **get_task_log_sources** (`Function`)
|
||||
- 📝 Get unique sources for a task's logs.
|
||||
- ƒ **resolve_task** (`Function`)
|
||||
- 📝 Resolve a task that is awaiting mapping.
|
||||
- ƒ **resume_task** (`Function`)
|
||||
@@ -1629,22 +1790,32 @@
|
||||
- 📝 SQLAlchemy model for dashboard validation history.
|
||||
- ƒ **generate_uuid** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **GitModels** (`Module`)
|
||||
- 📦 **GitModels** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Git-specific SQLAlchemy models for configuration and repository tracking.
|
||||
- 🏗️ Layer: Model
|
||||
- 📦 **backend.src.models.task** (`Module`)
|
||||
- ℂ **GitServerConfig** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Configuration for a Git server connection.
|
||||
- ℂ **GitRepository** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Tracking for a local Git repository linked to a dashboard.
|
||||
- ℂ **DeploymentEnvironment** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Target Superset environments for dashboard deployment.
|
||||
- 📦 **backend.src.models.task** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Defines the database schema for task execution records.
|
||||
- 🏗️ Layer: Domain
|
||||
- 🔒 Invariant: All primary keys are UUID strings.
|
||||
- 🔗 DEPENDS_ON -> `sqlalchemy`
|
||||
- ℂ **TaskRecord** (`Class`)
|
||||
- ℂ **TaskRecord** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Represents a persistent record of a task execution.
|
||||
- 📦 **backend.src.models.connection** (`Module`)
|
||||
- ℂ **TaskLogRecord** (`Class`) `[CRITICAL]`
|
||||
- 📝 Represents a single persistent log entry for a task.
|
||||
- 🔒 Invariant: Each log entry belongs to exactly one task.
|
||||
- 🔗 DEPENDS_ON -> `TaskRecord`
|
||||
- 📦 **backend.src.models.connection** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Defines the database schema for external database connection configurations.
|
||||
- 🏗️ Layer: Domain
|
||||
- 🔒 Invariant: All primary keys are UUID strings.
|
||||
- 🔗 DEPENDS_ON -> `sqlalchemy`
|
||||
- ℂ **ConnectionConfig** (`Class`)
|
||||
- ℂ **ConnectionConfig** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Stores credentials for external databases used for column mapping.
|
||||
- 📦 **backend.src.models.mapping** (`Module`)
|
||||
- 📝 Defines the database schema for environment metadata and database mappings using SQLAlchemy.
|
||||
@@ -1657,14 +1828,17 @@
|
||||
- 📝 Represents a Superset instance environment.
|
||||
- ℂ **DatabaseMapping** (`Class`)
|
||||
- 📝 Represents a mapping between source and target databases.
|
||||
- ℂ **MigrationJob** (`Class`)
|
||||
- ℂ **MigrationJob** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Represents a single migration execution job.
|
||||
- ℂ **FileCategory** (`Class`)
|
||||
- 📝 Enumeration of supported file categories in the storage system.
|
||||
- ℂ **StorageConfig** (`Class`)
|
||||
- 📝 Configuration model for the storage system, defining paths and naming patterns.
|
||||
- ℂ **StoredFile** (`Class`)
|
||||
- 📝 Data model representing metadata for a file stored in the system.
|
||||
- 📦 **backend.src.models.storage** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Data models for the storage system.
|
||||
- 🏗️ Layer: Domain
|
||||
- ℂ **FileCategory** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Enumeration of supported file categories in the storage system.
|
||||
- ℂ **StorageConfig** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Configuration model for the storage system, defining paths and naming patterns.
|
||||
- ℂ **StoredFile** (`Class`) `[TRIVIAL]`
|
||||
- 📝 Data model representing metadata for a file stored in the system.
|
||||
- 📦 **backend.src.models.dashboard** (`Module`)
|
||||
- 📝 Defines data models for dashboard metadata and selection.
|
||||
- 🏗️ Layer: Model
|
||||
@@ -1696,11 +1870,19 @@
|
||||
- 🏗️ Layer: Domain
|
||||
- 🔗 DEPENDS_ON -> `backend.src.core.database`
|
||||
- 🔗 DEPENDS_ON -> `backend.src.models.llm`
|
||||
- ℂ **EncryptionManager** (`Class`)
|
||||
- ℂ **EncryptionManager** (`Class`) `[CRITICAL]`
|
||||
- 📝 Handles encryption and decryption of sensitive data like API keys.
|
||||
- 🔒 Invariant: Uses a secret key from environment or a default one (fallback only for dev).
|
||||
- ƒ **EncryptionManager.__init__** (`Function`)
|
||||
- 📝 Initialize the encryption manager with a Fernet key.
|
||||
- ƒ **EncryptionManager.encrypt** (`Function`)
|
||||
- 📝 Encrypt a plaintext string.
|
||||
- ƒ **EncryptionManager.decrypt** (`Function`)
|
||||
- 📝 Decrypt an encrypted string.
|
||||
- ℂ **LLMProviderService** (`Class`)
|
||||
- 📝 Service to manage LLM provider lifecycle.
|
||||
- ƒ **LLMProviderService.__init__** (`Function`)
|
||||
- 📝 Initialize the service with database session.
|
||||
- ƒ **get_all_providers** (`Function`)
|
||||
- 📝 Returns all configured LLM providers.
|
||||
- ƒ **get_provider** (`Function`)
|
||||
@@ -1804,7 +1986,7 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for backup plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes the dashboard backup logic.
|
||||
- 📝 Executes the dashboard backup logic with TaskContext support.
|
||||
- 📦 **DebugPluginModule** (`Module`)
|
||||
- 📝 Implements a plugin for system diagnostics and debugging Superset API responses.
|
||||
- 🏗️ Layer: Plugins
|
||||
@@ -1823,7 +2005,7 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for the debug plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes the debug logic.
|
||||
- 📝 Executes the debug logic with TaskContext support.
|
||||
- ƒ **_test_db_api** (`Function`)
|
||||
- 📝 Tests database API connectivity for source and target environments.
|
||||
- ƒ **_get_dataset_structure** (`Function`)
|
||||
@@ -1846,7 +2028,7 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for the search plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes the dataset search logic.
|
||||
- 📝 Executes the dataset search logic with TaskContext support.
|
||||
- ƒ **_get_context** (`Function`)
|
||||
- 📝 Extracts a small context around the match for display.
|
||||
- 📦 **MapperPluginModule** (`Module`)
|
||||
@@ -1867,7 +2049,7 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for the mapper plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes the dataset mapping logic.
|
||||
- 📝 Executes the dataset mapping logic with TaskContext support.
|
||||
- 📦 **backend.src.plugins.git_plugin** (`Module`)
|
||||
- 📝 Предоставляет плагин для версионирования и развертывания дашбордов Superset.
|
||||
- 🏗️ Layer: Plugin
|
||||
@@ -1891,7 +2073,7 @@
|
||||
- ƒ **initialize** (`Function`)
|
||||
- 📝 Выполняет начальную настройку плагина.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Основной метод выполнения задач плагина.
|
||||
- 📝 Основной метод выполнения задач плагина с поддержкой TaskContext.
|
||||
- 🔗 CALLS -> `self._handle_sync`
|
||||
- 🔗 CALLS -> `self._handle_deploy`
|
||||
- ƒ **_handle_sync** (`Function`)
|
||||
@@ -1924,18 +2106,18 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for migration plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes the dashboard migration logic.
|
||||
- 📝 Executes the dashboard migration logic with TaskContext support.
|
||||
- 📦 **MigrationPlugin.execute** (`Action`)
|
||||
- 📝 Execute the migration logic with proper task logging.
|
||||
- ƒ **schedule_dashboard_validation** (`Function`)
|
||||
- 📝 Schedules a recurring dashboard validation task.
|
||||
- ƒ **_parse_cron** (`Function`)
|
||||
- 📝 Basic cron parser placeholder.
|
||||
- 📦 **scheduler** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for backend/src/plugins/llm_analysis/scheduler.py
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **job_func** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **_parse_cron** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ℂ **LLMProviderType** (`Class`)
|
||||
- 📝 Enum for supported LLM providers.
|
||||
- ℂ **LLMProviderConfig** (`Class`)
|
||||
@@ -1946,13 +2128,19 @@
|
||||
- 📝 Model for a single issue detected during validation.
|
||||
- ℂ **ValidationResult** (`Class`)
|
||||
- 📝 Model for dashboard validation result.
|
||||
- 📦 **backend.src.plugins.llm_analysis.plugin** (`Module`)
|
||||
- 📝 Implements DashboardValidationPlugin and DocumentationPlugin.
|
||||
- 🏗️ Layer: Domain
|
||||
- ℂ **DashboardValidationPlugin** (`Class`)
|
||||
- 📝 Plugin for automated dashboard health analysis using LLMs.
|
||||
- ℂ **DocumentationPlugin** (`Class`)
|
||||
- 📝 Plugin for automated dataset documentation using LLMs.
|
||||
- ℂ **DashboardValidationPlugin** (`Class`)
|
||||
- 📝 Plugin for automated dashboard health analysis using LLMs.
|
||||
- 🔗 IMPLEMENTS -> `backend.src.core.plugin_base.PluginBase`
|
||||
- ƒ **DashboardValidationPlugin.execute** (`Function`)
|
||||
- 📝 Executes the dashboard validation task with TaskContext support.
|
||||
- ℂ **DocumentationPlugin** (`Class`)
|
||||
- 📝 Plugin for automated dataset documentation using LLMs.
|
||||
- 🔗 IMPLEMENTS -> `backend.src.core.plugin_base.PluginBase`
|
||||
- ƒ **DocumentationPlugin.execute** (`Function`)
|
||||
- 📝 Executes the dataset documentation task with TaskContext support.
|
||||
- 📦 **plugin** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for backend/src/plugins/llm_analysis/plugin.py
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **id** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **name** (`Function`) `[TRIVIAL]`
|
||||
@@ -1977,22 +2165,37 @@
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **execute** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **backend.src.plugins.llm_analysis.service** (`Module`)
|
||||
- 📝 Services for LLM interaction and dashboard screenshots.
|
||||
- 🏗️ Layer: Domain
|
||||
- 🔗 DEPENDS_ON -> `playwright`
|
||||
- 🔗 DEPENDS_ON -> `openai`
|
||||
- 🔗 DEPENDS_ON -> `tenacity`
|
||||
- ℂ **ScreenshotService** (`Class`)
|
||||
- 📝 Handles capturing screenshots of Superset dashboards.
|
||||
- ƒ **capture_dashboard** (`Function`)
|
||||
- 📝 Captures a screenshot of a dashboard using Playwright.
|
||||
- ℂ **LLMClient** (`Class`)
|
||||
- 📝 Wrapper for LLM provider APIs.
|
||||
- ℂ **ScreenshotService** (`Class`)
|
||||
- 📝 Handles capturing screenshots of Superset dashboards.
|
||||
- ƒ **ScreenshotService.__init__** (`Function`)
|
||||
- 📝 Initializes the ScreenshotService with environment configuration.
|
||||
- ƒ **ScreenshotService.capture_dashboard** (`Function`)
|
||||
- 📝 Captures a full-page screenshot of a dashboard using Playwright and CDP.
|
||||
- ℂ **LLMClient** (`Class`)
|
||||
- 📝 Wrapper for LLM provider APIs.
|
||||
- ƒ **LLMClient.__init__** (`Function`)
|
||||
- 📝 Initializes the LLMClient with provider settings.
|
||||
- ƒ **LLMClient.get_json_completion** (`Function`)
|
||||
- 📝 Helper to handle LLM calls with JSON mode and fallback parsing.
|
||||
- ƒ **LLMClient.analyze_dashboard** (`Function`)
|
||||
- 📝 Sends dashboard data (screenshot + logs) to LLM for health analysis.
|
||||
- 📦 **service** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for backend/src/plugins/llm_analysis/service.py
|
||||
- 🏗️ Layer: Unknown
|
||||
- ƒ **__init__** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **capture_dashboard** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **switch_tabs** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **__init__** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **_should_retry** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **get_json_completion** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **analyze_dashboard** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **StoragePlugin** (`Module`)
|
||||
- 📝 Provides core filesystem operations for managing backups and repositories.
|
||||
- 🏗️ Layer: App
|
||||
@@ -2016,7 +2219,7 @@
|
||||
- ƒ **get_schema** (`Function`)
|
||||
- 📝 Returns the JSON schema for storage plugin parameters.
|
||||
- ƒ **execute** (`Function`)
|
||||
- 📝 Executes storage-related tasks (placeholder for PluginBase compliance).
|
||||
- 📝 Executes storage-related tasks with TaskContext support.
|
||||
- ƒ **get_storage_root** (`Function`)
|
||||
- 📝 Resolves the absolute path to the storage root.
|
||||
- ƒ **resolve_path** (`Function`)
|
||||
@@ -2044,12 +2247,73 @@
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **test_environment_model** (`Function`)
|
||||
- 📝 Tests that Environment model correctly stores values.
|
||||
- ƒ **test_belief_scope_logs_entry_action_exit** (`Function`)
|
||||
- 📝 Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs.
|
||||
- 📦 **test_task_logger** (`Module`)
|
||||
- 📝 Unit tests for TaskLogger and TaskContext.
|
||||
- 🏗️ Layer: Test
|
||||
- ℂ **TestTaskLogger** (`Class`)
|
||||
- 📝 Test suite for TaskLogger.
|
||||
- ƒ **setup_method** (`Function`)
|
||||
- 📝 Setup for each test method.
|
||||
- ƒ **test_init** (`Function`)
|
||||
- 📝 Test TaskLogger initialization.
|
||||
- ƒ **test_with_source** (`Function`)
|
||||
- 📝 Test creating a sub-logger with different source.
|
||||
- ƒ **test_debug** (`Function`)
|
||||
- 📝 Test debug log level.
|
||||
- ƒ **test_info** (`Function`)
|
||||
- 📝 Test info log level.
|
||||
- ƒ **test_warning** (`Function`)
|
||||
- 📝 Test warning log level.
|
||||
- ƒ **test_error** (`Function`)
|
||||
- 📝 Test error log level.
|
||||
- ƒ **test_error_with_metadata** (`Function`)
|
||||
- 📝 Test error logging with metadata.
|
||||
- ƒ **test_progress** (`Function`)
|
||||
- 📝 Test progress logging.
|
||||
- ƒ **test_progress_clamping** (`Function`)
|
||||
- 📝 Test progress value clamping (0-100).
|
||||
- ƒ **test_source_override** (`Function`)
|
||||
- 📝 Test overriding the default source.
|
||||
- ƒ **test_sub_logger_source_independence** (`Function`)
|
||||
- 📝 Test sub-logger independence from parent.
|
||||
- ℂ **TestTaskContext** (`Class`)
|
||||
- 📝 Test suite for TaskContext.
|
||||
- ƒ **setup_method** (`Function`)
|
||||
- 📝 Setup for each test method.
|
||||
- ƒ **test_init** (`Function`)
|
||||
- 📝 Test TaskContext initialization.
|
||||
- ƒ **test_task_id_property** (`Function`)
|
||||
- 📝 Test task_id property.
|
||||
- ƒ **test_logger_property** (`Function`)
|
||||
- 📝 Test logger property.
|
||||
- ƒ **test_params_property** (`Function`)
|
||||
- 📝 Test params property.
|
||||
- ƒ **test_get_param** (`Function`)
|
||||
- 📝 Test getting a specific parameter.
|
||||
- ƒ **test_create_sub_context** (`Function`)
|
||||
- 📝 Test creating a sub-context with different source.
|
||||
- ƒ **test_context_logger_delegates_to_task_logger** (`Function`)
|
||||
- 📝 Test context logger delegates to TaskLogger.
|
||||
- ƒ **test_sub_context_with_source** (`Function`)
|
||||
- 📝 Test sub-context logger uses new source.
|
||||
- ƒ **test_multiple_sub_contexts** (`Function`)
|
||||
- 📝 Test creating multiple sub-contexts.
|
||||
- ƒ **test_belief_scope_logs_entry_action_exit_at_debug** (`Function`)
|
||||
- 📝 Test that belief_scope generates [ID][Entry], [ID][Action], and [ID][Exit] logs at DEBUG level.
|
||||
- ƒ **test_belief_scope_error_handling** (`Function`)
|
||||
- 📝 Test that belief_scope logs Coherence:Failed on exception.
|
||||
- ƒ **test_belief_scope_success_coherence** (`Function`)
|
||||
- 📝 Test that belief_scope logs Coherence:OK on success.
|
||||
- ƒ **test_belief_scope_not_visible_at_info** (`Function`)
|
||||
- 📝 Test that belief_scope Entry/Exit/Coherence logs are NOT visible at INFO level.
|
||||
- ƒ **test_task_log_level_default** (`Function`)
|
||||
- 📝 Test that default task log level is INFO.
|
||||
- ƒ **test_should_log_task_level** (`Function`)
|
||||
- 📝 Test that should_log_task_level correctly filters log levels.
|
||||
- ƒ **test_configure_logger_task_log_level** (`Function`)
|
||||
- 📝 Test that configure_logger updates task_log_level.
|
||||
- ƒ **test_enable_belief_state_flag** (`Function`)
|
||||
- 📝 Test that enable_belief_state flag controls belief_scope logging.
|
||||
- 📦 **test_auth** (`Module`) `[TRIVIAL]`
|
||||
- 📝 Auto-generated module for backend/tests/test_auth.py
|
||||
- 🏗️ Layer: Unknown
|
||||
@@ -2071,3 +2335,34 @@
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- ƒ **test_ad_group_mapping** (`Function`) `[TRIVIAL]`
|
||||
- 📝 Auto-detected function (orphan)
|
||||
- 📦 **test_log_persistence** (`Module`)
|
||||
- 📝 Unit tests for TaskLogPersistenceService.
|
||||
- 🏗️ Layer: Test
|
||||
- ℂ **TestLogPersistence** (`Class`)
|
||||
- 📝 Test suite for TaskLogPersistenceService.
|
||||
- ƒ **setup_class** (`Function`)
|
||||
- 📝 Setup test database and service instance.
|
||||
- ƒ **teardown_class** (`Function`)
|
||||
- 📝 Clean up test database.
|
||||
- ƒ **setup_method** (`Function`)
|
||||
- 📝 Setup for each test method.
|
||||
- ƒ **teardown_method** (`Function`)
|
||||
- 📝 Cleanup after each test method.
|
||||
- ƒ **test_add_log_single** (`Function`)
|
||||
- 📝 Test adding a single log entry.
|
||||
- ƒ **test_add_log_batch** (`Function`)
|
||||
- 📝 Test adding multiple log entries in batch.
|
||||
- ƒ **test_get_logs_by_task_id** (`Function`)
|
||||
- 📝 Test retrieving logs by task ID.
|
||||
- ƒ **test_get_logs_with_filters** (`Function`)
|
||||
- 📝 Test retrieving logs with level and source filters.
|
||||
- ƒ **test_get_logs_with_pagination** (`Function`)
|
||||
- 📝 Test retrieving logs with pagination.
|
||||
- ƒ **test_get_logs_with_search** (`Function`)
|
||||
- 📝 Test retrieving logs with search query.
|
||||
- ƒ **test_get_log_stats** (`Function`)
|
||||
- 📝 Test retrieving log statistics.
|
||||
- ƒ **test_get_log_sources** (`Function`)
|
||||
- 📝 Test retrieving unique log sources.
|
||||
- ƒ **test_delete_logs_by_task_id** (`Function`)
|
||||
- 📝 Test deleting logs by task ID.
|
||||
|
||||
Reference in New Issue
Block a user