semantic update
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.core.auth.jwt:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: jwt, token, session, auth
|
||||
# @PURPOSE: JWT token generation and validation logic.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
# [DEF:backend.src.core.auth.logger:Module]
|
||||
#
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: auth, logger, audit, security
|
||||
# @PURPOSE: Audit logging for security-related events.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:ConfigModels:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: config, models, pydantic
|
||||
# @PURPOSE: Defines the data models for application configuration using Pydantic.
|
||||
# @LAYER: Core
|
||||
@@ -34,6 +35,7 @@ class Environment(BaseModel):
|
||||
# @PURPOSE: Defines the configuration for the application's logging system.
|
||||
class LoggingConfig(BaseModel):
|
||||
level: str = "INFO"
|
||||
task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR)
|
||||
file_path: Optional[str] = "logs/app.log"
|
||||
max_bytes: int = 10 * 1024 * 1024
|
||||
backup_count: int = 5
|
||||
|
||||
@@ -19,6 +19,9 @@ _belief_state = threading.local()
|
||||
# Global flag for belief state logging
|
||||
_enable_belief_state = True
|
||||
|
||||
# Global task log level filter
|
||||
_task_log_level = "INFO"
|
||||
|
||||
# [DEF:BeliefFormatter:Class]
|
||||
# @PURPOSE: Custom logging formatter that adds belief state prefixes to log messages.
|
||||
class BeliefFormatter(logging.Formatter):
|
||||
@@ -58,12 +61,12 @@ class LogEntry(BaseModel):
|
||||
# @SEMANTICS: logging, context, belief_state
|
||||
@contextmanager
|
||||
def belief_scope(anchor_id: str, message: str = ""):
|
||||
# Log Entry if enabled
|
||||
# Log Entry if enabled (DEBUG level to reduce noise)
|
||||
if _enable_belief_state:
|
||||
entry_msg = f"[{anchor_id}][Entry]"
|
||||
if message:
|
||||
entry_msg += f" {message}"
|
||||
logger.info(entry_msg)
|
||||
logger.debug(entry_msg)
|
||||
|
||||
# Set thread-local anchor_id
|
||||
old_anchor = getattr(_belief_state, 'anchor_id', None)
|
||||
@@ -71,13 +74,13 @@ def belief_scope(anchor_id: str, message: str = ""):
|
||||
|
||||
try:
|
||||
yield
|
||||
# Log Coherence OK and Exit
|
||||
logger.info(f"[{anchor_id}][Coherence:OK]")
|
||||
# Log Coherence OK and Exit (DEBUG level to reduce noise)
|
||||
logger.debug(f"[{anchor_id}][Coherence:OK]")
|
||||
if _enable_belief_state:
|
||||
logger.info(f"[{anchor_id}][Exit]")
|
||||
logger.debug(f"[{anchor_id}][Exit]")
|
||||
except Exception as e:
|
||||
# Log Coherence Failed
|
||||
logger.info(f"[{anchor_id}][Coherence:Failed] {str(e)}")
|
||||
# Log Coherence Failed (DEBUG level to reduce noise)
|
||||
logger.debug(f"[{anchor_id}][Coherence:Failed] {str(e)}")
|
||||
raise
|
||||
finally:
|
||||
# Restore old anchor
|
||||
@@ -88,12 +91,13 @@ def belief_scope(anchor_id: str, message: str = ""):
|
||||
# [DEF:configure_logger:Function]
|
||||
# @PURPOSE: Configures the logger with the provided logging settings.
|
||||
# @PRE: config is a valid LoggingConfig instance.
|
||||
# @POST: Logger level, handlers, and belief state flag are updated.
|
||||
# @POST: Logger level, handlers, belief state flag, and task log level are updated.
|
||||
# @PARAM: config (LoggingConfig) - The logging configuration.
|
||||
# @SEMANTICS: logging, configuration, initialization
|
||||
def configure_logger(config):
|
||||
global _enable_belief_state
|
||||
global _enable_belief_state, _task_log_level
|
||||
_enable_belief_state = config.enable_belief_state
|
||||
_task_log_level = config.task_log_level.upper()
|
||||
|
||||
# Set logger level
|
||||
level = getattr(logging, config.level.upper(), logging.INFO)
|
||||
@@ -130,6 +134,36 @@ def configure_logger(config):
|
||||
))
|
||||
# [/DEF:configure_logger:Function]
|
||||
|
||||
# [DEF:get_task_log_level:Function]
|
||||
# @PURPOSE: Returns the current task log level filter.
|
||||
# @PRE: None.
|
||||
# @POST: Returns the task log level string.
|
||||
# @RETURN: str - The current task log level (DEBUG, INFO, WARNING, ERROR).
|
||||
# @SEMANTICS: logging, configuration, getter
|
||||
def get_task_log_level() -> str:
|
||||
"""Returns the current task log level filter."""
|
||||
return _task_log_level
|
||||
# [/DEF:get_task_log_level:Function]
|
||||
|
||||
# [DEF:should_log_task_level:Function]
|
||||
# @PURPOSE: Checks if a log level should be recorded based on task_log_level setting.
|
||||
# @PRE: level is a valid log level string.
|
||||
# @POST: Returns True if level meets or exceeds task_log_level threshold.
|
||||
# @PARAM: level (str) - The log level to check.
|
||||
# @RETURN: bool - True if the level should be logged.
|
||||
# @SEMANTICS: logging, filter, level
|
||||
def should_log_task_level(level: str) -> bool:
|
||||
"""Checks if a log level should be recorded based on task_log_level setting."""
|
||||
level_order = {"DEBUG": 0, "INFO": 1, "WARNING": 2, "ERROR": 3}
|
||||
current_level = _task_log_level.upper()
|
||||
check_level = level.upper()
|
||||
|
||||
current_order = level_order.get(current_level, 1) # Default to INFO
|
||||
check_order = level_order.get(check_level, 1)
|
||||
|
||||
return check_order >= current_order
|
||||
# [/DEF:should_log_task_level:Function]
|
||||
|
||||
# [DEF:WebSocketLogHandler:Class]
|
||||
# @SEMANTICS: logging, handler, websocket, buffer
|
||||
# @PURPOSE: A custom logging handler that captures log records into a buffer. It is designed to be extended for real-time log streaming over WebSockets.
|
||||
|
||||
@@ -7,6 +7,7 @@ from jsonschema import validate
|
||||
from .logger import belief_scope
|
||||
|
||||
# [DEF:PluginLoader:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: plugin, loader, dynamic, import
|
||||
# @PURPOSE: Scans a specified directory for Python modules, dynamically loads them, and registers any classes that are valid implementations of the PluginBase interface.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:SchedulerModule:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: scheduler, apscheduler, cron, backup
|
||||
# @PURPOSE: Manages scheduled tasks using APScheduler.
|
||||
# @LAYER: Core
|
||||
@@ -14,6 +15,7 @@ import asyncio
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:SchedulerService:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: scheduler, service, apscheduler
|
||||
# @PURPOSE: Provides a service to manage scheduled backup tasks.
|
||||
class SchedulerService:
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:TaskManagerPackage:Module]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: task, manager, package, exports
|
||||
# @PURPOSE: Exports the public API of the task manager package.
|
||||
# @LAYER: Core
|
||||
|
||||
@@ -1,47 +1,76 @@
|
||||
# [DEF:TaskCleanupModule:Module]
|
||||
# @SEMANTICS: task, cleanup, retention
|
||||
# @PURPOSE: Implements task cleanup and retention policies.
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, cleanup, retention, logs
|
||||
# @PURPOSE: Implements task cleanup and retention policies, including associated logs.
|
||||
# @LAYER: Core
|
||||
# @RELATION: Uses TaskPersistenceService to delete old tasks.
|
||||
# @RELATION: Uses TaskPersistenceService and TaskLogPersistenceService to delete old tasks and logs.
|
||||
|
||||
from datetime import datetime, timedelta
|
||||
from .persistence import TaskPersistenceService
|
||||
from typing import List
|
||||
from .persistence import TaskPersistenceService, TaskLogPersistenceService
|
||||
from ..logger import logger, belief_scope
|
||||
from ..config_manager import ConfigManager
|
||||
|
||||
# [DEF:TaskCleanupService:Class]
|
||||
# @PURPOSE: Provides methods to clean up old task records.
|
||||
# @PURPOSE: Provides methods to clean up old task records and their associated logs.
|
||||
# @TIER: STANDARD
|
||||
class TaskCleanupService:
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initializes the cleanup service with dependencies.
|
||||
# @PRE: persistence_service and config_manager are valid.
|
||||
# @POST: Cleanup service is ready.
|
||||
def __init__(self, persistence_service: TaskPersistenceService, config_manager: ConfigManager):
|
||||
def __init__(
|
||||
self,
|
||||
persistence_service: TaskPersistenceService,
|
||||
log_persistence_service: TaskLogPersistenceService,
|
||||
config_manager: ConfigManager
|
||||
):
|
||||
self.persistence_service = persistence_service
|
||||
self.log_persistence_service = log_persistence_service
|
||||
self.config_manager = config_manager
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:run_cleanup:Function]
|
||||
# @PURPOSE: Deletes tasks older than the configured retention period.
|
||||
# @PURPOSE: Deletes tasks older than the configured retention period and their logs.
|
||||
# @PRE: Config manager has valid settings.
|
||||
# @POST: Old tasks are deleted from persistence.
|
||||
# @POST: Old tasks and their logs are deleted from persistence.
|
||||
def run_cleanup(self):
|
||||
with belief_scope("TaskCleanupService.run_cleanup"):
|
||||
settings = self.config_manager.get_config().settings
|
||||
retention_days = settings.task_retention_days
|
||||
|
||||
# This is a simplified implementation.
|
||||
# In a real scenario, we would query IDs of tasks older than retention_days.
|
||||
# For now, we'll log the action.
|
||||
logger.info(f"Cleaning up tasks older than {retention_days} days.")
|
||||
|
||||
# Re-loading tasks to check for limit
|
||||
# Load tasks to check for limit
|
||||
tasks = self.persistence_service.load_tasks(limit=1000)
|
||||
if len(tasks) > settings.task_retention_limit:
|
||||
to_delete = [t.id for t in tasks[settings.task_retention_limit:]]
|
||||
to_delete: List[str] = [t.id for t in tasks[settings.task_retention_limit:]]
|
||||
|
||||
# Delete logs first (before task records)
|
||||
self.log_persistence_service.delete_logs_for_tasks(to_delete)
|
||||
|
||||
# Then delete task records
|
||||
self.persistence_service.delete_tasks(to_delete)
|
||||
logger.info(f"Deleted {len(to_delete)} tasks exceeding limit of {settings.task_retention_limit}")
|
||||
|
||||
logger.info(f"Deleted {len(to_delete)} tasks and their logs exceeding limit of {settings.task_retention_limit}")
|
||||
# [/DEF:run_cleanup:Function]
|
||||
|
||||
# [DEF:delete_task_with_logs:Function]
|
||||
# @PURPOSE: Delete a single task and all its associated logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Task and all its logs are deleted.
|
||||
# @PARAM: task_id (str) - The task ID to delete.
|
||||
def delete_task_with_logs(self, task_id: str) -> None:
|
||||
"""Delete a single task and all its associated logs."""
|
||||
with belief_scope("TaskCleanupService.delete_task_with_logs", f"task_id={task_id}"):
|
||||
# Delete logs first
|
||||
self.log_persistence_service.delete_logs_for_task(task_id)
|
||||
|
||||
# Then delete task record
|
||||
self.persistence_service.delete_tasks([task_id])
|
||||
|
||||
logger.info(f"Deleted task {task_id} and its associated logs")
|
||||
# [/DEF:delete_task_with_logs:Function]
|
||||
|
||||
# [/DEF:TaskCleanupService:Class]
|
||||
# [/DEF:TaskCleanupModule:Module]
|
||||
115
backend/src/core/task_manager/context.py
Normal file
115
backend/src/core/task_manager/context.py
Normal file
@@ -0,0 +1,115 @@
|
||||
# [DEF:TaskContextModule:Module]
|
||||
# @SEMANTICS: task, context, plugin, execution, logger
|
||||
# @PURPOSE: Provides execution context passed to plugins during task execution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: DEPENDS_ON -> TaskLogger, USED_BY -> plugins
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each TaskContext is bound to a single task execution.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from .task_logger import TaskLogger
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskContext:Class]
|
||||
# @SEMANTICS: context, task, execution, plugin
|
||||
# @PURPOSE: A container passed to plugin.execute() providing the logger and other task-specific utilities.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: logger is always a valid TaskLogger instance.
|
||||
# @UX_STATE: Idle -> Active -> Complete
|
||||
class TaskContext:
|
||||
"""
|
||||
Execution context provided to plugins during task execution.
|
||||
|
||||
Usage:
|
||||
def execute(params: dict, context: TaskContext = None):
|
||||
if context:
|
||||
context.logger.info("Starting process")
|
||||
context.logger.progress("Processing items", percent=50)
|
||||
# ... plugin logic
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskContext with task-specific resources.
|
||||
# @PRE: task_id is a valid task identifier, add_log_fn is callable.
|
||||
# @POST: TaskContext is ready to be passed to plugin.execute().
|
||||
# @PARAM: task_id (str) - The ID of the task.
|
||||
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
|
||||
# @PARAM: params (Dict) - Task parameters.
|
||||
# @PARAM: default_source (str) - Default source for logs (default: "plugin").
|
||||
def __init__(
|
||||
self,
|
||||
task_id: str,
|
||||
add_log_fn: Callable,
|
||||
params: Dict[str, Any],
|
||||
default_source: str = "plugin"
|
||||
):
|
||||
self._task_id = task_id
|
||||
self._params = params
|
||||
self._logger = TaskLogger(
|
||||
task_id=task_id,
|
||||
add_log_fn=add_log_fn,
|
||||
source=default_source
|
||||
)
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:task_id:Function]
|
||||
# @PURPOSE: Get the task ID.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the task ID string.
|
||||
# @RETURN: str - The task ID.
|
||||
@property
|
||||
def task_id(self) -> str:
|
||||
return self._task_id
|
||||
# [/DEF:task_id:Function]
|
||||
|
||||
# [DEF:logger:Function]
|
||||
# @PURPOSE: Get the TaskLogger instance for this context.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the TaskLogger instance.
|
||||
# @RETURN: TaskLogger - The logger instance.
|
||||
@property
|
||||
def logger(self) -> TaskLogger:
|
||||
return self._logger
|
||||
# [/DEF:logger:Function]
|
||||
|
||||
# [DEF:params:Function]
|
||||
# @PURPOSE: Get the task parameters.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns the parameters dictionary.
|
||||
# @RETURN: Dict[str, Any] - The task parameters.
|
||||
@property
|
||||
def params(self) -> Dict[str, Any]:
|
||||
return self._params
|
||||
# [/DEF:params:Function]
|
||||
|
||||
# [DEF:get_param:Function]
|
||||
# @PURPOSE: Get a specific parameter value with optional default.
|
||||
# @PRE: TaskContext must be initialized.
|
||||
# @POST: Returns parameter value or default.
|
||||
# @PARAM: key (str) - Parameter key.
|
||||
# @PARAM: default (Any) - Default value if key not found.
|
||||
# @RETURN: Any - Parameter value or default.
|
||||
def get_param(self, key: str, default: Any = None) -> Any:
|
||||
return self._params.get(key, default)
|
||||
# [/DEF:get_param:Function]
|
||||
|
||||
# [DEF:create_sub_context:Function]
|
||||
# @PURPOSE: Create a sub-context with a different default source.
|
||||
# @PRE: source is a non-empty string.
|
||||
# @POST: Returns new TaskContext with different logger source.
|
||||
# @PARAM: source (str) - New default source for logging.
|
||||
# @RETURN: TaskContext - New context with different source.
|
||||
def create_sub_context(self, source: str) -> "TaskContext":
|
||||
"""Create a sub-context with a different default source for logging."""
|
||||
return TaskContext(
|
||||
task_id=self._task_id,
|
||||
add_log_fn=self._logger._add_log,
|
||||
params=self._params,
|
||||
default_source=source
|
||||
)
|
||||
# [/DEF:create_sub_context:Function]
|
||||
|
||||
# [/DEF:TaskContext:Class]
|
||||
|
||||
# [/DEF:TaskContextModule:Module]
|
||||
@@ -8,23 +8,33 @@
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
import asyncio
|
||||
import threading
|
||||
import inspect
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime
|
||||
from typing import Dict, Any, List, Optional
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
|
||||
from .models import Task, TaskStatus, LogEntry
|
||||
from .persistence import TaskPersistenceService
|
||||
from ..logger import logger, belief_scope
|
||||
from .models import Task, TaskStatus, LogEntry, LogFilter, LogStats, TaskLog
|
||||
from .persistence import TaskPersistenceService, TaskLogPersistenceService
|
||||
from .context import TaskContext
|
||||
from ..logger import logger, belief_scope, should_log_task_level
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskManager:Class]
|
||||
# @SEMANTICS: task, manager, lifecycle, execution, state
|
||||
# @PURPOSE: Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Task IDs are unique within the registry.
|
||||
# @INVARIANT: Each task has exactly one status at any time.
|
||||
# @INVARIANT: Log entries are never deleted after being added to a task.
|
||||
class TaskManager:
|
||||
"""
|
||||
Manages the lifecycle of tasks, including their creation, execution, and state tracking.
|
||||
"""
|
||||
|
||||
# Log flush interval in seconds
|
||||
LOG_FLUSH_INTERVAL = 2.0
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskManager with dependencies.
|
||||
# @PRE: plugin_loader is initialized.
|
||||
@@ -35,8 +45,18 @@ class TaskManager:
|
||||
self.plugin_loader = plugin_loader
|
||||
self.tasks: Dict[str, Task] = {}
|
||||
self.subscribers: Dict[str, List[asyncio.Queue]] = {}
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.executor = ThreadPoolExecutor(max_workers=5) # For CPU-bound plugin execution
|
||||
self.persistence_service = TaskPersistenceService()
|
||||
self.log_persistence_service = TaskLogPersistenceService()
|
||||
|
||||
# Log buffer: task_id -> List[LogEntry]
|
||||
self._log_buffer: Dict[str, List[LogEntry]] = {}
|
||||
self._log_buffer_lock = threading.Lock()
|
||||
|
||||
# Flusher thread for batch writing logs
|
||||
self._flusher_stop_event = threading.Event()
|
||||
self._flusher_thread = threading.Thread(target=self._flusher_loop, daemon=True)
|
||||
self._flusher_thread.start()
|
||||
|
||||
try:
|
||||
self.loop = asyncio.get_running_loop()
|
||||
@@ -47,6 +67,59 @@ class TaskManager:
|
||||
# Load persisted tasks on startup
|
||||
self.load_persisted_tasks()
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:_flusher_loop:Function]
|
||||
# @PURPOSE: Background thread that periodically flushes log buffer to database.
|
||||
# @PRE: TaskManager is initialized.
|
||||
# @POST: Logs are batch-written to database every LOG_FLUSH_INTERVAL seconds.
|
||||
def _flusher_loop(self):
|
||||
"""Background thread that flushes log buffer to database."""
|
||||
while not self._flusher_stop_event.is_set():
|
||||
self._flush_logs()
|
||||
self._flusher_stop_event.wait(self.LOG_FLUSH_INTERVAL)
|
||||
# [/DEF:_flusher_loop:Function]
|
||||
|
||||
# [DEF:_flush_logs:Function]
|
||||
# @PURPOSE: Flush all buffered logs to the database.
|
||||
# @PRE: None.
|
||||
# @POST: All buffered logs are written to task_logs table.
|
||||
def _flush_logs(self):
|
||||
"""Flush all buffered logs to the database."""
|
||||
with self._log_buffer_lock:
|
||||
task_ids = list(self._log_buffer.keys())
|
||||
|
||||
for task_id in task_ids:
|
||||
with self._log_buffer_lock:
|
||||
logs = self._log_buffer.pop(task_id, [])
|
||||
|
||||
if logs:
|
||||
try:
|
||||
self.log_persistence_service.add_logs(task_id, logs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to flush logs for task {task_id}: {e}")
|
||||
# Re-add logs to buffer on failure
|
||||
with self._log_buffer_lock:
|
||||
if task_id not in self._log_buffer:
|
||||
self._log_buffer[task_id] = []
|
||||
self._log_buffer[task_id].extend(logs)
|
||||
# [/DEF:_flush_logs:Function]
|
||||
|
||||
# [DEF:_flush_task_logs:Function]
|
||||
# @PURPOSE: Flush logs for a specific task immediately.
|
||||
# @PRE: task_id exists.
|
||||
# @POST: Task's buffered logs are written to database.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
def _flush_task_logs(self, task_id: str):
|
||||
"""Flush logs for a specific task immediately."""
|
||||
with self._log_buffer_lock:
|
||||
logs = self._log_buffer.pop(task_id, [])
|
||||
|
||||
if logs:
|
||||
try:
|
||||
self.log_persistence_service.add_logs(task_id, logs)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to flush logs for task {task_id}: {e}")
|
||||
# [/DEF:_flush_task_logs:Function]
|
||||
|
||||
# [DEF:create_task:Function]
|
||||
# @PURPOSE: Creates and queues a new task for execution.
|
||||
@@ -78,7 +151,7 @@ class TaskManager:
|
||||
# [/DEF:create_task:Function]
|
||||
|
||||
# [DEF:_run_task:Function]
|
||||
# @PURPOSE: Internal method to execute a task.
|
||||
# @PURPOSE: Internal method to execute a task with TaskContext support.
|
||||
# @PRE: Task exists in registry.
|
||||
# @POST: Task is executed, status updated to SUCCESS or FAILED.
|
||||
# @PARAM: task_id (str) - The ID of the task to run.
|
||||
@@ -91,30 +164,54 @@ class TaskManager:
|
||||
task.status = TaskStatus.RUNNING
|
||||
task.started_at = datetime.utcnow()
|
||||
self.persistence_service.persist_task(task)
|
||||
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'")
|
||||
self._add_log(task_id, "INFO", f"Task started for plugin '{plugin.name}'", source="system")
|
||||
|
||||
try:
|
||||
# Execute plugin
|
||||
# Prepare params and check if plugin supports new TaskContext
|
||||
params = {**task.params, "_task_id": task_id}
|
||||
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
plugin.execute,
|
||||
params
|
||||
# Check if plugin's execute method accepts 'context' parameter
|
||||
sig = inspect.signature(plugin.execute)
|
||||
accepts_context = 'context' in sig.parameters
|
||||
|
||||
if accepts_context:
|
||||
# Create TaskContext for new-style plugins
|
||||
context = TaskContext(
|
||||
task_id=task_id,
|
||||
add_log_fn=self._add_log,
|
||||
params=params,
|
||||
default_source="plugin"
|
||||
)
|
||||
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params, context=context)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
lambda: plugin.execute(params, context=context)
|
||||
)
|
||||
else:
|
||||
# Backward compatibility: old-style plugins without context
|
||||
if asyncio.iscoroutinefunction(plugin.execute):
|
||||
task.result = await plugin.execute(params)
|
||||
else:
|
||||
task.result = await self.loop.run_in_executor(
|
||||
self.executor,
|
||||
plugin.execute,
|
||||
params
|
||||
)
|
||||
|
||||
logger.info(f"Task {task_id} completed successfully")
|
||||
task.status = TaskStatus.SUCCESS
|
||||
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'")
|
||||
self._add_log(task_id, "INFO", f"Task completed successfully for plugin '{plugin.name}'", source="system")
|
||||
except Exception as e:
|
||||
logger.error(f"Task {task_id} failed: {e}")
|
||||
task.status = TaskStatus.FAILED
|
||||
self._add_log(task_id, "ERROR", f"Task failed: {e}", {"error_type": type(e).__name__})
|
||||
self._add_log(task_id, "ERROR", f"Task failed: {e}", source="system", metadata={"error_type": type(e).__name__})
|
||||
finally:
|
||||
task.finished_at = datetime.utcnow()
|
||||
# Flush any remaining buffered logs before persisting task
|
||||
self._flush_task_logs(task_id)
|
||||
self.persistence_service.persist_task(task)
|
||||
logger.info(f"Task {task_id} execution finished with status: {task.status}")
|
||||
# [/DEF:_run_task:Function]
|
||||
@@ -224,36 +321,106 @@ class TaskManager:
|
||||
# [/DEF:get_tasks:Function]
|
||||
|
||||
# [DEF:get_task_logs:Function]
|
||||
# @PURPOSE: Retrieves logs for a specific task.
|
||||
# @PURPOSE: Retrieves logs for a specific task (from memory for running, persistence for completed).
|
||||
# @PRE: task_id is a string.
|
||||
# @POST: Returns list of LogEntry objects.
|
||||
# @POST: Returns list of LogEntry or TaskLog objects.
|
||||
# @PARAM: task_id (str) - ID of the task.
|
||||
# @PARAM: log_filter (Optional[LogFilter]) - Filter parameters.
|
||||
# @RETURN: List[LogEntry] - List of log entries.
|
||||
def get_task_logs(self, task_id: str) -> List[LogEntry]:
|
||||
def get_task_logs(self, task_id: str, log_filter: Optional[LogFilter] = None) -> List[LogEntry]:
|
||||
with belief_scope("TaskManager.get_task_logs", f"task_id={task_id}"):
|
||||
task = self.tasks.get(task_id)
|
||||
|
||||
# For completed tasks, fetch from persistence
|
||||
if task and task.status in [TaskStatus.SUCCESS, TaskStatus.FAILED]:
|
||||
if log_filter is None:
|
||||
log_filter = LogFilter()
|
||||
task_logs = self.log_persistence_service.get_logs(task_id, log_filter)
|
||||
# Convert TaskLog to LogEntry for backward compatibility
|
||||
return [
|
||||
LogEntry(
|
||||
timestamp=log.timestamp,
|
||||
level=log.level,
|
||||
message=log.message,
|
||||
source=log.source,
|
||||
metadata=log.metadata
|
||||
)
|
||||
for log in task_logs
|
||||
]
|
||||
|
||||
# For running/pending tasks, return from memory
|
||||
return task.logs if task else []
|
||||
# [/DEF:get_task_logs:Function]
|
||||
|
||||
# [DEF:get_task_log_stats:Function]
|
||||
# @PURPOSE: Get statistics about logs for a task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns LogStats with counts by level and source.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: LogStats - Statistics about task logs.
|
||||
def get_task_log_stats(self, task_id: str) -> LogStats:
|
||||
with belief_scope("TaskManager.get_task_log_stats", f"task_id={task_id}"):
|
||||
return self.log_persistence_service.get_log_stats(task_id)
|
||||
# [/DEF:get_task_log_stats:Function]
|
||||
|
||||
# [DEF:get_task_log_sources:Function]
|
||||
# @PURPOSE: Get unique sources for a task's logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of unique source strings.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: List[str] - Unique source names.
|
||||
def get_task_log_sources(self, task_id: str) -> List[str]:
|
||||
with belief_scope("TaskManager.get_task_log_sources", f"task_id={task_id}"):
|
||||
return self.log_persistence_service.get_sources(task_id)
|
||||
# [/DEF:get_task_log_sources:Function]
|
||||
|
||||
# [DEF:_add_log:Function]
|
||||
# @PURPOSE: Adds a log entry to a task and notifies subscribers.
|
||||
# @PURPOSE: Adds a log entry to a task buffer and notifies subscribers.
|
||||
# @PRE: Task exists.
|
||||
# @POST: Log added to task and pushed to queues.
|
||||
# @POST: Log added to buffer and pushed to queues (if level meets task_log_level filter).
|
||||
# @PARAM: task_id (str) - ID of the task.
|
||||
# @PARAM: level (str) - Log level.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: context (Optional[Dict]) - Log context.
|
||||
def _add_log(self, task_id: str, level: str, message: str, context: Optional[Dict[str, Any]] = None):
|
||||
# @PARAM: source (str) - Source component (default: "system").
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
|
||||
# @PARAM: context (Optional[Dict]) - Legacy context (for backward compatibility).
|
||||
def _add_log(
|
||||
self,
|
||||
task_id: str,
|
||||
level: str,
|
||||
message: str,
|
||||
source: str = "system",
|
||||
metadata: Optional[Dict[str, Any]] = None,
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
):
|
||||
with belief_scope("TaskManager._add_log", f"task_id={task_id}"):
|
||||
task = self.tasks.get(task_id)
|
||||
if not task:
|
||||
return
|
||||
|
||||
log_entry = LogEntry(level=level, message=message, context=context)
|
||||
task.logs.append(log_entry)
|
||||
self.persistence_service.persist_task(task)
|
||||
# Filter logs based on task_log_level configuration
|
||||
if not should_log_task_level(level):
|
||||
return
|
||||
|
||||
# Notify subscribers
|
||||
# Create log entry with new fields
|
||||
log_entry = LogEntry(
|
||||
level=level,
|
||||
message=message,
|
||||
source=source,
|
||||
metadata=metadata,
|
||||
context=context # Keep for backward compatibility
|
||||
)
|
||||
|
||||
# Add to in-memory logs (for backward compatibility with legacy JSON field)
|
||||
task.logs.append(log_entry)
|
||||
|
||||
# Add to buffer for batch persistence
|
||||
with self._log_buffer_lock:
|
||||
if task_id not in self._log_buffer:
|
||||
self._log_buffer[task_id] = []
|
||||
self._log_buffer[task_id].append(log_entry)
|
||||
|
||||
# Notify subscribers (for real-time WebSocket updates)
|
||||
if task_id in self.subscribers:
|
||||
for queue in self.subscribers[task_id]:
|
||||
self.loop.call_soon_threadsafe(queue.put_nowait, log_entry)
|
||||
@@ -353,7 +520,7 @@ class TaskManager:
|
||||
# [/DEF:resume_task_with_password:Function]
|
||||
|
||||
# [DEF:clear_tasks:Function]
|
||||
# @PURPOSE: Clears tasks based on status filter.
|
||||
# @PURPOSE: Clears tasks based on status filter (also deletes associated logs).
|
||||
# @PRE: status is Optional[TaskStatus].
|
||||
# @POST: Tasks matching filter (or all non-active) cleared from registry and database.
|
||||
# @PARAM: status (Optional[TaskStatus]) - Filter by task status.
|
||||
@@ -387,9 +554,13 @@ class TaskManager:
|
||||
|
||||
del self.tasks[tid]
|
||||
|
||||
# Remove from persistence
|
||||
# Remove from persistence (task_records and task_logs via CASCADE)
|
||||
self.persistence_service.delete_tasks(tasks_to_remove)
|
||||
|
||||
# Also explicitly delete logs (in case CASCADE is not set up)
|
||||
if tasks_to_remove:
|
||||
self.log_persistence_service.delete_logs_for_tasks(tasks_to_remove)
|
||||
|
||||
logger.info(f"Cleared {len(tasks_to_remove)} tasks.")
|
||||
return len(tasks_to_remove)
|
||||
# [/DEF:clear_tasks:Function]
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
# [DEF:TaskManagerModels:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, models, pydantic, enum, state
|
||||
# @PURPOSE: Defines the data models and enumerations used by the Task Manager.
|
||||
# @LAYER: Core
|
||||
@@ -16,6 +17,7 @@ from pydantic import BaseModel, Field
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskStatus:Enum]
|
||||
# @TIER: TRIVIAL
|
||||
# @SEMANTICS: task, status, state, enum
|
||||
# @PURPOSE: Defines the possible states a task can be in during its lifecycle.
|
||||
class TaskStatus(str, Enum):
|
||||
@@ -27,17 +29,73 @@ class TaskStatus(str, Enum):
|
||||
AWAITING_INPUT = "AWAITING_INPUT"
|
||||
# [/DEF:TaskStatus:Enum]
|
||||
|
||||
# [DEF:LogLevel:Enum]
|
||||
# @SEMANTICS: log, level, severity, enum
|
||||
# @PURPOSE: Defines the possible log levels for task logging.
|
||||
# @TIER: STANDARD
|
||||
class LogLevel(str, Enum):
|
||||
DEBUG = "DEBUG"
|
||||
INFO = "INFO"
|
||||
WARNING = "WARNING"
|
||||
ERROR = "ERROR"
|
||||
# [/DEF:LogLevel:Enum]
|
||||
|
||||
# [DEF:LogEntry:Class]
|
||||
# @SEMANTICS: log, entry, record, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single, structured log entry associated with a task.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each log entry has a unique timestamp and source.
|
||||
class LogEntry(BaseModel):
|
||||
timestamp: datetime = Field(default_factory=datetime.utcnow)
|
||||
level: str
|
||||
level: str = Field(default="INFO")
|
||||
message: str
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
source: str = Field(default="system") # Component attribution: plugin, superset_api, git, etc.
|
||||
context: Optional[Dict[str, Any]] = None # Legacy field, kept for backward compatibility
|
||||
metadata: Optional[Dict[str, Any]] = None # Structured metadata (e.g., dashboard_id, progress)
|
||||
# [/DEF:LogEntry:Class]
|
||||
|
||||
# [DEF:TaskLog:Class]
|
||||
# @SEMANTICS: task, log, persistent, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a persisted log entry from the database.
|
||||
# @TIER: STANDARD
|
||||
# @RELATION: MAPS_TO -> TaskLogRecord
|
||||
class TaskLog(BaseModel):
|
||||
id: int
|
||||
task_id: str
|
||||
timestamp: datetime
|
||||
level: str
|
||||
source: str
|
||||
message: str
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
|
||||
class Config:
|
||||
from_attributes = True
|
||||
# [/DEF:TaskLog:Class]
|
||||
|
||||
# [DEF:LogFilter:Class]
|
||||
# @SEMANTICS: log, filter, query, pydantic
|
||||
# @PURPOSE: Filter parameters for querying task logs.
|
||||
# @TIER: STANDARD
|
||||
class LogFilter(BaseModel):
|
||||
level: Optional[str] = None # Filter by log level
|
||||
source: Optional[str] = None # Filter by source component
|
||||
search: Optional[str] = None # Text search in message
|
||||
offset: int = Field(default=0, ge=0)
|
||||
limit: int = Field(default=100, ge=1, le=1000)
|
||||
# [/DEF:LogFilter:Class]
|
||||
|
||||
# [DEF:LogStats:Class]
|
||||
# @SEMANTICS: log, stats, aggregation, pydantic
|
||||
# @PURPOSE: Statistics about log entries for a task.
|
||||
# @TIER: STANDARD
|
||||
class LogStats(BaseModel):
|
||||
total_count: int
|
||||
by_level: Dict[str, int] # {"INFO": 10, "ERROR": 2}
|
||||
by_source: Dict[str, int] # {"plugin": 5, "superset_api": 7}
|
||||
# [/DEF:LogStats:Class]
|
||||
|
||||
# [DEF:Task:Class]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: task, job, execution, state, pydantic
|
||||
# @PURPOSE: A Pydantic model representing a single execution instance of a plugin, including its status, parameters, and logs.
|
||||
class Task(BaseModel):
|
||||
|
||||
@@ -11,9 +11,10 @@ from typing import List, Optional, Dict, Any
|
||||
import json
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
from ...models.task import TaskRecord
|
||||
from sqlalchemy import and_, or_
|
||||
from ...models.task import TaskRecord, TaskLogRecord
|
||||
from ..database import TasksSessionLocal
|
||||
from .models import Task, TaskStatus, LogEntry
|
||||
from .models import Task, TaskStatus, LogEntry, TaskLog, LogFilter, LogStats
|
||||
from ..logger import logger, belief_scope
|
||||
# [/SECTION]
|
||||
|
||||
@@ -170,4 +171,215 @@ class TaskPersistenceService:
|
||||
# [/DEF:delete_tasks:Function]
|
||||
|
||||
# [/DEF:TaskPersistenceService:Class]
|
||||
|
||||
# [DEF:TaskLogPersistenceService:Class]
|
||||
# @SEMANTICS: persistence, service, database, log, sqlalchemy
|
||||
# @PURPOSE: Provides methods to save and query task logs from the task_logs table.
|
||||
# @TIER: CRITICAL
|
||||
# @RELATION: DEPENDS_ON -> TaskLogRecord
|
||||
# @INVARIANT: Log entries are batch-inserted for performance.
|
||||
class TaskLogPersistenceService:
|
||||
"""
|
||||
Service for persisting and querying task logs.
|
||||
Supports batch inserts, filtering, and statistics.
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the log persistence service.
|
||||
# @POST: Service is ready.
|
||||
def __init__(self):
|
||||
pass
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:add_logs:Function]
|
||||
# @PURPOSE: Batch insert log entries for a task.
|
||||
# @PRE: logs is a list of LogEntry objects.
|
||||
# @POST: All logs inserted into task_logs table.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @PARAM: logs (List[LogEntry]) - Log entries to insert.
|
||||
# @SIDE_EFFECT: Writes to task_logs table.
|
||||
def add_logs(self, task_id: str, logs: List[LogEntry]) -> None:
|
||||
if not logs:
|
||||
return
|
||||
with belief_scope("TaskLogPersistenceService.add_logs", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
for log in logs:
|
||||
record = TaskLogRecord(
|
||||
task_id=task_id,
|
||||
timestamp=log.timestamp,
|
||||
level=log.level,
|
||||
source=log.source or "system",
|
||||
message=log.message,
|
||||
metadata_json=json.dumps(log.metadata) if log.metadata else None
|
||||
)
|
||||
session.add(record)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to add logs for task {task_id}: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:add_logs:Function]
|
||||
|
||||
# [DEF:get_logs:Function]
|
||||
# @PURPOSE: Query logs for a task with filtering and pagination.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of TaskLog objects matching filters.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @PARAM: log_filter (LogFilter) - Filter parameters.
|
||||
# @RETURN: List[TaskLog] - Filtered log entries.
|
||||
def get_logs(self, task_id: str, log_filter: LogFilter) -> List[TaskLog]:
|
||||
with belief_scope("TaskLogPersistenceService.get_logs", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
query = session.query(TaskLogRecord).filter(TaskLogRecord.task_id == task_id)
|
||||
|
||||
# Apply filters
|
||||
if log_filter.level:
|
||||
query = query.filter(TaskLogRecord.level == log_filter.level.upper())
|
||||
if log_filter.source:
|
||||
query = query.filter(TaskLogRecord.source == log_filter.source)
|
||||
if log_filter.search:
|
||||
search_pattern = f"%{log_filter.search}%"
|
||||
query = query.filter(TaskLogRecord.message.ilike(search_pattern))
|
||||
|
||||
# Order by timestamp ascending (oldest first)
|
||||
query = query.order_by(TaskLogRecord.timestamp.asc())
|
||||
|
||||
# Apply pagination
|
||||
records = query.offset(log_filter.offset).limit(log_filter.limit).all()
|
||||
|
||||
logs = []
|
||||
for record in records:
|
||||
metadata = None
|
||||
if record.metadata_json:
|
||||
try:
|
||||
metadata = json.loads(record.metadata_json)
|
||||
except json.JSONDecodeError:
|
||||
metadata = None
|
||||
|
||||
logs.append(TaskLog(
|
||||
id=record.id,
|
||||
task_id=record.task_id,
|
||||
timestamp=record.timestamp,
|
||||
level=record.level,
|
||||
source=record.source,
|
||||
message=record.message,
|
||||
metadata=metadata
|
||||
))
|
||||
|
||||
return logs
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_logs:Function]
|
||||
|
||||
# [DEF:get_log_stats:Function]
|
||||
# @PURPOSE: Get statistics about logs for a task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns LogStats with counts by level and source.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: LogStats - Statistics about task logs.
|
||||
def get_log_stats(self, task_id: str) -> LogStats:
|
||||
with belief_scope("TaskLogPersistenceService.get_log_stats", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
# Get total count
|
||||
total_count = session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).count()
|
||||
|
||||
# Get counts by level
|
||||
from sqlalchemy import func
|
||||
level_counts = session.query(
|
||||
TaskLogRecord.level,
|
||||
func.count(TaskLogRecord.id)
|
||||
).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).group_by(TaskLogRecord.level).all()
|
||||
|
||||
by_level = {level: count for level, count in level_counts}
|
||||
|
||||
# Get counts by source
|
||||
source_counts = session.query(
|
||||
TaskLogRecord.source,
|
||||
func.count(TaskLogRecord.id)
|
||||
).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).group_by(TaskLogRecord.source).all()
|
||||
|
||||
by_source = {source: count for source, count in source_counts}
|
||||
|
||||
return LogStats(
|
||||
total_count=total_count,
|
||||
by_level=by_level,
|
||||
by_source=by_source
|
||||
)
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_log_stats:Function]
|
||||
|
||||
# [DEF:get_sources:Function]
|
||||
# @PURPOSE: Get unique sources for a task's logs.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: Returns list of unique source strings.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @RETURN: List[str] - Unique source names.
|
||||
def get_sources(self, task_id: str) -> List[str]:
|
||||
with belief_scope("TaskLogPersistenceService.get_sources", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
from sqlalchemy import distinct
|
||||
sources = session.query(distinct(TaskLogRecord.source)).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).all()
|
||||
return [s[0] for s in sources]
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:get_sources:Function]
|
||||
|
||||
# [DEF:delete_logs_for_task:Function]
|
||||
# @PURPOSE: Delete all logs for a specific task.
|
||||
# @PRE: task_id is a valid task ID.
|
||||
# @POST: All logs for the task are deleted.
|
||||
# @PARAM: task_id (str) - The task ID.
|
||||
# @SIDE_EFFECT: Deletes from task_logs table.
|
||||
def delete_logs_for_task(self, task_id: str) -> None:
|
||||
with belief_scope("TaskLogPersistenceService.delete_logs_for_task", f"task_id={task_id}"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id == task_id
|
||||
).delete(synchronize_session=False)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to delete logs for task {task_id}: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:delete_logs_for_task:Function]
|
||||
|
||||
# [DEF:delete_logs_for_tasks:Function]
|
||||
# @PURPOSE: Delete all logs for multiple tasks.
|
||||
# @PRE: task_ids is a list of task IDs.
|
||||
# @POST: All logs for the tasks are deleted.
|
||||
# @PARAM: task_ids (List[str]) - List of task IDs.
|
||||
def delete_logs_for_tasks(self, task_ids: List[str]) -> None:
|
||||
if not task_ids:
|
||||
return
|
||||
with belief_scope("TaskLogPersistenceService.delete_logs_for_tasks"):
|
||||
session: Session = TasksSessionLocal()
|
||||
try:
|
||||
session.query(TaskLogRecord).filter(
|
||||
TaskLogRecord.task_id.in_(task_ids)
|
||||
).delete(synchronize_session=False)
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
session.rollback()
|
||||
logger.error(f"Failed to delete logs for tasks: {e}")
|
||||
finally:
|
||||
session.close()
|
||||
# [/DEF:delete_logs_for_tasks:Function]
|
||||
|
||||
# [/DEF:TaskLogPersistenceService:Class]
|
||||
# [/DEF:TaskPersistenceModule:Module]
|
||||
168
backend/src/core/task_manager/task_logger.py
Normal file
168
backend/src/core/task_manager/task_logger.py
Normal file
@@ -0,0 +1,168 @@
|
||||
# [DEF:TaskLoggerModule:Module]
|
||||
# @SEMANTICS: task, logger, context, plugin, attribution
|
||||
# @PURPOSE: Provides a dedicated logger for tasks with automatic source attribution.
|
||||
# @LAYER: Core
|
||||
# @RELATION: DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: Each TaskLogger instance is bound to a specific task_id and default source.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import Dict, Any, Optional, Callable
|
||||
from datetime import datetime
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:TaskLogger:Class]
|
||||
# @SEMANTICS: logger, task, source, attribution
|
||||
# @PURPOSE: A wrapper around TaskManager._add_log that carries task_id and source context.
|
||||
# @TIER: CRITICAL
|
||||
# @INVARIANT: All log calls include the task_id and source.
|
||||
# @UX_STATE: Idle -> Logging -> (system records log)
|
||||
class TaskLogger:
|
||||
"""
|
||||
A dedicated logger for tasks that automatically tags logs with source attribution.
|
||||
|
||||
Usage:
|
||||
logger = TaskLogger(task_id="abc123", add_log_fn=task_manager._add_log, source="plugin")
|
||||
logger.info("Starting backup process")
|
||||
logger.error("Failed to connect", metadata={"error_code": 500})
|
||||
|
||||
# Create sub-logger with different source
|
||||
api_logger = logger.with_source("superset_api")
|
||||
api_logger.info("Fetching dashboards")
|
||||
"""
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the TaskLogger with task context.
|
||||
# @PRE: add_log_fn is a callable that accepts (task_id, level, message, context, source, metadata).
|
||||
# @POST: TaskLogger is ready to log messages.
|
||||
# @PARAM: task_id (str) - The ID of the task.
|
||||
# @PARAM: add_log_fn (Callable) - Function to add log to TaskManager.
|
||||
# @PARAM: source (str) - Default source for logs (default: "plugin").
|
||||
def __init__(
|
||||
self,
|
||||
task_id: str,
|
||||
add_log_fn: Callable,
|
||||
source: str = "plugin"
|
||||
):
|
||||
self._task_id = task_id
|
||||
self._add_log = add_log_fn
|
||||
self._default_source = source
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:with_source:Function]
|
||||
# @PURPOSE: Create a sub-logger with a different default source.
|
||||
# @PRE: source is a non-empty string.
|
||||
# @POST: Returns new TaskLogger with the same task_id but different source.
|
||||
# @PARAM: source (str) - New default source.
|
||||
# @RETURN: TaskLogger - New logger instance.
|
||||
def with_source(self, source: str) -> "TaskLogger":
|
||||
"""Create a sub-logger with a different source context."""
|
||||
return TaskLogger(
|
||||
task_id=self._task_id,
|
||||
add_log_fn=self._add_log,
|
||||
source=source
|
||||
)
|
||||
# [/DEF:with_source:Function]
|
||||
|
||||
# [DEF:_log:Function]
|
||||
# @PURPOSE: Internal method to log a message at a given level.
|
||||
# @PRE: level is a valid log level string.
|
||||
# @POST: Log entry added via add_log_fn.
|
||||
# @PARAM: level (str) - Log level (DEBUG, INFO, WARNING, ERROR).
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source for this log entry.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional structured data.
|
||||
def _log(
|
||||
self,
|
||||
level: str,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
"""Internal logging method."""
|
||||
self._add_log(
|
||||
task_id=self._task_id,
|
||||
level=level,
|
||||
message=message,
|
||||
source=source or self._default_source,
|
||||
metadata=metadata
|
||||
)
|
||||
# [/DEF:_log:Function]
|
||||
|
||||
# [DEF:debug:Function]
|
||||
# @PURPOSE: Log a DEBUG level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def debug(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("DEBUG", message, source, metadata)
|
||||
# [/DEF:debug:Function]
|
||||
|
||||
# [DEF:info:Function]
|
||||
# @PURPOSE: Log an INFO level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def info(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("INFO", message, source, metadata)
|
||||
# [/DEF:info:Function]
|
||||
|
||||
# [DEF:warning:Function]
|
||||
# @PURPOSE: Log a WARNING level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def warning(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("WARNING", message, source, metadata)
|
||||
# [/DEF:warning:Function]
|
||||
|
||||
# [DEF:error:Function]
|
||||
# @PURPOSE: Log an ERROR level message.
|
||||
# @PARAM: message (str) - Log message.
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
# @PARAM: metadata (Optional[Dict]) - Additional data.
|
||||
def error(
|
||||
self,
|
||||
message: str,
|
||||
source: Optional[str] = None,
|
||||
metadata: Optional[Dict[str, Any]] = None
|
||||
) -> None:
|
||||
self._log("ERROR", message, source, metadata)
|
||||
# [/DEF:error:Function]
|
||||
|
||||
# [DEF:progress:Function]
|
||||
# @PURPOSE: Log a progress update with percentage.
|
||||
# @PRE: percent is between 0 and 100.
|
||||
# @POST: Log entry with progress metadata added.
|
||||
# @PARAM: message (str) - Progress message.
|
||||
# @PARAM: percent (float) - Progress percentage (0-100).
|
||||
# @PARAM: source (Optional[str]) - Override source.
|
||||
def progress(
|
||||
self,
|
||||
message: str,
|
||||
percent: float,
|
||||
source: Optional[str] = None
|
||||
) -> None:
|
||||
"""Log a progress update with percentage."""
|
||||
metadata = {"progress": min(100, max(0, percent))}
|
||||
self._log("INFO", message, source, metadata)
|
||||
# [/DEF:progress:Function]
|
||||
|
||||
# [/DEF:TaskLogger:Class]
|
||||
|
||||
# [/DEF:TaskLoggerModule:Module]
|
||||
Reference in New Issue
Block a user