linter + новые таски

backend/src/api/routes/__init__.py

@@ -1 +1,3 @@
 from . import plugins, tasks, settings, connections, environments, mappings, migration, git, storage, admin
+
+__all__ = ['plugins', 'tasks', 'settings', 'connections', 'environments', 'mappings', 'migration', 'git', 'storage', 'admin']

backend/src/api/routes/admin.py

@@ -21,8 +21,8 @@ from ...schemas.auth import (
     RoleSchema, RoleCreate, RoleUpdate, PermissionSchema,
     ADGroupMappingSchema, ADGroupMappingCreate
 )
-from ...models.auth import User, Role, Permission, ADGroupMapping
-from ...dependencies import has_permission, get_current_user
+from ...models.auth import User, Role, ADGroupMapping
+from ...dependencies import has_permission
 from ...core.logger import logger, belief_scope
 # [/SECTION]
 

backend/src/api/routes/connections.py

@@ -11,7 +11,7 @@ from fastapi import APIRouter, Depends, HTTPException, status
 from sqlalchemy.orm import Session
 from ...core.database import get_db
 from ...models.connection import ConnectionConfig
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from datetime import datetime
 from ...core.logger import logger, belief_scope
 # [/SECTION]

backend/src/api/routes/dashboards.py

@@ -0,0 +1,105 @@
+# [DEF:backend.src.api.routes.dashboards:Module]
+#
+# @TIER: STANDARD
+# @SEMANTICS: api, dashboards, resources, hub
+# @PURPOSE:   API endpoints for the Dashboard Hub - listing dashboards with Git and task status
+# @LAYER:     API
+# @RELATION:  DEPENDS_ON -> backend.src.dependencies
+# @RELATION:  DEPENDS_ON -> backend.src.services.resource_service
+# @RELATION:  DEPENDS_ON -> backend.src.core.superset_client
+#
+# @INVARIANT: All dashboard responses include git_status and last_task metadata
+
+# [SECTION: IMPORTS]
+from fastapi import APIRouter, Depends, HTTPException
+from typing import List, Optional
+from pydantic import BaseModel, Field
+from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission
+from ...core.logger import logger, belief_scope
+# [/SECTION]
+
+router = APIRouter()
+
+# [DEF:GitStatus:DataClass]
+class GitStatus(BaseModel):
+    branch: Optional[str] = None
+    sync_status: Optional[str] = Field(None, pattern="^OK|DIFF$")
+# [/DEF:GitStatus:DataClass]
+
+# [DEF:LastTask:DataClass]
+class LastTask(BaseModel):
+    task_id: Optional[str] = None
+    status: Optional[str] = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$")
+# [/DEF:LastTask:DataClass]
+
+# [DEF:DashboardItem:DataClass]
+class DashboardItem(BaseModel):
+    id: int
+    title: str
+    slug: Optional[str] = None
+    url: Optional[str] = None
+    last_modified: Optional[str] = None
+    git_status: Optional[GitStatus] = None
+    last_task: Optional[LastTask] = None
+# [/DEF:DashboardItem:DataClass]
+
+# [DEF:DashboardsResponse:DataClass]
+class DashboardsResponse(BaseModel):
+    dashboards: List[DashboardItem]
+    total: int
+# [/DEF:DashboardsResponse:DataClass]
+
+# [DEF:get_dashboards:Function]
+# @PURPOSE: Fetch list of dashboards from a specific environment with Git status and last task status
+# @PRE: env_id must be a valid environment ID
+# @POST: Returns a list of dashboards with enhanced metadata
+# @PARAM: env_id (str) - The environment ID to fetch dashboards from
+# @PARAM: search (Optional[str]) - Filter by title/slug
+# @RETURN: DashboardsResponse - List of dashboards with status metadata
+# @RELATION: CALLS -> ResourceService.get_dashboards_with_status
+@router.get("/api/dashboards", response_model=DashboardsResponse)
+async def get_dashboards(
+    env_id: str,
+    search: Optional[str] = None,
+    config_manager=Depends(get_config_manager),
+    task_manager=Depends(get_task_manager),
+    resource_service=Depends(get_resource_service),
+    _ = Depends(has_permission("plugin:migration", "READ"))
+):
+    with belief_scope("get_dashboards", f"env_id={env_id}, search={search}"):
+        # Validate environment exists
+        environments = config_manager.get_environments()
+        env = next((e for e in environments if e.id == env_id), None)
+        if not env:
+            logger.error(f"[get_dashboards][Coherence:Failed] Environment not found: {env_id}")
+            raise HTTPException(status_code=404, detail="Environment not found")
+        
+        try:
+            # Get all tasks for status lookup
+            all_tasks = task_manager.get_all_tasks()
+            
+            # Fetch dashboards with status using ResourceService
+            dashboards = await resource_service.get_dashboards_with_status(env, all_tasks)
+            
+            # Apply search filter if provided
+            if search:
+                search_lower = search.lower()
+                dashboards = [
+                    d for d in dashboards 
+                    if search_lower in d.get('title', '').lower() 
+                    or search_lower in d.get('slug', '').lower()
+                ]
+            
+            logger.info(f"[get_dashboards][Coherence:OK] Returning {len(dashboards)} dashboards")
+            
+            return DashboardsResponse(
+                dashboards=dashboards,
+                total=len(dashboards)
+            )
+            
+        except Exception as e:
+            logger.error(f"[get_dashboards][Coherence:Failed] Failed to fetch dashboards: {e}")
+            raise HTTPException(status_code=503, detail=f"Failed to fetch dashboards: {str(e)}")
+# [/DEF:get_dashboards:Function]
+
+# [/DEF:backend.src.api.routes.dashboards:Module]

backend/src/api/routes/datasets.py

@@ -0,0 +1,103 @@
+# [DEF:backend.src.api.routes.datasets:Module]
+#
+# @TIER: STANDARD
+# @SEMANTICS: api, datasets, resources, hub
+# @PURPOSE:   API endpoints for the Dataset Hub - listing datasets with mapping progress
+# @LAYER:     API
+# @RELATION:  DEPENDS_ON -> backend.src.dependencies
+# @RELATION:  DEPENDS_ON -> backend.src.services.resource_service
+# @RELATION:  DEPENDS_ON -> backend.src.core.superset_client
+#
+# @INVARIANT: All dataset responses include last_task metadata
+
+# [SECTION: IMPORTS]
+from fastapi import APIRouter, Depends, HTTPException
+from typing import List, Optional
+from pydantic import BaseModel, Field
+from ...dependencies import get_config_manager, get_task_manager, get_resource_service, has_permission
+from ...core.logger import logger, belief_scope
+# [/SECTION]
+
+router = APIRouter()
+
+# [DEF:MappedFields:DataClass]
+class MappedFields(BaseModel):
+    total: int
+    mapped: int
+# [/DEF:MappedFields:DataClass]
+
+# [DEF:LastTask:DataClass]
+class LastTask(BaseModel):
+    task_id: Optional[str] = None
+    status: Optional[str] = Field(None, pattern="^RUNNING|SUCCESS|ERROR|WAITING_INPUT$")
+# [/DEF:LastTask:DataClass]
+
+# [DEF:DatasetItem:DataClass]
+class DatasetItem(BaseModel):
+    id: int
+    table_name: str
+    schema: str
+    database: str
+    mapped_fields: Optional[MappedFields] = None
+    last_task: Optional[LastTask] = None
+# [/DEF:DatasetItem:DataClass]
+
+# [DEF:DatasetsResponse:DataClass]
+class DatasetsResponse(BaseModel):
+    datasets: List[DatasetItem]
+    total: int
+# [/DEF:DatasetsResponse:DataClass]
+
+# [DEF:get_datasets:Function]
+# @PURPOSE: Fetch list of datasets from a specific environment with mapping progress
+# @PRE:     env_id must be a valid environment ID
+# @POST:    Returns a list of datasets with enhanced metadata
+# @PARAM:   env_id (str) - The environment ID to fetch datasets from
+# @PARAM:   search (Optional[str]) - Filter by table name
+# @RETURN:  DatasetsResponse - List of datasets with status metadata
+# @RELATION: CALLS -> ResourceService.get_datasets_with_status
+@router.get("/api/datasets", response_model=DatasetsResponse)
+async def get_datasets(
+    env_id: str,
+    search: Optional[str] = None,
+    config_manager=Depends(get_config_manager),
+    task_manager=Depends(get_task_manager),
+    resource_service=Depends(get_resource_service),
+    _ = Depends(has_permission("plugin:migration", "READ"))
+):
+    with belief_scope("get_datasets", f"env_id={env_id}, search={search}"):
+        # Validate environment exists
+        environments = config_manager.get_environments()
+        env = next((e for e in environments if e.id == env_id), None)
+        if not env:
+            logger.error(f"[get_datasets][Coherence:Failed] Environment not found: {env_id}")
+            raise HTTPException(status_code=404, detail="Environment not found")
+        
+        try:
+            # Get all tasks for status lookup
+            all_tasks = task_manager.get_all_tasks()
+            
+            # Fetch datasets with status using ResourceService
+            datasets = await resource_service.get_datasets_with_status(env, all_tasks)
+            
+            # Apply search filter if provided
+            if search:
+                search_lower = search.lower()
+                datasets = [
+                    d for d in datasets 
+                    if search_lower in d.get('table_name', '').lower()
+                ]
+            
+            logger.info(f"[get_datasets][Coherence:OK] Returning {len(datasets)} datasets")
+            
+            return DatasetsResponse(
+                datasets=datasets,
+                total=len(datasets)
+            )
+            
+        except Exception as e:
+            logger.error(f"[get_datasets][Coherence:Failed] Failed to fetch datasets: {e}")
+            raise HTTPException(status_code=503, detail=f"Failed to fetch datasets: {str(e)}")
+# [/DEF:get_datasets:Function]
+
+# [/DEF:backend.src.api.routes.datasets:Module]

backend/src/api/routes/environments.py

@@ -11,11 +11,10 @@
 
 # [SECTION: IMPORTS]
 from fastapi import APIRouter, Depends, HTTPException
-from typing import List, Dict, Optional
+from typing import List, Optional
 from ...dependencies import get_config_manager, get_scheduler_service, has_permission
 from ...core.superset_client import SupersetClient
 from pydantic import BaseModel, Field
-from ...core.config_models import Environment as EnvModel
 from ...core.logger import belief_scope
 # [/SECTION]
 

backend/src/api/routes/git.py

@@ -16,10 +16,10 @@ from typing import List, Optional
 import typing
 from src.dependencies import get_config_manager, has_permission
 from src.core.database import get_db
-from src.models.git import GitServerConfig, GitStatus, DeploymentEnvironment, GitRepository
+from src.models.git import GitServerConfig, GitRepository
 from src.api.routes.git_schemas import (
     GitServerConfigSchema, GitServerConfigCreate,
-    GitRepositorySchema, BranchSchema, BranchCreate,
+    BranchSchema, BranchCreate,
     BranchCheckout, CommitSchema, CommitCreate,
     DeploymentEnvironmentSchema, DeployRequest, RepoInitRequest
 )

backend/src/api/routes/git_schemas.py

@@ -11,7 +11,6 @@
 from pydantic import BaseModel, Field
 from typing import List, Optional
 from datetime import datetime
-from uuid import UUID
 from src.models.git import GitProvider, GitStatus, SyncStatus
 
 # [DEF:GitServerConfigBase:Class]

backend/src/api/routes/migration.py

@@ -7,7 +7,7 @@
 # @RELATION:  DEPENDS_ON -> backend.src.models.dashboard
 
 from fastapi import APIRouter, Depends, HTTPException
-from typing import List, Dict
+from typing import List
 from ...dependencies import get_config_manager, get_task_manager, has_permission
 from ...models.dashboard import DashboardMetadata, DashboardSelection
 from ...core.superset_client import SupersetClient

backend/src/api/routes/settings.py

@@ -17,9 +17,8 @@ from ...core.config_models import AppConfig, Environment, GlobalSettings, Loggin
 from ...models.storage import StorageConfig
 from ...dependencies import get_config_manager, has_permission
 from ...core.config_manager import ConfigManager
-from ...core.logger import logger, belief_scope, get_task_log_level
+from ...core.logger import logger, belief_scope
 from ...core.superset_client import SupersetClient
-import os
 # [/SECTION]
 
 # [DEF:LoggingConfigResponse:Class]
@@ -279,4 +278,37 @@ async def update_logging_config(
         )
 # [/DEF:update_logging_config:Function]
 
+# [DEF:ConsolidatedSettingsResponse:Class]
+class ConsolidatedSettingsResponse(BaseModel):
+    environments: List[dict]
+    connections: List[dict]
+    llm: dict
+    logging: dict
+    storage: dict
+# [/DEF:ConsolidatedSettingsResponse:Class]
+
+# [DEF:get_consolidated_settings:Function]
+# @PURPOSE: Retrieves all settings categories in a single call
+# @PRE:     Config manager is available.
+# @POST:    Returns all consolidated settings.
+# @RETURN: ConsolidatedSettingsResponse - All settings categories.
+@router.get("/consolidated", response_model=ConsolidatedSettingsResponse)
+async def get_consolidated_settings(
+    config_manager: ConfigManager = Depends(get_config_manager),
+    _ = Depends(has_permission("admin:settings", "READ"))
+):
+    with belief_scope("get_consolidated_settings"):
+        logger.info("[get_consolidated_settings][Entry] Fetching all consolidated settings")
+        
+        config = config_manager.get_config()
+        
+        return ConsolidatedSettingsResponse(
+            environments=config.environments,
+            connections=config.settings.connections,
+            llm=config.settings.llm,
+            logging=config.settings.logging,
+            storage=config.settings.storage
+        )
+# [/DEF:get_consolidated_settings:Function]
+
 # [/DEF:SettingsRouter:Module]

backend/src/api/routes/tasks.py

@@ -6,7 +6,7 @@
 # @RELATION: Depends on the TaskManager. It is included by the main app.
 from typing import List, Dict, Any, Optional
 from fastapi import APIRouter, Depends, HTTPException, status, Query
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 from ...core.logger import belief_scope
 
 from ...core.task_manager import TaskManager, Task, TaskStatus, LogEntry