linter + новые таски
This commit is contained in:
18
backend/src/services/__init__.py
Normal file
18
backend/src/services/__init__.py
Normal file
@@ -0,0 +1,18 @@
|
||||
# [DEF:backend.src.services:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: services, package, init
|
||||
# @PURPOSE: Package initialization for services module
|
||||
# @LAYER: Core
|
||||
# @RELATION: EXPORTS -> resource_service, mapping_service
|
||||
# @NOTE: Only export services that don't cause circular imports
|
||||
# @NOTE: GitService, AuthService, LLMProviderService have circular import issues - import directly when needed
|
||||
|
||||
# Only export services that don't cause circular imports
|
||||
from .mapping_service import MappingService
|
||||
from .resource_service import ResourceService
|
||||
|
||||
__all__ = [
|
||||
'MappingService',
|
||||
'ResourceService',
|
||||
]
|
||||
# [/DEF:backend.src.services:Module]
|
||||
@@ -10,11 +10,11 @@
|
||||
# @INVARIANT: Authentication must verify both credentials and account status.
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import Optional, Dict, Any, List
|
||||
from typing import Dict, Any
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models.auth import User, Role
|
||||
from ..core.auth.repository import AuthRepository
|
||||
from ..core.auth.security import verify_password, get_password_hash
|
||||
from ..core.auth.security import verify_password
|
||||
from ..core.auth.jwt import create_access_token
|
||||
from ..core.logger import belief_scope
|
||||
# [/SECTION]
|
||||
|
||||
@@ -10,11 +10,10 @@
|
||||
# @INVARIANT: All Git operations must be performed on a valid local directory.
|
||||
|
||||
import os
|
||||
import shutil
|
||||
import httpx
|
||||
from git import Repo, RemoteProgress
|
||||
from git import Repo
|
||||
from fastapi import HTTPException
|
||||
from typing import List, Optional
|
||||
from typing import List
|
||||
from datetime import datetime
|
||||
from src.core.logger import logger, belief_scope
|
||||
from src.models.git import GitProvider
|
||||
@@ -167,7 +166,7 @@ class GitService:
|
||||
|
||||
# Handle empty repository case (no commits)
|
||||
if not repo.heads and not repo.remotes:
|
||||
logger.warning(f"[create_branch][Action] Repository is empty. Creating initial commit to enable branching.")
|
||||
logger.warning("[create_branch][Action] Repository is empty. Creating initial commit to enable branching.")
|
||||
readme_path = os.path.join(repo.working_dir, "README.md")
|
||||
if not os.path.exists(readme_path):
|
||||
with open(readme_path, "w") as f:
|
||||
@@ -178,7 +177,7 @@ class GitService:
|
||||
# Verify source branch exists
|
||||
try:
|
||||
repo.commit(from_branch)
|
||||
except:
|
||||
except Exception:
|
||||
logger.warning(f"[create_branch][Action] Source branch {from_branch} not found, using HEAD")
|
||||
from_branch = repo.head
|
||||
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
from typing import List, Optional
|
||||
from sqlalchemy.orm import Session
|
||||
from ..models.llm import LLMProvider
|
||||
from ..plugins.llm_analysis.models import LLMProviderConfig, LLMProviderType
|
||||
from ..plugins.llm_analysis.models import LLMProviderConfig
|
||||
from ..core.logger import belief_scope, logger
|
||||
from cryptography.fernet import Fernet
|
||||
import os
|
||||
|
||||
251
backend/src/services/resource_service.py
Normal file
251
backend/src/services/resource_service.py
Normal file
@@ -0,0 +1,251 @@
|
||||
# [DEF:backend.src.services.resource_service:Module]
|
||||
# @TIER: STANDARD
|
||||
# @SEMANTICS: service, resources, dashboards, datasets, tasks, git
|
||||
# @PURPOSE: Shared service for fetching resource data with Git status and task status
|
||||
# @LAYER: Service
|
||||
# @RELATION: DEPENDS_ON -> backend.src.core.superset_client
|
||||
# @RELATION: DEPENDS_ON -> backend.src.core.task_manager
|
||||
# @RELATION: DEPENDS_ON -> backend.src.services.git_service
|
||||
# @INVARIANT: All resources include metadata about their current state
|
||||
|
||||
# [SECTION: IMPORTS]
|
||||
from typing import List, Dict, Optional, Any
|
||||
from ..core.superset_client import SupersetClient
|
||||
from ..core.task_manager.models import Task
|
||||
from ..services.git_service import GitService
|
||||
from ..core.logger import logger, belief_scope
|
||||
# [/SECTION]
|
||||
|
||||
# [DEF:ResourceService:Class]
|
||||
# @PURPOSE: Provides centralized access to resource data with enhanced metadata
|
||||
class ResourceService:
|
||||
|
||||
# [DEF:__init__:Function]
|
||||
# @PURPOSE: Initialize the resource service with dependencies
|
||||
# @PRE: None
|
||||
# @POST: ResourceService is ready to fetch resources
|
||||
def __init__(self):
|
||||
with belief_scope("ResourceService.__init__"):
|
||||
self.git_service = GitService()
|
||||
logger.info("[ResourceService][Action] Initialized ResourceService")
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
# [DEF:get_dashboards_with_status:Function]
|
||||
# @PURPOSE: Fetch dashboards from environment with Git status and last task status
|
||||
# @PRE: env is a valid Environment object
|
||||
# @POST: Returns list of dashboards with enhanced metadata
|
||||
# @PARAM: env (Environment) - The environment to fetch from
|
||||
# @PARAM: tasks (List[Task]) - List of tasks to check for status
|
||||
# @RETURN: List[Dict] - Dashboards with git_status and last_task fields
|
||||
# @RELATION: CALLS -> SupersetClient.get_dashboards_summary
|
||||
# @RELATION: CALLS -> self._get_git_status_for_dashboard
|
||||
# @RELATION: CALLS -> self._get_last_task_for_resource
|
||||
async def get_dashboards_with_status(
|
||||
self,
|
||||
env: Any,
|
||||
tasks: Optional[List[Task]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
with belief_scope("get_dashboards_with_status", f"env={env.id}"):
|
||||
client = SupersetClient(env)
|
||||
dashboards = client.get_dashboards_summary()
|
||||
|
||||
# Enhance each dashboard with Git status and task status
|
||||
result = []
|
||||
for dashboard in dashboards:
|
||||
# dashboard is already a dict, no need to call .dict()
|
||||
dashboard_dict = dashboard
|
||||
dashboard_id = dashboard_dict.get('id')
|
||||
|
||||
# Get Git status if repo exists
|
||||
git_status = self._get_git_status_for_dashboard(dashboard_id)
|
||||
dashboard_dict['git_status'] = git_status
|
||||
|
||||
# Get last task status
|
||||
last_task = self._get_last_task_for_resource(
|
||||
f"dashboard-{dashboard_id}",
|
||||
tasks
|
||||
)
|
||||
dashboard_dict['last_task'] = last_task
|
||||
|
||||
result.append(dashboard_dict)
|
||||
|
||||
logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} dashboards with status")
|
||||
return result
|
||||
# [/DEF:get_dashboards_with_status:Function]
|
||||
|
||||
# [DEF:get_datasets_with_status:Function]
|
||||
# @PURPOSE: Fetch datasets from environment with mapping progress and last task status
|
||||
# @PRE: env is a valid Environment object
|
||||
# @POST: Returns list of datasets with enhanced metadata
|
||||
# @PARAM: env (Environment) - The environment to fetch from
|
||||
# @PARAM: tasks (List[Task]) - List of tasks to check for status
|
||||
# @RETURN: List[Dict] - Datasets with mapped_fields and last_task fields
|
||||
# @RELATION: CALLS -> SupersetClient.get_datasets_summary
|
||||
# @RELATION: CALLS -> self._get_last_task_for_resource
|
||||
async def get_datasets_with_status(
|
||||
self,
|
||||
env: Any,
|
||||
tasks: Optional[List[Task]] = None
|
||||
) -> List[Dict[str, Any]]:
|
||||
with belief_scope("get_datasets_with_status", f"env={env.id}"):
|
||||
client = SupersetClient(env)
|
||||
datasets = client.get_datasets_summary()
|
||||
|
||||
# Enhance each dataset with task status
|
||||
result = []
|
||||
for dataset in datasets:
|
||||
# dataset is already a dict, no need to call .dict()
|
||||
dataset_dict = dataset
|
||||
dataset_id = dataset_dict.get('id')
|
||||
|
||||
# Get last task status
|
||||
last_task = self._get_last_task_for_resource(
|
||||
f"dataset-{dataset_id}",
|
||||
tasks
|
||||
)
|
||||
dataset_dict['last_task'] = last_task
|
||||
|
||||
result.append(dataset_dict)
|
||||
|
||||
logger.info(f"[ResourceService][Coherence:OK] Fetched {len(result)} datasets with status")
|
||||
return result
|
||||
# [/DEF:get_datasets_with_status:Function]
|
||||
|
||||
# [DEF:get_activity_summary:Function]
|
||||
# @PURPOSE: Get summary of active and recent tasks for the activity indicator
|
||||
# @PRE: tasks is a list of Task objects
|
||||
# @POST: Returns summary with active_count and recent_tasks
|
||||
# @PARAM: tasks (List[Task]) - List of tasks to summarize
|
||||
# @RETURN: Dict - Activity summary
|
||||
def get_activity_summary(self, tasks: List[Task]) -> Dict[str, Any]:
|
||||
with belief_scope("get_activity_summary"):
|
||||
# Count active (RUNNING, WAITING_INPUT) tasks
|
||||
active_tasks = [
|
||||
t for t in tasks
|
||||
if t.status in ['RUNNING', 'WAITING_INPUT']
|
||||
]
|
||||
|
||||
# Get recent tasks (last 5)
|
||||
recent_tasks = sorted(
|
||||
tasks,
|
||||
key=lambda t: t.created_at,
|
||||
reverse=True
|
||||
)[:5]
|
||||
|
||||
# Format recent tasks for frontend
|
||||
recent_tasks_formatted = []
|
||||
for task in recent_tasks:
|
||||
resource_name = self._extract_resource_name_from_task(task)
|
||||
recent_tasks_formatted.append({
|
||||
'task_id': str(task.id),
|
||||
'resource_name': resource_name,
|
||||
'resource_type': self._extract_resource_type_from_task(task),
|
||||
'status': task.status,
|
||||
'started_at': task.created_at.isoformat() if task.created_at else None
|
||||
})
|
||||
|
||||
return {
|
||||
'active_count': len(active_tasks),
|
||||
'recent_tasks': recent_tasks_formatted
|
||||
}
|
||||
# [/DEF:get_activity_summary:Function]
|
||||
|
||||
# [DEF:_get_git_status_for_dashboard:Function]
|
||||
# @PURPOSE: Get Git sync status for a dashboard
|
||||
# @PRE: dashboard_id is a valid integer
|
||||
# @POST: Returns git status or None if no repo exists
|
||||
# @PARAM: dashboard_id (int) - The dashboard ID
|
||||
# @RETURN: Optional[Dict] - Git status with branch and sync_status
|
||||
# @RELATION: CALLS -> GitService.get_repo
|
||||
def _get_git_status_for_dashboard(self, dashboard_id: int) -> Optional[Dict[str, Any]]:
|
||||
try:
|
||||
repo = self.git_service.get_repo(dashboard_id)
|
||||
if not repo:
|
||||
return None
|
||||
|
||||
# Check if there are uncommitted changes
|
||||
try:
|
||||
# Get current branch
|
||||
branch = repo.active_branch.name
|
||||
|
||||
# Check for uncommitted changes
|
||||
is_dirty = repo.is_dirty()
|
||||
|
||||
# Check for unpushed commits
|
||||
unpushed = len(list(repo.iter_commits(f'{branch}@{{u}}..{branch}'))) if '@{u}' in str(repo.refs) else 0
|
||||
|
||||
if is_dirty or unpushed > 0:
|
||||
sync_status = 'DIFF'
|
||||
else:
|
||||
sync_status = 'OK'
|
||||
|
||||
return {
|
||||
'branch': branch,
|
||||
'sync_status': sync_status
|
||||
}
|
||||
except Exception:
|
||||
logger.warning(f"[ResourceService][Warning] Failed to get git status for dashboard {dashboard_id}")
|
||||
return None
|
||||
except Exception:
|
||||
# No repo exists for this dashboard
|
||||
return None
|
||||
# [/DEF:_get_git_status_for_dashboard:Function]
|
||||
|
||||
# [DEF:_get_last_task_for_resource:Function]
|
||||
# @PURPOSE: Get the most recent task for a specific resource
|
||||
# @PRE: resource_id is a valid string
|
||||
# @POST: Returns task summary or None if no tasks found
|
||||
# @PARAM: resource_id (str) - The resource identifier (e.g., "dashboard-123")
|
||||
# @PARAM: tasks (Optional[List[Task]]) - List of tasks to search
|
||||
# @RETURN: Optional[Dict] - Task summary with task_id and status
|
||||
def _get_last_task_for_resource(
|
||||
self,
|
||||
resource_id: str,
|
||||
tasks: Optional[List[Task]] = None
|
||||
) -> Optional[Dict[str, Any]]:
|
||||
if not tasks:
|
||||
return None
|
||||
|
||||
# Filter tasks for this resource
|
||||
resource_tasks = []
|
||||
for task in tasks:
|
||||
params = task.params or {}
|
||||
if params.get('resource_id') == resource_id:
|
||||
resource_tasks.append(task)
|
||||
|
||||
if not resource_tasks:
|
||||
return None
|
||||
|
||||
# Get most recent task
|
||||
last_task = max(resource_tasks, key=lambda t: t.created_at)
|
||||
|
||||
return {
|
||||
'task_id': str(last_task.id),
|
||||
'status': last_task.status
|
||||
}
|
||||
# [/DEF:_get_last_task_for_resource:Function]
|
||||
|
||||
# [DEF:_extract_resource_name_from_task:Function]
|
||||
# @PURPOSE: Extract resource name from task params
|
||||
# @PRE: task is a valid Task object
|
||||
# @POST: Returns resource name or task ID
|
||||
# @PARAM: task (Task) - The task to extract from
|
||||
# @RETURN: str - Resource name or fallback
|
||||
def _extract_resource_name_from_task(self, task: Task) -> str:
|
||||
params = task.params or {}
|
||||
return params.get('resource_name', f"Task {task.id}")
|
||||
# [/DEF:_extract_resource_name_from_task:Function]
|
||||
|
||||
# [DEF:_extract_resource_type_from_task:Function]
|
||||
# @PURPOSE: Extract resource type from task params
|
||||
# @PRE: task is a valid Task object
|
||||
# @POST: Returns resource type or 'unknown'
|
||||
# @PARAM: task (Task) - The task to extract from
|
||||
# @RETURN: str - Resource type
|
||||
def _extract_resource_type_from_task(self, task: Task) -> str:
|
||||
params = task.params or {}
|
||||
return params.get('resource_type', 'unknown')
|
||||
# [/DEF:_extract_resource_type_from_task:Function]
|
||||
|
||||
# [/DEF:ResourceService:Class]
|
||||
# [/DEF:backend.src.services.resource_service:Module]
|
||||
Reference in New Issue
Block a user