From 09e59ba88b012954bc6ee210fedb5fcdaf5c095f Mon Sep 17 00:00:00 2001 From: busya Date: Wed, 4 Mar 2026 10:04:40 +0300 Subject: [PATCH] fix repo place --- backend/git_repos/10 | 1 - backend/src/api/routes/git.py | 37 +++++- backend/src/services/git_service.py | 170 ++++++++++++++++++++++++++-- 3 files changed, 194 insertions(+), 14 deletions(-) delete mode 160000 backend/git_repos/10 diff --git a/backend/git_repos/10 b/backend/git_repos/10 deleted file mode 160000 index dec2896..0000000 --- a/backend/git_repos/10 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit dec289695ffe3ddf27acbce8106d20e3a524be89 diff --git a/backend/src/api/routes/git.py b/backend/src/api/routes/git.py index da9171c..65ac99f 100644 --- a/backend/src/api/routes/git.py +++ b/backend/src/api/routes/git.py @@ -175,6 +175,38 @@ def _resolve_dashboard_id_from_ref( return dashboard_id # [/DEF:_resolve_dashboard_id_from_ref:Function] + +# [DEF:_resolve_repo_key_from_ref:Function] +# @PURPOSE: Resolve repository folder key with slug-first strategy and deterministic fallback. +# @PRE: dashboard_id is resolved and valid. +# @POST: Returns safe key to be used in local repository path. +# @RETURN: str +def _resolve_repo_key_from_ref( + dashboard_ref: str, + dashboard_id: int, + config_manager, + env_id: Optional[str] = None, +) -> str: + normalized_ref = str(dashboard_ref or "").strip() + if normalized_ref and not normalized_ref.isdigit(): + return normalized_ref + + if env_id: + try: + environments = config_manager.get_environments() + env = next((e for e in environments if e.id == env_id), None) + if env: + payload = SupersetClient(env).get_dashboard(dashboard_id) + dashboard_data = payload.get("result", payload) if isinstance(payload, dict) else {} + dashboard_slug = dashboard_data.get("slug") + if dashboard_slug: + return str(dashboard_slug) + except Exception: + pass + + return f"dashboard-{dashboard_id}" +# [/DEF:_resolve_repo_key_from_ref:Function] + # [DEF:get_git_configs:Function] # @PURPOSE: List all configured Git servers. # @PRE: Database session `db` is available. @@ -417,6 +449,7 @@ async def init_repository( ): with belief_scope("init_repository"): dashboard_id = _resolve_dashboard_id_from_ref(dashboard_ref, config_manager, env_id) + repo_key = _resolve_repo_key_from_ref(dashboard_ref, dashboard_id, config_manager, env_id) # 1. Get config config = db.query(GitServerConfig).filter(GitServerConfig.id == init_data.config_id).first() if not config: @@ -425,10 +458,10 @@ async def init_repository( try: # 2. Perform Git clone/init logger.info(f"[init_repository][Action] Initializing repo for dashboard {dashboard_id}") - git_service.init_repo(dashboard_id, init_data.remote_url, config.pat) + git_service.init_repo(dashboard_id, init_data.remote_url, config.pat, repo_key=repo_key) # 3. Save to DB - repo_path = git_service._get_repo_path(dashboard_id) + repo_path = git_service._get_repo_path(dashboard_id, repo_key=repo_key) db_repo = db.query(GitRepository).filter(GitRepository.dashboard_id == dashboard_id).first() if not db_repo: db_repo = GitRepository( diff --git a/backend/src/services/git_service.py b/backend/src/services/git_service.py index 01eade6..01694cc 100644 --- a/backend/src/services/git_service.py +++ b/backend/src/services/git_service.py @@ -1,23 +1,33 @@ # [DEF:backend.src.services.git_service:Module] # +# @TIER: STANDARD # @SEMANTICS: git, service, gitpython, repository, version_control # @PURPOSE: Core Git logic using GitPython to manage dashboard repositories. # @LAYER: Service # @RELATION: INHERITS_FROM -> None # @RELATION: USED_BY -> src.api.routes.git # @RELATION: USED_BY -> src.plugins.git_plugin +# @RELATION: DEPENDS_ON -> src.core.database.SessionLocal +# @RELATION: DEPENDS_ON -> src.models.config.AppConfigRecord +# @RELATION: DEPENDS_ON -> src.models.git.GitRepository # # @INVARIANT: All Git operations must be performed on a valid local directory. import os import httpx +import re +import shutil from git import Repo from fastapi import HTTPException from typing import Any, Dict, List, Optional from datetime import datetime +from pathlib import Path from urllib.parse import quote, urlparse from src.core.logger import logger, belief_scope from src.models.git import GitProvider +from src.models.git import GitRepository +from src.models.config import AppConfigRecord +from src.core.database import SessionLocal # [DEF:GitService:Class] # @PURPOSE: Wrapper for GitPython operations with semantic logging and error handling. @@ -33,28 +43,164 @@ class GitService: # @POST: GitService is initialized; base_path directory exists. def __init__(self, base_path: str = "git_repos"): with belief_scope("GitService.__init__"): - # Resolve relative to the backend directory - # Path(__file__) is backend/src/services/git_service.py - # parents[2] is backend/ - from pathlib import Path backend_root = Path(__file__).parents[2] - - self.base_path = str((backend_root / base_path).resolve()) + self.legacy_base_path = str((backend_root / "git_repos").resolve()) + self.base_path = self._resolve_base_path(base_path) if not os.path.exists(self.base_path): os.makedirs(self.base_path) # [/DEF:__init__:Function] + # [DEF:_resolve_base_path:Function] + # @PURPOSE: Resolve base repository directory from explicit argument or global storage settings. + # @PRE: base_path is a string path. + # @POST: Returns absolute path for Git repositories root. + # @RETURN: str + def _resolve_base_path(self, base_path: str) -> str: + # Resolve relative to backend directory for backward compatibility. + backend_root = Path(__file__).parents[2] + fallback_path = str((backend_root / base_path).resolve()) + + if base_path != "git_repos": + return fallback_path + + try: + session = SessionLocal() + try: + config_row = session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() + finally: + session.close() + + payload = (config_row.payload if config_row and config_row.payload else {}) if config_row else {} + storage_cfg = payload.get("settings", {}).get("storage", {}) if isinstance(payload, dict) else {} + + root_path = str(storage_cfg.get("root_path", "")).strip() + repo_path = str(storage_cfg.get("repo_path", "")).strip() + if not root_path: + return fallback_path + + project_root = Path(__file__).parents[3] + root = Path(root_path) + if not root.is_absolute(): + root = (project_root / root).resolve() + + repo_root = Path(repo_path) if repo_path else Path("repositorys") + if repo_root.is_absolute(): + return str(repo_root.resolve()) + return str((root / repo_root).resolve()) + except Exception as e: + logger.warning(f"[_resolve_base_path][Coherence:Failed] Falling back to default path: {e}") + return fallback_path + # [/DEF:_resolve_base_path:Function] + + # [DEF:_normalize_repo_key:Function] + # @PURPOSE: Convert user/dashboard-provided key to safe filesystem directory name. + # @PRE: repo_key can be None/empty. + # @POST: Returns normalized non-empty key. + # @RETURN: str + def _normalize_repo_key(self, repo_key: Optional[str]) -> str: + raw_key = str(repo_key or "").strip().lower() + normalized = re.sub(r"[^a-z0-9._-]+", "-", raw_key).strip("._-") + return normalized or "dashboard" + # [/DEF:_normalize_repo_key:Function] + + # [DEF:_update_repo_local_path:Function] + # @PURPOSE: Persist repository local_path in GitRepository table when record exists. + # @PRE: dashboard_id is valid integer. + # @POST: local_path is updated for existing record. + # @RETURN: None + def _update_repo_local_path(self, dashboard_id: int, local_path: str) -> None: + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + if db_repo: + db_repo.local_path = local_path + session.commit() + finally: + session.close() + except Exception as e: + logger.warning(f"[_update_repo_local_path][Coherence:Failed] {e}") + # [/DEF:_update_repo_local_path:Function] + + # [DEF:_migrate_repo_directory:Function] + # @PURPOSE: Move legacy repository directory to target path and sync DB metadata. + # @PRE: source_path exists. + # @POST: Repository content available at target_path. + # @RETURN: str + def _migrate_repo_directory(self, dashboard_id: int, source_path: str, target_path: str) -> str: + source_abs = os.path.abspath(source_path) + target_abs = os.path.abspath(target_path) + if source_abs == target_abs: + return source_abs + + if os.path.exists(target_abs): + logger.warning( + f"[_migrate_repo_directory][Action] Target already exists, keeping source path: {target_abs}" + ) + return source_abs + + Path(target_abs).parent.mkdir(parents=True, exist_ok=True) + try: + os.replace(source_abs, target_abs) + except OSError: + shutil.move(source_abs, target_abs) + + self._update_repo_local_path(dashboard_id, target_abs) + logger.info( + f"[_migrate_repo_directory][Coherence:OK] Repository migrated for dashboard {dashboard_id}: {source_abs} -> {target_abs}" + ) + return target_abs + # [/DEF:_migrate_repo_directory:Function] + # [DEF:_get_repo_path:Function] # @PURPOSE: Resolves the local filesystem path for a dashboard's repository. # @PARAM: dashboard_id (int) + # @PARAM: repo_key (Optional[str]) - Slug-like key used when DB local_path is absent. # @PRE: dashboard_id is an integer. - # @POST: Returns the absolute or relative path to the dashboard's repo. + # @POST: Returns DB-local_path when present, otherwise base_path/. # @RETURN: str - def _get_repo_path(self, dashboard_id: int) -> str: + def _get_repo_path(self, dashboard_id: int, repo_key: Optional[str] = None) -> str: with belief_scope("GitService._get_repo_path"): if dashboard_id is None: raise ValueError("dashboard_id cannot be None") - return os.path.join(self.base_path, str(dashboard_id)) + fallback_key = repo_key if repo_key is not None else str(dashboard_id) + normalized_key = self._normalize_repo_key(fallback_key) + target_path = os.path.join(self.base_path, normalized_key) + + try: + session = SessionLocal() + try: + db_repo = ( + session.query(GitRepository) + .filter(GitRepository.dashboard_id == int(dashboard_id)) + .first() + ) + finally: + session.close() + if db_repo and db_repo.local_path: + db_path = os.path.abspath(db_repo.local_path) + if os.path.exists(db_path): + if ( + os.path.abspath(self.base_path) != os.path.abspath(self.legacy_base_path) + and db_path.startswith(os.path.abspath(self.legacy_base_path) + os.sep) + ): + return self._migrate_repo_directory(dashboard_id, db_path, target_path) + return db_path + except Exception as e: + logger.warning(f"[_get_repo_path][Coherence:Failed] Could not resolve local_path from DB: {e}") + + legacy_id_path = os.path.join(self.legacy_base_path, str(dashboard_id)) + if os.path.exists(legacy_id_path) and not os.path.exists(target_path): + return self._migrate_repo_directory(dashboard_id, legacy_id_path, target_path) + + if os.path.exists(target_path): + self._update_repo_local_path(dashboard_id, target_path) + + return target_path # [/DEF:_get_repo_path:Function] # [DEF:init_repo:Function] @@ -62,12 +208,14 @@ class GitService: # @PARAM: dashboard_id (int) # @PARAM: remote_url (str) # @PARAM: pat (str) - Personal Access Token for authentication. + # @PARAM: repo_key (Optional[str]) - Slug-like key for deterministic folder naming on first init. # @PRE: dashboard_id is int, remote_url is valid Git URL, pat is provided. # @POST: Repository is cloned or opened at the local path. # @RETURN: Repo - GitPython Repo object. - def init_repo(self, dashboard_id: int, remote_url: str, pat: str) -> Repo: + def init_repo(self, dashboard_id: int, remote_url: str, pat: str, repo_key: Optional[str] = None) -> Repo: with belief_scope("GitService.init_repo"): - repo_path = self._get_repo_path(dashboard_id) + repo_path = self._get_repo_path(dashboard_id, repo_key=repo_key or str(dashboard_id)) + Path(repo_path).parent.mkdir(parents=True, exist_ok=True) # Inject PAT into remote URL if needed if pat and "://" in remote_url: