From 0cf0ef25f1a6d134b3c148768a581cfa20e480b4 Mon Sep 17 00:00:00 2001 From: busya Date: Fri, 20 Feb 2026 20:47:39 +0300 Subject: [PATCH] db + docker --- .ai/MODULE_MAP.md | 1250 +++++++++++++++++ .ai/PROJECT_MAP.md | 40 +- .ai/ROOT.md | 1 + .dockerignore | 27 + Dockerfile | 36 + README.md | 69 +- backend/src/core/auth/config.py | 7 +- backend/src/core/config_manager.py | 567 ++++---- backend/src/core/config_models.py | 10 +- backend/src/core/database.py | 39 +- backend/src/dependencies.py | 16 +- backend/src/models/config.py | 26 + backend/src/models/storage.py | 2 + .../src/scripts/migrate_sqlite_to_postgres.py | 350 +++++ docker-compose.yml | 42 + generate_semantic_map.py | 164 ++- semantics/semantic_map.json | 638 +++++---- ut | 15 + 18 files changed, 2711 insertions(+), 588 deletions(-) create mode 100644 .ai/MODULE_MAP.md create mode 100644 .dockerignore create mode 100644 Dockerfile mode change 100755 => 100644 backend/src/core/config_manager.py create mode 100644 backend/src/models/config.py create mode 100644 backend/src/scripts/migrate_sqlite_to_postgres.py create mode 100644 docker-compose.yml create mode 100644 ut diff --git a/.ai/MODULE_MAP.md b/.ai/MODULE_MAP.md new file mode 100644 index 0000000..ca7fc00 --- /dev/null +++ b/.ai/MODULE_MAP.md @@ -0,0 +1,1250 @@ +# Module Map + +> High-level module structure for AI Context. Generated automatically. + +**Generated:** 2026-02-20T11:30:24.325166 + +## Summary + +- **Total Modules:** 63 +- **Total Entities:** 1214 + +## Module Hierarchy + + ### 📁 `shots/` + + - 🏗️ **Layers:** Domain (Business Logic), Domain (Core), Interface (API), UI (Presentation) + - 📊 **Tiers:** CRITICAL: 2, STANDARD: 7, TRIVIAL: 1 + - 📄 **Files:** 4 + - 📦 **Entities:** 10 + + **Key Entities:** + + - 🧩 **FrontendComponentShot** (Component) `[CRITICAL]` + - Action button to spawn a new task with full UX feedback cycl... + - 📦 **BackendRouteShot** (Module) + - Reference implementation of a task-based route using GRACE-P... + - 📦 **PluginExampleShot** (Module) + - Reference implementation of a plugin following GRACE standar... + - 📦 **TransactionCore** (Module) `[CRITICAL]` + - Core banking transaction processor with ACID guarantees. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> [DEF:Infra:AuditLog] + - 🔗 DEPENDS_ON -> [DEF:Infra:PostgresDB] + - 🔗 IMPLEMENTS -> [DEF:Std:API_FastAPI] + - 🔗 INHERITS -> PluginBase + +### 📁 `backend/` + +- 🏗️ **Layers:** Unknown, Utility +- 📊 **Tiers:** STANDARD: 2, TRIVIAL: 2 +- 📄 **Files:** 2 +- 📦 **Entities:** 4 + +**Key Entities:** + + - 📦 **backend.delete_running_tasks** (Module) + - Script to delete tasks with RUNNING status from the database... + - 📦 **test_auth_debug** (Module) `[TRIVIAL]` + - Auto-generated module for backend/test_auth_debug.py + + ### 📁 `src/` + + - 🏗️ **Layers:** API, Core, UI (API) + - 📊 **Tiers:** CRITICAL: 2, STANDARD: 18, TRIVIAL: 3 + - 📄 **Files:** 2 + - 📦 **Entities:** 23 + + **Key Entities:** + + - 📦 **AppModule** (Module) `[CRITICAL]` + - The main entry point for the FastAPI application. It initial... + - 📦 **Dependencies** (Module) + - Manages creation and provision of shared application depende... + + ### 📁 `api/` + + - 🏗️ **Layers:** API + - 📊 **Tiers:** STANDARD: 7 + - 📄 **Files:** 1 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 📦 **backend.src.api.auth** (Module) + - Authentication API endpoints. + + ### 📁 `routes/` + + - 🏗️ **Layers:** API, UI (API), Unknown + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 137, TRIVIAL: 3 + - 📄 **Files:** 15 + - 📦 **Entities:** 141 + + **Key Entities:** + + - ℂ **BranchCheckout** (Class) + - Schema for branch checkout requests. + - ℂ **BranchCreate** (Class) + - Schema for branch creation requests. + - ℂ **BranchSchema** (Class) + - Schema for representing a Git branch metadata. + - ℂ **CommitCreate** (Class) + - Schema for staging and committing changes. + - ℂ **CommitSchema** (Class) + - Schema for representing Git commit details. + - ℂ **ConflictResolution** (Class) + - Schema for resolving merge conflicts. + - ℂ **ConnectionCreate** (Class) + - Pydantic model for creating a connection. + - ℂ **ConnectionSchema** (Class) + - Pydantic model for connection response. + - ℂ **ConsolidatedSettingsResponse** (Class) + - ℂ **DeployRequest** (Class) + - Schema for dashboard deployment requests. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> ConfigManager + - 🔗 DEPENDS_ON -> ConfigModels + - 🔗 DEPENDS_ON -> backend.src.core.database + - 🔗 DEPENDS_ON -> backend.src.core.superset_client + - 🔗 DEPENDS_ON -> backend.src.dependencies + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** API + - 📊 **Tiers:** STANDARD: 16, TRIVIAL: 2 + - 📄 **Files:** 2 + - 📦 **Entities:** 18 + + **Key Entities:** + + - 📦 **backend.src.api.routes.__tests__.test_dashboards** (Module) + - Unit tests for Dashboards API endpoints + - 📦 **backend.src.api.routes.__tests__.test_datasets** (Module) + - Unit tests for Datasets API endpoints + + ### 📁 `core/` + + - 🏗️ **Layers:** Core + - 📊 **Tiers:** STANDARD: 109 + - 📄 **Files:** 9 + - 📦 **Entities:** 109 + + **Key Entities:** + + - ℂ **AuthSessionLocal** (Class) + - A session factory for the authentication database. + - ℂ **BeliefFormatter** (Class) + - Custom logging formatter that adds belief state prefixes to ... + - ℂ **ConfigManager** (Class) + - A class to handle application configuration persistence and ... + - ℂ **LogEntry** (Class) + - A Pydantic model representing a single, structured log entry... + - ℂ **MigrationEngine** (Class) + - Engine for transforming Superset export ZIPs. + - ℂ **PluginBase** (Class) + - Defines the abstract base class that all plugins must implem... + - ℂ **PluginConfig** (Class) + - A Pydantic model used to represent the validated configurati... + - ℂ **PluginLoader** (Class) + - Scans a specified directory for Python modules, dynamically ... + - ℂ **SchedulerService** (Class) + - Provides a service to manage scheduled backup tasks. + - ℂ **SessionLocal** (Class) + - A session factory for the main mappings database. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> ConfigModels + - 🔗 DEPENDS_ON -> PyYAML + - 🔗 DEPENDS_ON -> sqlalchemy + + ### 📁 `auth/` + + - 🏗️ **Layers:** Core + - 📊 **Tiers:** STANDARD: 27 + - 📄 **Files:** 6 + - 📦 **Entities:** 27 + + **Key Entities:** + + - ℂ **AuthConfig** (Class) + - Holds authentication-related settings. + - ℂ **AuthRepository** (Class) + - Encapsulates database operations for authentication. + - 📦 **backend.src.core.auth.config** (Module) + - Centralized configuration for authentication and authorizati... + - 📦 **backend.src.core.auth.jwt** (Module) + - JWT token generation and validation logic. + - 📦 **backend.src.core.auth.logger** (Module) + - Audit logging for security-related events. + - 📦 **backend.src.core.auth.oauth** (Module) + - ADFS OIDC configuration and client using Authlib. + - 📦 **backend.src.core.auth.repository** (Module) + - Data access layer for authentication-related entities. + - 📦 **backend.src.core.auth.security** (Module) + - Utility for password hashing and verification using Passlib. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> authlib + - 🔗 DEPENDS_ON -> jose + - 🔗 DEPENDS_ON -> passlib + - 🔗 DEPENDS_ON -> pydantic + - 🔗 DEPENDS_ON -> sqlalchemy + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** Domain + - 📊 **Tiers:** STANDARD: 1, TRIVIAL: 9 + - 📄 **Files:** 1 + - 📦 **Entities:** 10 + + **Key Entities:** + + - 📦 **test_auth** (Module) + - Unit tests for authentication module + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** Infra + - 📊 **Tiers:** STANDARD: 9 + - 📄 **Files:** 1 + - 📦 **Entities:** 9 + + **Key Entities:** + + - 📦 **test_logger** (Module) + - Unit tests for logger module + + ### 📁 `task_manager/` + + - 🏗️ **Layers:** Core + - 📊 **Tiers:** CRITICAL: 7, STANDARD: 63, TRIVIAL: 4 + - 📄 **Files:** 7 + - 📦 **Entities:** 74 + + **Key Entities:** + + - ℂ **LogEntry** (Class) `[CRITICAL]` + - A Pydantic model representing a single, structured log entry... + - ℂ **LogFilter** (Class) + - Filter parameters for querying task logs. + - ℂ **LogStats** (Class) + - Statistics about log entries for a task. + - ℂ **Task** (Class) + - A Pydantic model representing a single execution instance of... + - ℂ **TaskCleanupService** (Class) + - Provides methods to clean up old task records and their asso... + - ℂ **TaskContext** (Class) `[CRITICAL]` + - A container passed to plugin.execute() providing the logger ... + - ℂ **TaskLog** (Class) + - A Pydantic model representing a persisted log entry from the... + - ℂ **TaskLogPersistenceService** (Class) `[CRITICAL]` + - Provides methods to save and query task logs from the task_l... + - ℂ **TaskLogger** (Class) `[CRITICAL]` + - A wrapper around TaskManager._add_log that carries task_id a... + - ℂ **TaskManager** (Class) `[CRITICAL]` + - Manages the lifecycle of tasks, including their creation, ex... + + **Dependencies:** + + - 🔗 DEPENDS_ON -> TaskLogRecord + - 🔗 DEPENDS_ON -> TaskLogger, USED_BY -> plugins + - 🔗 DEPENDS_ON -> TaskManager, CALLS -> TaskManager._add_log + + ### 📁 `utils/` + + - 🏗️ **Layers:** Core, Domain, Infra + - 📊 **Tiers:** STANDARD: 48, TRIVIAL: 1 + - 📄 **Files:** 4 + - 📦 **Entities:** 49 + + **Key Entities:** + + - ℂ **APIClient** (Class) + - Инкапсулирует HTTP-логику для работы с API, включая сессии, ... + - ℂ **AuthenticationError** (Class) + - Exception raised when authentication fails. + - ℂ **DashboardNotFoundError** (Class) + - Exception raised when a dashboard cannot be found. + - ℂ **DatasetMapper** (Class) + - Класс для меппинга и обновления verbose_map в датасетах Supe... + - ℂ **InvalidZipFormatError** (Class) + - Exception raised when a file is not a valid ZIP archive. + - ℂ **NetworkError** (Class) + - Exception raised when a network level error occurs. + - ℂ **PermissionDeniedError** (Class) + - Exception raised when access is denied. + - ℂ **SupersetAPIError** (Class) + - Base exception for all Superset API related errors. + - 📦 **backend.core.utils.dataset_mapper** (Module) + - Этот модуль отвечает за обновление метаданных (verbose_map) ... + - 📦 **backend.core.utils.fileio** (Module) + - Предоставляет набор утилит для управления файловыми операция... + + **Dependencies:** + + - 🔗 DEPENDS_ON -> backend.core.superset_client + - 🔗 DEPENDS_ON -> backend.src.core.logger + - 🔗 DEPENDS_ON -> pandas + - 🔗 DEPENDS_ON -> psycopg2 + - 🔗 DEPENDS_ON -> pyyaml + + ### 📁 `models/` + + - 🏗️ **Layers:** Domain, Model + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 15, TRIVIAL: 17 + - 📄 **Files:** 8 + - 📦 **Entities:** 33 + + **Key Entities:** + + - ℂ **ADGroupMapping** (Class) + - Maps an Active Directory group to a local System Role. + - ℂ **ConnectionConfig** (Class) `[TRIVIAL]` + - Stores credentials for external databases used for column ma... + - ℂ **DashboardMetadata** (Class) `[TRIVIAL]` + - Represents a dashboard available for migration. + - ℂ **DashboardSelection** (Class) `[TRIVIAL]` + - Represents the user's selection of dashboards to migrate. + - ℂ **DatabaseMapping** (Class) + - Represents a mapping between source and target databases. + - ℂ **DeploymentEnvironment** (Class) `[TRIVIAL]` + - Target Superset environments for dashboard deployment. + - ℂ **Environment** (Class) + - Represents a Superset instance environment. + - ℂ **FileCategory** (Class) `[TRIVIAL]` + - Enumeration of supported file categories in the storage syst... + - ℂ **GitRepository** (Class) `[TRIVIAL]` + - Tracking for a local Git repository linked to a dashboard. + - ℂ **GitServerConfig** (Class) `[TRIVIAL]` + - Configuration for a Git server connection. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> Role + - 🔗 DEPENDS_ON -> TaskRecord + - 🔗 DEPENDS_ON -> sqlalchemy + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** Domain + - 📊 **Tiers:** STANDARD: 1, TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 2 + + **Key Entities:** + + - 📦 **test_models** (Module) `[TRIVIAL]` + - Unit tests for data models + + ### 📁 `plugins/` + + - 🏗️ **Layers:** App, Plugin, Plugins + - 📊 **Tiers:** STANDARD: 63 + - 📄 **Files:** 6 + - 📦 **Entities:** 63 + + **Key Entities:** + + - ℂ **BackupPlugin** (Class) + - Implementation of the backup plugin logic. + - ℂ **DebugPlugin** (Class) + - Plugin for system diagnostics and debugging. + - ℂ **GitPlugin** (Class) + - Реализация плагина Git Integration для управления версиями д... + - ℂ **MapperPlugin** (Class) + - Plugin for mapping dataset columns verbose names. + - ℂ **MigrationPlugin** (Class) + - Implementation of the migration plugin logic. + - ℂ **SearchPlugin** (Class) + - Plugin for searching text patterns in Superset datasets. + - 📦 **BackupPlugin** (Module) + - A plugin that provides functionality to back up Superset das... + - 📦 **DebugPluginModule** (Module) + - Implements a plugin for system diagnostics and debugging Sup... + - 📦 **MapperPluginModule** (Module) + - Implements a plugin for mapping dataset columns using extern... + - 📦 **MigrationPlugin** (Module) + - A plugin that provides functionality to migrate Superset das... + + **Dependencies:** + + - 🔗 DEPENDS_ON -> superset_tool.client + - 🔗 DEPENDS_ON -> superset_tool.utils + - 🔗 IMPLEMENTS -> PluginBase + + ### 📁 `git/` + + - 🏗️ **Layers:** Unknown + - 📊 **Tiers:** STANDARD: 2, TRIVIAL: 2 + - 📄 **Files:** 1 + - 📦 **Entities:** 4 + + **Key Entities:** + + - ℂ **GitLLMExtension** (Class) + - Provides LLM capabilities to the Git plugin. + - 📦 **llm_extension** (Module) `[TRIVIAL]` + - Auto-generated module for backend/src/plugins/git/llm_extens... + + ### 📁 `llm_analysis/` + + - 🏗️ **Layers:** Unknown + - 📊 **Tiers:** STANDARD: 18, TRIVIAL: 23 + - 📄 **Files:** 4 + - 📦 **Entities:** 41 + + **Key Entities:** + + - ℂ **DashboardValidationPlugin** (Class) + - Plugin for automated dashboard health analysis using LLMs. + - ℂ **DetectedIssue** (Class) + - Model for a single issue detected during validation. + - ℂ **DocumentationPlugin** (Class) + - Plugin for automated dataset documentation using LLMs. + - ℂ **LLMClient** (Class) + - Wrapper for LLM provider APIs. + - ℂ **LLMProviderConfig** (Class) + - Configuration for an LLM provider. + - ℂ **LLMProviderType** (Class) + - Enum for supported LLM providers. + - ℂ **ScreenshotService** (Class) + - Handles capturing screenshots of Superset dashboards. + - ℂ **ValidationResult** (Class) + - Model for dashboard validation result. + - ℂ **ValidationStatus** (Class) + - Enum for dashboard validation status. + - 📦 **plugin** (Module) `[TRIVIAL]` + - Auto-generated module for backend/src/plugins/llm_analysis/p... + + **Dependencies:** + + - 🔗 IMPLEMENTS -> backend.src.core.plugin_base.PluginBase + + ### 📁 `storage/` + + - 🏗️ **Layers:** App + - 📊 **Tiers:** STANDARD: 18 + - 📄 **Files:** 1 + - 📦 **Entities:** 18 + + **Key Entities:** + + - ℂ **StoragePlugin** (Class) + - Implementation of the storage management plugin. + - 📦 **StoragePlugin** (Module) + - Provides core filesystem operations for managing backups and... + + **Dependencies:** + + - 🔗 DEPENDS_ON -> backend.src.models.storage + - 🔗 IMPLEMENTS -> PluginBase + + ### 📁 `schemas/` + + - 🏗️ **Layers:** API + - 📊 **Tiers:** STANDARD: 10, TRIVIAL: 3 + - 📄 **Files:** 1 + - 📦 **Entities:** 13 + + **Key Entities:** + + - ℂ **ADGroupMappingCreate** (Class) + - Schema for creating an AD Group mapping. + - ℂ **ADGroupMappingSchema** (Class) + - Represents an AD Group to Role mapping in API responses. + - ℂ **PermissionSchema** (Class) `[TRIVIAL]` + - Represents a permission in API responses. + - ℂ **RoleCreate** (Class) + - Schema for creating a new role. + - ℂ **RoleSchema** (Class) + - Represents a role in API responses. + - ℂ **RoleUpdate** (Class) + - Schema for updating an existing role. + - ℂ **Token** (Class) `[TRIVIAL]` + - Represents a JWT access token response. + - ℂ **TokenData** (Class) `[TRIVIAL]` + - Represents the data encoded in a JWT token. + - ℂ **User** (Class) + - Schema for user data in API responses. + - ℂ **UserBase** (Class) + - Base schema for user data. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> pydantic + + ### 📁 `scripts/` + + - 🏗️ **Layers:** Scripts, Unknown + - 📊 **Tiers:** STANDARD: 7, TRIVIAL: 2 + - 📄 **Files:** 4 + - 📦 **Entities:** 9 + + **Key Entities:** + + - 📦 **backend.src.scripts.create_admin** (Module) + - CLI tool for creating the initial admin user. + - 📦 **backend.src.scripts.init_auth_db** (Module) + - Initializes the auth database and creates the necessary tabl... + - 📦 **backend.src.scripts.seed_permissions** (Module) + - Populates the auth database with initial system permissions. + - 📦 **test_dataset_dashboard_relations** (Module) `[TRIVIAL]` + - Auto-generated module for backend/src/scripts/test_dataset_d... + + ### 📁 `services/` + + - 🏗️ **Layers:** Core, Domain, Service + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 50, TRIVIAL: 5 + - 📄 **Files:** 6 + - 📦 **Entities:** 56 + + **Key Entities:** + + - ℂ **AuthService** (Class) + - Provides high-level authentication services. + - ℂ **EncryptionManager** (Class) `[CRITICAL]` + - Handles encryption and decryption of sensitive data like API... + - ℂ **GitService** (Class) + - Wrapper for GitPython operations with semantic logging and e... + - ℂ **LLMProviderService** (Class) + - Service to manage LLM provider lifecycle. + - ℂ **MappingService** (Class) + - Service for handling database mapping logic. + - ℂ **ResourceService** (Class) + - Provides centralized access to resource data with enhanced m... + - 📦 **backend.src.services** (Module) + - Package initialization for services module + - 📦 **backend.src.services.auth_service** (Module) + - Orchestrates authentication business logic. + - 📦 **backend.src.services.git_service** (Module) + - Core Git logic using GitPython to manage dashboard repositor... + - 📦 **backend.src.services.llm_provider** (Module) + - Service for managing LLM provider configurations with encryp... + + **Dependencies:** + + - 🔗 DEPENDS_ON -> backend.src.core.database + - 🔗 DEPENDS_ON -> backend.src.core.superset_client + - 🔗 DEPENDS_ON -> backend.src.core.task_manager + - 🔗 DEPENDS_ON -> backend.src.core.utils.matching + - 🔗 DEPENDS_ON -> backend.src.models.llm + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** Service + - 📊 **Tiers:** STANDARD: 7 + - 📄 **Files:** 1 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 📦 **backend.src.services.__tests__.test_resource_service** (Module) + - Unit tests for ResourceService + + ### 📁 `tests/` + + - 🏗️ **Layers:** Domain (Tests), Test, Unknown + - 📊 **Tiers:** STANDARD: 54, TRIVIAL: 19 + - 📄 **Files:** 7 + - 📦 **Entities:** 73 + + **Key Entities:** + + - ℂ **TestLogPersistence** (Class) + - Test suite for TaskLogPersistenceService. + - ℂ **TestTaskContext** (Class) + - Test suite for TaskContext. + - ℂ **TestTaskLogger** (Class) + - Test suite for TaskLogger. + - 📦 **backend.tests.test_dashboards_api** (Module) + - Contract-driven tests for Dashboard Hub API + - 📦 **test_auth** (Module) `[TRIVIAL]` + - Auto-generated module for backend/tests/test_auth.py + - 📦 **test_log_persistence** (Module) + - Unit tests for TaskLogPersistenceService. + - 📦 **test_resource_hubs** (Module) `[TRIVIAL]` + - Auto-generated module for backend/tests/test_resource_hubs.p... + - 📦 **test_task_logger** (Module) + - Unit tests for TaskLogger and TaskContext. + + ### 📁 `components/` + + - 🏗️ **Layers:** Component, Feature, UI, Unknown + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 45, TRIVIAL: 7 + - 📄 **Files:** 13 + - 📦 **Entities:** 53 + + **Key Entities:** + + - 🧩 **DashboardGrid** (Component) + - Displays a grid of dashboards with selection and pagination. + - 🧩 **DynamicForm** (Component) + - Generates a form dynamically based on a JSON schema. + - 🧩 **EnvSelector** (Component) + - Provides a UI component for selecting source and target envi... + - 🧩 **Footer** (Component) `[TRIVIAL]` + - Displays the application footer with copyright information. + - 🧩 **MappingTable** (Component) + - Displays and allows editing of database mappings. + - 🧩 **MissingMappingModal** (Component) + - Prompts the user to provide a database mapping when one is m... + - 🧩 **Navbar** (Component) + - Main navigation bar for the application. + - 🧩 **PasswordPrompt** (Component) + - A modal component to prompt the user for database passwords ... + - 🧩 **TaskHistory** (Component) + - Displays a list of recent tasks with their status and allows... + - 🧩 **TaskList** (Component) + - Displays a list of tasks with their status and execution det... + + ### 📁 `auth/` + + - 🏗️ **Layers:** Component + - 📊 **Tiers:** TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 1 + + **Key Entities:** + + - 🧩 **ProtectedRoute** (Component) `[TRIVIAL]` + - Wraps content to ensure only authenticated users can access ... + + ### 📁 `git/` + + - 🏗️ **Layers:** Component + - 📊 **Tiers:** STANDARD: 26 + - 📄 **Files:** 6 + - 📦 **Entities:** 26 + + **Key Entities:** + + - 🧩 **BranchSelector** (Component) + - UI для выбора и создания веток Git. + - 🧩 **CommitHistory** (Component) + - Displays the commit history for a specific dashboard. + - 🧩 **CommitModal** (Component) + - Модальное окно для создания коммита с просмотром изменений (... + - 🧩 **ConflictResolver** (Component) + - UI for resolving merge conflicts (Keep Mine / Keep Theirs). + - 🧩 **DeploymentModal** (Component) + - Modal for deploying a dashboard to a target environment. + - 🧩 **GitManager** (Component) + - Центральный компонент для управления Git-операциями конкретн... + + ### 📁 `llm/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** STANDARD: 2, TRIVIAL: 10 + - 📄 **Files:** 3 + - 📦 **Entities:** 12 + + **Key Entities:** + + - 🧩 **DocPreview** (Component) + - UI component for previewing generated dataset documentation ... + - 🧩 **ProviderConfig** (Component) + - UI form for managing LLM provider configurations. + - 📦 **DocPreview** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/llm/DocPre... + - 📦 **ProviderConfig** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/llm/Provid... + - 📦 **ValidationReport** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/llm/Valida... + + ### 📁 `storage/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** STANDARD: 7 + - 📄 **Files:** 2 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 🧩 **FileList** (Component) + - Displays a table of files with metadata and actions. + - 🧩 **FileUpload** (Component) + - Provides a form for uploading files to a specific category. + + ### 📁 `tasks/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** STANDARD: 4, TRIVIAL: 10 + - 📄 **Files:** 3 + - 📦 **Entities:** 14 + + **Key Entities:** + + - 🧩 **LogEntryRow** (Component) + - Renders a single log entry with stacked layout optimized for... + - 🧩 **LogFilterBar** (Component) + - Compact filter toolbar for logs — level, source, and text se... + - 🧩 **TaskLogPanel** (Component) + - Combines log filtering and display into a single cohesive da... + - 📦 **LogFilterBar** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/tasks/LogF... + - 📦 **TaskLogPanel** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/tasks/Task... + + ### 📁 `tools/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** STANDARD: 14, TRIVIAL: 2 + - 📄 **Files:** 4 + - 📦 **Entities:** 16 + + **Key Entities:** + + - 🧩 **ConnectionForm** (Component) + - UI component for creating a new database connection configur... + - 🧩 **ConnectionList** (Component) + - UI component for listing and deleting saved database connect... + - 🧩 **DebugTool** (Component) + - UI component for system diagnostics and debugging API respon... + - 🧩 **MapperTool** (Component) + - UI component for mapping dataset column verbose names using ... + - 📦 **MapperTool** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/components/tools/Mapp... + + ### 📁 `lib/` + + - 🏗️ **Layers:** Infra, Infra-API, UI, UI-State + - 📊 **Tiers:** STANDARD: 20, TRIVIAL: 3 + - 📄 **Files:** 5 + - 📦 **Entities:** 23 + + **Key Entities:** + + - 🧩 **Counter** (Component) `[TRIVIAL]` + - Simple counter demo component + - 📦 **Utils** (Module) `[TRIVIAL]` + - General utility functions (class merging) + - 📦 **api_module** (Module) + - Handles all communication with the backend API. + - 📦 **stores_module** (Module) + - Global state management using Svelte stores. + - 📦 **toasts_module** (Module) + - Manages toast notifications using a Svelte writable store. + + ### 📁 `auth/` + + - 🏗️ **Layers:** Feature + - 📊 **Tiers:** STANDARD: 7 + - 📄 **Files:** 1 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 🗄️ **authStore** (Store) + - Manages the global authentication state on the frontend. + + ### 📁 `layout/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** CRITICAL: 3, STANDARD: 4, TRIVIAL: 23 + - 📄 **Files:** 4 + - 📦 **Entities:** 30 + + **Key Entities:** + + - 🧩 **Breadcrumbs** (Component) + - Display page hierarchy navigation + - 🧩 **Sidebar** (Component) `[CRITICAL]` + - Persistent left sidebar with resource categories navigation + - 🧩 **TaskDrawer** (Component) `[CRITICAL]` + - Global task drawer for monitoring background operations + - 🧩 **TopNavbar** (Component) `[CRITICAL]` + - Unified top navigation bar with Logo, Search, Activity, and ... + - 📦 **Breadcrumbs** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/components/layout... + - 📦 **Sidebar** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/components/layout... + - 📦 **TaskDrawer** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/components/layout... + - 📦 **TopNavbar** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/components/layout... + + ### 📁 `i18n/` + + - 🏗️ **Layers:** Infra + - 📊 **Tiers:** STANDARD: 4 + - 📄 **Files:** 1 + - 📦 **Entities:** 4 + + **Key Entities:** + + - 📦 **i18n** (Module) + - Determines the starting locale. + - 🗄️ **locale** (Store) + - Holds the current active locale string. + - 🗄️ **t** (Store) + - Derived store providing the translation dictionary. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> locales/en.json + - 🔗 DEPENDS_ON -> locales/ru.json + + ### 📁 `stores/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 2, TRIVIAL: 12 + - 📄 **Files:** 3 + - 📦 **Entities:** 15 + + **Key Entities:** + + - 📦 **sidebar** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/stores/sidebar.js + - 📦 **taskDrawer** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/lib/stores/taskDrawer... + - 🗄️ **activity** (Store) + - Track active task count for navbar indicator + - 🗄️ **sidebar** (Store) + - Manage sidebar visibility and navigation state + - 🗄️ **taskDrawer** (Store) `[CRITICAL]` + - Manage Task Drawer visibility and resource-to-task mapping + + **Dependencies:** + + - 🔗 DEPENDS_ON -> WebSocket connection, taskDrawer store + + ### 📁 `__tests__/` + + - 🏗️ **Layers:** Domain (Tests), UI + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 8 + - 📄 **Files:** 5 + - 📦 **Entities:** 9 + + **Key Entities:** + + - 📦 **frontend.src.lib.stores.__tests__.sidebar** (Module) + - Unit tests for sidebar store + - 📦 **frontend.src.lib.stores.__tests__.test_activity** (Module) + - Unit tests for activity store + - 📦 **frontend.src.lib.stores.__tests__.test_sidebar** (Module) + - Unit tests for sidebar store + - 📦 **frontend.src.lib.stores.__tests__.test_taskDrawer** (Module) `[CRITICAL]` + - Unit tests for task drawer store + - 📦 **setupTests** (Module) + - Global test setup with mocks for SvelteKit modules + + **Dependencies:** + + - 🔗 DEPENDS_ON -> frontend.src.lib.stores.taskDrawer + + ### 📁 `mocks/` + + - 📊 **Tiers:** STANDARD: 3 + - 📄 **Files:** 3 + - 📦 **Entities:** 3 + + ### 📁 `ui/` + + - 🏗️ **Layers:** Atom + - 📊 **Tiers:** TRIVIAL: 7 + - 📄 **Files:** 7 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 🧩 **Button** (Component) `[TRIVIAL]` + - Define component interface and default values (Svelte 5 Rune... + - 🧩 **Card** (Component) `[TRIVIAL]` + - Standardized container with padding and elevation. + - 🧩 **Input** (Component) `[TRIVIAL]` + - Standardized text input component with label and error handl... + - 🧩 **LanguageSwitcher** (Component) `[TRIVIAL]` + - Dropdown component to switch between supported languages. + - 🧩 **PageHeader** (Component) `[TRIVIAL]` + - Standardized page header with title and action area. + - 🧩 **Select** (Component) `[TRIVIAL]` + - Standardized dropdown selection component. + - 📦 **ui** (Module) `[TRIVIAL]` + - Central export point for standardized UI components. + + ### 📁 `utils/` + + - 🏗️ **Layers:** Infra + - 📊 **Tiers:** TRIVIAL: 2 + - 📄 **Files:** 1 + - 📦 **Entities:** 2 + + **Key Entities:** + + - 📦 **Debounce** (Module) `[TRIVIAL]` + - Debounce utility for limiting function execution rate + + ### 📁 `pages/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** STANDARD: 11 + - 📄 **Files:** 2 + - 📦 **Entities:** 11 + + **Key Entities:** + + - 🧩 **Dashboard** (Component) + - Displays the list of available plugins and allows selecting ... + - 🧩 **Settings** (Component) + - The main settings page for the application, allowing managem... + + ### 📁 `routes/` + + - 🏗️ **Layers:** Infra, UI + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 3, TRIVIAL: 1 + - 📄 **Files:** 5 + - 📦 **Entities:** 5 + + **Key Entities:** + + - 📦 **RootLayoutConfig** (Module) `[TRIVIAL]` + - Root layout configuration (SPA mode) + - 📦 **layout** (Module) + + ### 📁 `roles/` + + - 🏗️ **Layers:** Domain + - 📊 **Tiers:** STANDARD: 6 + - 📄 **Files:** 1 + - 📦 **Entities:** 6 + + **Key Entities:** + + - 🧩 **AdminRolesPage** (Component) + - UI for managing system roles and their permissions. + + ### 📁 `settings/` + + - 🏗️ **Layers:** Feature + - 📊 **Tiers:** STANDARD: 5 + - 📄 **Files:** 1 + - 📦 **Entities:** 5 + + **Key Entities:** + + - 🧩 **AdminSettingsPage** (Component) + - UI for configuring Active Directory Group to local Role mapp... + + ### 📁 `llm/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** STANDARD: 1, TRIVIAL: 2 + - 📄 **Files:** 1 + - 📦 **Entities:** 3 + + **Key Entities:** + + - 🧩 **LLMSettingsPage** (Component) + - Admin settings page for LLM provider configuration. + - 📦 **+page** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/routes/admin/settings... + + ### 📁 `users/` + + - 🏗️ **Layers:** Feature + - 📊 **Tiers:** STANDARD: 6 + - 📄 **Files:** 1 + - 📦 **Entities:** 6 + + **Key Entities:** + + - 🧩 **AdminUsersPage** (Component) + - UI for managing system users and their roles. + + ### 📁 `datasets/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** CRITICAL: 1, TRIVIAL: 17 + - 📄 **Files:** 1 + - 📦 **Entities:** 18 + + **Key Entities:** + + - 📦 **+page** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/routes/datasets/+page... + + ### 📁 `[id]/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** CRITICAL: 1, TRIVIAL: 6 + - 📄 **Files:** 1 + - 📦 **Entities:** 7 + + **Key Entities:** + + - 📦 **+page** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/routes/datasets/[id]/... + + ### 📁 `git/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** STANDARD: 3 + - 📄 **Files:** 1 + - 📦 **Entities:** 3 + + **Key Entities:** + + - 🧩 **GitDashboardPage** (Component) + - Dashboard management page for Git integration. + + ### 📁 `login/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** STANDARD: 3 + - 📄 **Files:** 1 + - 📦 **Entities:** 3 + + **Key Entities:** + + - 🧩 **LoginPage** (Component) + - Provides the user interface for local and ADFS authenticatio... + + ### 📁 `migration/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** STANDARD: 10 + - 📄 **Files:** 1 + - 📦 **Entities:** 10 + + **Key Entities:** + + - 🧩 **DashboardSelectionSection** (Component) + - 🧩 **MigrationDashboard** (Component) + - Main dashboard for configuring and starting migrations. + + ### 📁 `mappings/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** STANDARD: 4 + - 📄 **Files:** 1 + - 📦 **Entities:** 4 + + **Key Entities:** + + - 🧩 **MappingManagement** (Component) + - Page for managing database mappings between environments. + + ### 📁 `settings/` + + - 🏗️ **Layers:** UI, Unknown + - 📊 **Tiers:** CRITICAL: 1, STANDARD: 1, TRIVIAL: 10 + - 📄 **Files:** 2 + - 📦 **Entities:** 12 + + **Key Entities:** + + - 📦 **+page** (Module) `[TRIVIAL]` + - Auto-generated module for frontend/src/routes/settings/+page... + + ### 📁 `connections/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** STANDARD: 2 + - 📄 **Files:** 1 + - 📦 **Entities:** 2 + + **Key Entities:** + + - 🧩 **ConnectionsSettingsPage** (Component) + - Page for managing database connection configurations. + + ### 📁 `git/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** STANDARD: 5 + - 📄 **Files:** 1 + - 📦 **Entities:** 5 + + **Key Entities:** + + - 🧩 **GitSettingsPage** (Component) + - Manage Git server configurations for dashboard versioning. + + ### 📁 `storage/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 1 + + ### 📁 `repos/` + + - 📊 **Tiers:** STANDARD: 3 + - 📄 **Files:** 1 + - 📦 **Entities:** 3 + + ### 📁 `tasks/` + + - 🏗️ **Layers:** Page + - 📊 **Tiers:** STANDARD: 5 + - 📄 **Files:** 1 + - 📦 **Entities:** 5 + + **Key Entities:** + + - 🧩 **TaskManagementPage** (Component) + - Page for managing and monitoring tasks. + + ### 📁 `debug/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 1 + + **Key Entities:** + + - 🧩 **DebugPage** (Component) `[TRIVIAL]` + - Page for system diagnostics and debugging. + + ### 📁 `mapper/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 1 + + **Key Entities:** + + - 🧩 **MapperPage** (Component) `[TRIVIAL]` + - Page for the dataset column mapper tool. + + ### 📁 `storage/` + + - 🏗️ **Layers:** UI + - 📊 **Tiers:** STANDARD: 5 + - 📄 **Files:** 1 + - 📦 **Entities:** 5 + + **Key Entities:** + + - 🧩 **StoragePage** (Component) + - Main page for file storage management. + + ### 📁 `services/` + + - 🏗️ **Layers:** Service + - 📊 **Tiers:** STANDARD: 33 + - 📄 **Files:** 6 + - 📦 **Entities:** 33 + + **Key Entities:** + + - 📦 **GitServiceClient** (Module) + - API client for Git operations, managing the communication be... + - 📦 **adminService** (Module) + - Service for Admin-related API calls (User and Role managemen... + - 📦 **storageService** (Module) + - Frontend API client for file storage management. + + **Dependencies:** + + - 🔗 DEPENDS_ON -> frontend.src.lib.api + + ### 📁 `types/` + + - 🏗️ **Layers:** Domain + - 📊 **Tiers:** TRIVIAL: 1 + - 📄 **Files:** 1 + - 📦 **Entities:** 1 + + **Key Entities:** + + - 📦 **DashboardTypes** (Module) `[TRIVIAL]` + - TypeScript interfaces for Dashboard entities + +### 📁 `root/` + +- 🏗️ **Layers:** DevOps/Tooling +- 📊 **Tiers:** CRITICAL: 12, STANDARD: 16, TRIVIAL: 7 +- 📄 **Files:** 1 +- 📦 **Entities:** 35 + +**Key Entities:** + + - ℂ **ComplianceIssue** (Class) `[TRIVIAL]` + - Represents a single compliance issue with severity. + - ℂ **SemanticEntity** (Class) `[CRITICAL]` + - Represents a code entity (Module, Function, Component) found... + - ℂ **SemanticMapGenerator** (Class) `[CRITICAL]` + - Orchestrates the mapping process with tier-based validation. + - ℂ **Severity** (Class) `[TRIVIAL]` + - Severity levels for compliance issues. + - ℂ **Tier** (Class) `[TRIVIAL]` + - Enumeration of semantic tiers defining validation strictness... + - 📦 **generate_semantic_map** (Module) `[CRITICAL]` + - Scans the codebase to generate a Semantic Map, Module Map, a... + +## Cross-Module Dependencies + +```mermaid +graph TD + api-->|USES|backend + api-->|USES|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|USES|backend + routes-->|USES|backend + routes-->|CALLS|backend + routes-->|CALLS|backend + routes-->|CALLS|backend + routes-->|CALLS|backend + routes-->|CALLS|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + routes-->|DEPENDS_ON|backend + __tests__-->|TESTS|backend + __tests__-->|TESTS|backend + core-->|USES|backend + core-->|USES|backend + core-->|USES|backend + core-->|USES|backend + auth-->|USES|backend + auth-->|USES|backend + auth-->|USES|backend + auth-->|USES|backend + utils-->|DEPENDS_ON|backend + utils-->|DEPENDS_ON|backend + utils-->|DEPENDS_ON|backend + models-->|INHERITS_FROM|backend + models-->|USED_BY|backend + models-->|INHERITS_FROM|backend + llm_analysis-->|IMPLEMENTS|backend + llm_analysis-->|IMPLEMENTS|backend + storage-->|DEPENDS_ON|backend + scripts-->|USES|backend + scripts-->|USES|backend + scripts-->|CALLS|backend + scripts-->|USES|backend + scripts-->|USES|backend + scripts-->|USES|backend + services-->|DEPENDS_ON|backend + services-->|DEPENDS_ON|backend + services-->|DEPENDS_ON|backend + services-->|DEPENDS_ON|backend + services-->|DEPENDS_ON|backend + services-->|USES|backend + services-->|USES|backend + services-->|USES|backend + services-->|DEPENDS_ON|backend + services-->|DEPENDS_ON|backend + __tests__-->|TESTS|backend + tests-->|TESTS|backend +``` diff --git a/.ai/PROJECT_MAP.md b/.ai/PROJECT_MAP.md index 173cdee..b330c9d 100644 --- a/.ai/PROJECT_MAP.md +++ b/.ai/PROJECT_MAP.md @@ -3,7 +3,7 @@ > Compressed view for AI Context. Generated automatically. - 📦 **generate_semantic_map** (`Module`) `[CRITICAL]` - - 📝 Scans the codebase to generate a Semantic Map and Compliance Report based on the System Standard. + - 📝 Scans the codebase to generate a Semantic Map, Module Map, and Compliance Report based on the System Standard. - 🏗️ Layer: DevOps/Tooling - 🔒 Invariant: All DEF anchors must have matching closing anchors; TIER determines validation strictness. - ƒ **__init__** (`Function`) `[TRIVIAL]` @@ -71,35 +71,50 @@ - 📝 Generates the token-optimized project map with enhanced Svelte details. - ƒ **_write_entity_md** (`Function`) `[CRITICAL]` - 📝 Recursive helper to write entity tree to Markdown with tier badges and enhanced details. + - ƒ **_generate_module_map** (`Function`) `[CRITICAL]` + - 📝 Generates a module-centric map grouping entities by directory structure. + - ƒ **_get_module_path** (`Function`) + - 📝 Extracts the module path from a file path. + - ƒ **_collect_all_entities** (`Function`) + - 📝 Flattens entity tree for easier grouping. - ƒ **to_dict** (`Function`) `[TRIVIAL]` - 📝 Auto-detected function (orphan) +- 📦 **TransactionCore** (`Module`) `[CRITICAL]` + - 📝 Core banking transaction processor with ACID guarantees. + - 🏗️ Layer: Domain (Core) + - 🔒 Invariant: Negative transfers are strictly forbidden. + - 🔗 DEPENDS_ON -> `[DEF:Infra:PostgresDB]` + - 🔗 DEPENDS_ON -> `[DEF:Infra:AuditLog]` + - ƒ **execute_transfer** (`Function`) + - 📝 Atomically move funds between accounts with audit trails. + - 🔗 CALLS -> `atomic_transaction` - 📦 **PluginExampleShot** (`Module`) - 📝 Reference implementation of a plugin following GRACE standards. - - 🔗 IMPLEMENTS -> `[DEF:Std:Plugin]` + - 🏗️ Layer: Domain (Business Logic) + - 🔒 Invariant: get_schema must return valid JSON Schema. + - 🔗 INHERITS -> `PluginBase` - ƒ **get_schema** (`Function`) - 📝 Defines input validation schema. - ƒ **execute** (`Function`) - - 📝 Core plugin logic with structured logging and progress reporting. + - 📝 Core plugin logic with structured logging and scope isolation. - ƒ **id** (`Function`) `[TRIVIAL]` - 📝 Auto-detected function (orphan) - - ƒ **name** (`Function`) `[TRIVIAL]` - - 📝 Auto-detected function (orphan) - - ƒ **description** (`Function`) `[TRIVIAL]` - - 📝 Auto-detected function (orphan) - - ƒ **version** (`Function`) `[TRIVIAL]` - - 📝 Auto-detected function (orphan) - 📦 **BackendRouteShot** (`Module`) - 📝 Reference implementation of a task-based route using GRACE-Poly. + - 🏗️ Layer: Interface (API) + - 🔒 Invariant: TaskManager must be available in dependency graph. - 🔗 IMPLEMENTS -> `[DEF:Std:API_FastAPI]` - ƒ **create_task** (`Function`) - 📝 Create and start a new task using TaskManager. Non-blocking. - 🔗 CALLS -> `task_manager.create_task` -- 🧩 **FrontendComponentShot** (`Component`) - - 📝 Reference implementation of a task-spawning component using - - 🏗️ Layer: UI +- 🧩 **FrontendComponentShot** (`Component`) `[CRITICAL]` + - 📝 Action button to spawn a new task with full UX feedback cycle. + - 🏗️ Layer: UI (Presentation) + - 🔒 Invariant: Must prevent double-submission while loading. - 📥 Props: plugin_id: any, params: any - ⬅️ READS_FROM `lib` - ⬅️ READS_FROM `t` + - ƒ **spawnTask** (`Function`) - 📦 **DashboardTypes** (`Module`) `[TRIVIAL]` - 📝 TypeScript interfaces for Dashboard entities - 🏗️ Layer: Domain @@ -746,6 +761,7 @@ - 🏗️ Layer: UI - ⚡ Events: cancel, resume - ➡️ WRITES_TO `props` + - ➡️ WRITES_TO `state` - ⬅️ READS_FROM `effect` - ƒ **handleSubmit** (`Function`) - 📝 Validates and dispatches the passwords to resume the task. diff --git a/.ai/ROOT.md b/.ai/ROOT.md index c01585a..99b05f4 100644 --- a/.ai/ROOT.md +++ b/.ai/ROOT.md @@ -30,6 +30,7 @@ Use these for code generation (Style Transfer). * Ref: `.ai/shots/critical_module.py` -> `[DEF:Shot:Critical_Module]` ## 3. DOMAIN MAP (Modules) +* **Module Map:** `.ai/MODULE_MAP.md` -> `[DEF:Module_Map]` * **Project Map:** `.ai/PROJECT_MAP.md` -> `[DEF:Project_Map]` * **Backend Core:** `backend/src/core` -> `[DEF:Module:Backend_Core]` * **Backend API:** `backend/src/api` -> `[DEF:Module:Backend_API]` diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..21834da --- /dev/null +++ b/.dockerignore @@ -0,0 +1,27 @@ +.git +.gitignore +.pytest_cache +.ruff_cache +.vscode +.ai +.specify +.kilocode +venv +backend/.venv +backend/.pytest_cache +frontend/node_modules +frontend/.svelte-kit +frontend/.vite +frontend/build +backend/__pycache__ +backend/src/__pycache__ +backend/tests/__pycache__ +**/__pycache__ +*.pyc +*.pyo +*.pyd +*.db +*.log +backups +semantics +specs diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..cfd2734 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +# Stage 1: Build frontend static assets +FROM node:20-alpine AS frontend-build +WORKDIR /app/frontend + +COPY frontend/package*.json ./ +RUN npm ci + +COPY frontend/ ./ +RUN npm run build + + +# Stage 2: Runtime image for backend + static frontend +FROM python:3.11-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 +ENV PYTHONUNBUFFERED=1 +ENV BACKEND_PORT=8000 + +WORKDIR /app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + curl \ + git \ + && rm -rf /var/lib/apt/lists/* + +COPY backend/requirements.txt /app/backend/requirements.txt +RUN pip install --no-cache-dir -r /app/backend/requirements.txt + +COPY backend/ /app/backend/ +COPY --from=frontend-build /app/frontend/build /app/frontend/build + +WORKDIR /app/backend + +EXPOSE 8000 + +CMD ["python", "-m", "uvicorn", "src.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/README.md b/README.md index 0708429..e4bbca7 100755 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ ## Технологический стек - **Backend**: Python 3.9+, FastAPI, SQLAlchemy, APScheduler, Pydantic. - **Frontend**: Node.js 18+, SvelteKit, Tailwind CSS. -- **Database**: SQLite (для хранения метаданных, задач и настроек доступа). +- **Database**: PostgreSQL (для хранения метаданных, задач, логов и конфигурации). ## Структура проекта - `backend/` — Серверная часть, API и логика плагинов. @@ -58,11 +58,15 @@ - `--skip-install`: Пропустить установку зависимостей. - `--help`: Показать справку. -Переменные окружения: -- `BACKEND_PORT`: Порт API (по умолчанию 8000). -- `FRONTEND_PORT`: Порт UI (по умолчанию 5173). +Переменные окружения: +- `BACKEND_PORT`: Порт API (по умолчанию 8000). +- `FRONTEND_PORT`: Порт UI (по умолчанию 5173). +- `POSTGRES_URL`: Базовый URL PostgreSQL по умолчанию для всех подсистем. +- `DATABASE_URL`: URL основной БД (если не задан, используется `POSTGRES_URL`). +- `TASKS_DATABASE_URL`: URL БД задач/логов (если не задан, используется `DATABASE_URL`). +- `AUTH_DATABASE_URL`: URL БД авторизации (если не задан, используется PostgreSQL дефолт). -## Разработка +## Разработка Проект следует строгим правилам разработки: 1. **Semantic Code Generation**: Использование протокола `.ai/standards/semantics.md` для обеспечения надежности кода. 2. **Design by Contract (DbC)**: Определение предусловий и постусловий для ключевых функций. @@ -71,7 +75,54 @@ ### Полезные команды - **Backend**: `cd backend && .venv/bin/python3 -m uvicorn src.app:app --reload` - **Frontend**: `cd frontend && npm run dev` -- **Тесты**: `cd backend && .venv/bin/pytest` - -## Контакты и вклад -Для добавления новых функций или исправления ошибок, пожалуйста, ознакомьтесь с `docs/plugin_dev.md` и создайте соответствующую спецификацию в `specs/`. +- **Тесты**: `cd backend && .venv/bin/pytest` + +## Docker и CI/CD +### Локальный запуск в Docker (приложение + PostgreSQL) +```bash +docker compose up --build +``` + +После старта: +- UI/API: `http://localhost:8000` +- PostgreSQL: `localhost:5432` (`postgres/postgres`, DB `ss_tools`) + +Остановить: +```bash +docker compose down +``` + +Полная очистка тома БД: +```bash +docker compose down -v +``` + +Если `postgres:16-alpine` не тянется из Docker Hub (TLS timeout), используйте fallback image: +```bash +POSTGRES_IMAGE=mirror.gcr.io/library/postgres:16-alpine docker compose up -d db +``` +или: +```bash +POSTGRES_IMAGE=bitnami/postgresql:latest docker compose up -d db +``` +Если на хосте уже занят `5432`, поднимайте Postgres на другом порту: +```bash +POSTGRES_HOST_PORT=5433 docker compose up -d db +``` + +### Миграция legacy-данных в PostgreSQL +Если нужно перенести старые данные из `tasks.db`/`config.json`: +```bash +cd backend +PYTHONPATH=. .venv/bin/python src/scripts/migrate_sqlite_to_postgres.py --sqlite-path tasks.db +``` + +### CI/CD +Добавлен workflow: `.github/workflows/ci-cd.yml` +- backend smoke tests +- frontend build +- docker build +- push образа в GHCR на `main/master` + +## Контакты и вклад +Для добавления новых функций или исправления ошибок, пожалуйста, ознакомьтесь с `docs/plugin_dev.md` и создайте соответствующую спецификацию в `specs/`. diff --git a/backend/src/core/auth/config.py b/backend/src/core/auth/config.py index 656fd19..2fd4a74 100644 --- a/backend/src/core/auth/config.py +++ b/backend/src/core/auth/config.py @@ -24,7 +24,10 @@ class AuthConfig(BaseSettings): REFRESH_TOKEN_EXPIRE_DAYS: int = 7 # Database Settings - AUTH_DATABASE_URL: str = Field(default="sqlite:///./backend/auth.db", env="AUTH_DATABASE_URL") + AUTH_DATABASE_URL: str = Field( + default="postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools", + env="AUTH_DATABASE_URL", + ) # ADFS Settings ADFS_CLIENT_ID: str = Field(default="", env="ADFS_CLIENT_ID") @@ -41,4 +44,4 @@ class AuthConfig(BaseSettings): auth_config = AuthConfig() # [/DEF:auth_config:Variable] -# [/DEF:backend.src.core.auth.config:Module] \ No newline at end of file +# [/DEF:backend.src.core.auth.config:Module] diff --git a/backend/src/core/config_manager.py b/backend/src/core/config_manager.py old mode 100755 new mode 100644 index ccc9a21..f69eeed --- a/backend/src/core/config_manager.py +++ b/backend/src/core/config_manager.py @@ -1,284 +1,283 @@ -# [DEF:ConfigManagerModule:Module] -# -# @SEMANTICS: config, manager, persistence, json -# @PURPOSE: Manages application configuration, including loading/saving to JSON and CRUD for environments. -# @LAYER: Core -# @RELATION: DEPENDS_ON -> ConfigModels -# @RELATION: CALLS -> logger -# @RELATION: WRITES_TO -> config.json -# -# @INVARIANT: Configuration must always be valid according to AppConfig model. -# @PUBLIC_API: ConfigManager - -# [SECTION: IMPORTS] -import json -import os -from pathlib import Path -from typing import Optional, List -from .config_models import AppConfig, Environment, GlobalSettings, StorageConfig -from .logger import logger, configure_logger, belief_scope -# [/SECTION] - -# [DEF:ConfigManager:Class] -# @PURPOSE: A class to handle application configuration persistence and management. -# @RELATION: WRITES_TO -> config.json -class ConfigManager: - - # [DEF:__init__:Function] - # @PURPOSE: Initializes the ConfigManager. - # @PRE: isinstance(config_path, str) and len(config_path) > 0 - # @POST: self.config is an instance of AppConfig - # @PARAM: config_path (str) - Path to the configuration file. - def __init__(self, config_path: str = "config.json"): - with belief_scope("__init__"): - # 1. Runtime check of @PRE - assert isinstance(config_path, str) and config_path, "config_path must be a non-empty string" - - logger.info(f"[ConfigManager][Entry] Initializing with {config_path}") - - # 2. Logic implementation - self.config_path = Path(config_path) - self.config: AppConfig = self._load_config() - - # Configure logger with loaded settings - configure_logger(self.config.settings.logging) - - # 3. Runtime check of @POST - assert isinstance(self.config, AppConfig), "self.config must be an instance of AppConfig" - - logger.info("[ConfigManager][Exit] Initialized") - # [/DEF:__init__:Function] - - # [DEF:_load_config:Function] - # @PURPOSE: Loads the configuration from disk or creates a default one. - # @PRE: self.config_path is set. - # @POST: isinstance(return, AppConfig) - # @RETURN: AppConfig - The loaded or default configuration. - def _load_config(self) -> AppConfig: - with belief_scope("_load_config"): - logger.debug(f"[_load_config][Entry] Loading from {self.config_path}") - - if not self.config_path.exists(): - logger.info("[_load_config][Action] Config file not found. Creating default.") - default_config = AppConfig( - environments=[], - settings=GlobalSettings() - ) - self._save_config_to_disk(default_config) - return default_config - try: - with open(self.config_path, "r") as f: - data = json.load(f) - - # Check for deprecated field - if "settings" in data and "backup_path" in data["settings"]: - del data["settings"]["backup_path"] - - config = AppConfig(**data) - logger.info("[_load_config][Coherence:OK] Configuration loaded") - return config - except Exception as e: - logger.error(f"[_load_config][Coherence:Failed] Error loading config: {e}") - # Fallback but try to preserve existing settings if possible? - # For now, return default to be safe, but log the error prominently. - return AppConfig( - environments=[], - settings=GlobalSettings(storage=StorageConfig()) - ) - # [/DEF:_load_config:Function] - - # [DEF:_save_config_to_disk:Function] - # @PURPOSE: Saves the provided configuration object to disk. - # @PRE: isinstance(config, AppConfig) - # @POST: Configuration saved to disk. - # @PARAM: config (AppConfig) - The configuration to save. - def _save_config_to_disk(self, config: AppConfig): - with belief_scope("_save_config_to_disk"): - logger.debug(f"[_save_config_to_disk][Entry] Saving to {self.config_path}") - - # 1. Runtime check of @PRE - assert isinstance(config, AppConfig), "config must be an instance of AppConfig" - - # 2. Logic implementation - try: - with open(self.config_path, "w") as f: - json.dump(config.dict(), f, indent=4) - logger.info("[_save_config_to_disk][Action] Configuration saved") - except Exception as e: - logger.error(f"[_save_config_to_disk][Coherence:Failed] Failed to save: {e}") - # [/DEF:_save_config_to_disk:Function] - - # [DEF:save:Function] - # @PURPOSE: Saves the current configuration state to disk. - # @PRE: self.config is set. - # @POST: self._save_config_to_disk called. - def save(self): - with belief_scope("save"): - self._save_config_to_disk(self.config) - # [/DEF:save:Function] - - # [DEF:get_config:Function] - # @PURPOSE: Returns the current configuration. - # @PRE: self.config is set. - # @POST: Returns self.config. - # @RETURN: AppConfig - The current configuration. - def get_config(self) -> AppConfig: - with belief_scope("get_config"): - return self.config - # [/DEF:get_config:Function] - - # [DEF:update_global_settings:Function] - # @PURPOSE: Updates the global settings and persists the change. - # @PRE: isinstance(settings, GlobalSettings) - # @POST: self.config.settings updated and saved. - # @PARAM: settings (GlobalSettings) - The new global settings. - def update_global_settings(self, settings: GlobalSettings): - with belief_scope("update_global_settings"): - logger.info("[update_global_settings][Entry] Updating settings") - - # 1. Runtime check of @PRE - assert isinstance(settings, GlobalSettings), "settings must be an instance of GlobalSettings" - - # 2. Logic implementation - self.config.settings = settings - self.save() - - # Reconfigure logger with new settings - configure_logger(settings.logging) - - logger.info("[update_global_settings][Exit] Settings updated") - # [/DEF:update_global_settings:Function] - - # [DEF:validate_path:Function] - # @PURPOSE: Validates if a path exists and is writable. - # @PRE: path is a string. - # @POST: Returns (bool, str) status. - # @PARAM: path (str) - The path to validate. - # @RETURN: tuple (bool, str) - (is_valid, message) - def validate_path(self, path: str) -> tuple[bool, str]: - with belief_scope("validate_path"): - p = os.path.abspath(path) - if not os.path.exists(p): - try: - os.makedirs(p, exist_ok=True) - except Exception as e: - return False, f"Path does not exist and could not be created: {e}" - - if not os.access(p, os.W_OK): - return False, "Path is not writable" - - return True, "Path is valid and writable" - # [/DEF:validate_path:Function] - - # [DEF:get_environments:Function] - # @PURPOSE: Returns the list of configured environments. - # @PRE: self.config is set. - # @POST: Returns list of environments. - # @RETURN: List[Environment] - List of environments. - def get_environments(self) -> List[Environment]: - with belief_scope("get_environments"): - return self.config.environments - # [/DEF:get_environments:Function] - - # [DEF:has_environments:Function] - # @PURPOSE: Checks if at least one environment is configured. - # @PRE: self.config is set. - # @POST: Returns boolean indicating if environments exist. - # @RETURN: bool - True if at least one environment exists. - def has_environments(self) -> bool: - with belief_scope("has_environments"): - return len(self.config.environments) > 0 - # [/DEF:has_environments:Function] - - # [DEF:get_environment:Function] - # @PURPOSE: Returns a single environment by ID. - # @PRE: self.config is set and isinstance(env_id, str) and len(env_id) > 0. - # @POST: Returns Environment object if found, None otherwise. - # @PARAM: env_id (str) - The ID of the environment to retrieve. - # @RETURN: Optional[Environment] - The environment with the given ID, or None. - def get_environment(self, env_id: str) -> Optional[Environment]: - with belief_scope("get_environment"): - for env in self.config.environments: - if env.id == env_id: - return env - return None - # [/DEF:get_environment:Function] - - # [DEF:add_environment:Function] - # @PURPOSE: Adds a new environment to the configuration. - # @PRE: isinstance(env, Environment) - # @POST: Environment added or updated in self.config.environments. - # @PARAM: env (Environment) - The environment to add. - def add_environment(self, env: Environment): - with belief_scope("add_environment"): - logger.info(f"[add_environment][Entry] Adding environment {env.id}") - - # 1. Runtime check of @PRE - assert isinstance(env, Environment), "env must be an instance of Environment" - - # 2. Logic implementation - # Check for duplicate ID and remove if exists - self.config.environments = [e for e in self.config.environments if e.id != env.id] - self.config.environments.append(env) - self.save() - - logger.info("[add_environment][Exit] Environment added") - # [/DEF:add_environment:Function] - - # [DEF:update_environment:Function] - # @PURPOSE: Updates an existing environment. - # @PRE: isinstance(env_id, str) and len(env_id) > 0 and isinstance(updated_env, Environment) - # @POST: Returns True if environment was found and updated. - # @PARAM: env_id (str) - The ID of the environment to update. - # @PARAM: updated_env (Environment) - The updated environment data. - # @RETURN: bool - True if updated, False otherwise. - def update_environment(self, env_id: str, updated_env: Environment) -> bool: - with belief_scope("update_environment"): - logger.info(f"[update_environment][Entry] Updating {env_id}") - - # 1. Runtime check of @PRE - assert env_id and isinstance(env_id, str), "env_id must be a non-empty string" - assert isinstance(updated_env, Environment), "updated_env must be an instance of Environment" - - # 2. Logic implementation - for i, env in enumerate(self.config.environments): - if env.id == env_id: - # If password is masked, keep the old one - if updated_env.password == "********": - updated_env.password = env.password - - self.config.environments[i] = updated_env - self.save() - logger.info(f"[update_environment][Coherence:OK] Updated {env_id}") - return True - - logger.warning(f"[update_environment][Coherence:Failed] Environment {env_id} not found") - return False - # [/DEF:update_environment:Function] - - # [DEF:delete_environment:Function] - # @PURPOSE: Deletes an environment by ID. - # @PRE: isinstance(env_id, str) and len(env_id) > 0 - # @POST: Environment removed from self.config.environments if it existed. - # @PARAM: env_id (str) - The ID of the environment to delete. - def delete_environment(self, env_id: str): - with belief_scope("delete_environment"): - logger.info(f"[delete_environment][Entry] Deleting {env_id}") - - # 1. Runtime check of @PRE - assert env_id and isinstance(env_id, str), "env_id must be a non-empty string" - - # 2. Logic implementation - original_count = len(self.config.environments) - self.config.environments = [e for e in self.config.environments if e.id != env_id] - - if len(self.config.environments) < original_count: - self.save() - logger.info(f"[delete_environment][Action] Deleted {env_id}") - else: - logger.warning(f"[delete_environment][Coherence:Failed] Environment {env_id} not found") - # [/DEF:delete_environment:Function] - -# [/DEF:ConfigManager:Class] - -# [/DEF:ConfigManagerModule:Module] +# [DEF:ConfigManagerModule:Module] +# +# @SEMANTICS: config, manager, persistence, postgresql +# @PURPOSE: Manages application configuration persisted in database with one-time migration from JSON. +# @LAYER: Core +# @RELATION: DEPENDS_ON -> ConfigModels +# @RELATION: DEPENDS_ON -> AppConfigRecord +# @RELATION: CALLS -> logger +# +# @INVARIANT: Configuration must always be valid according to AppConfig model. +# @PUBLIC_API: ConfigManager + +# [SECTION: IMPORTS] +import json +import os +from pathlib import Path +from typing import Optional, List + +from sqlalchemy.orm import Session + +from .config_models import AppConfig, Environment, GlobalSettings, StorageConfig +from .database import SessionLocal +from ..models.config import AppConfigRecord +from .logger import logger, configure_logger, belief_scope +# [/SECTION] + + +# [DEF:ConfigManager:Class] +# @PURPOSE: A class to handle application configuration persistence and management. +class ConfigManager: + # [DEF:__init__:Function] + # @PURPOSE: Initializes the ConfigManager. + # @PRE: isinstance(config_path, str) and len(config_path) > 0 + # @POST: self.config is an instance of AppConfig + # @PARAM: config_path (str) - Path to legacy JSON config (used only for initial migration fallback). + def __init__(self, config_path: str = "config.json"): + with belief_scope("__init__"): + assert isinstance(config_path, str) and config_path, "config_path must be a non-empty string" + + logger.info(f"[ConfigManager][Entry] Initializing with legacy path {config_path}") + + self.config_path = Path(config_path) + self.config: AppConfig = self._load_config() + + configure_logger(self.config.settings.logging) + assert isinstance(self.config, AppConfig), "self.config must be an instance of AppConfig" + + logger.info("[ConfigManager][Exit] Initialized") + # [/DEF:__init__:Function] + + # [DEF:_default_config:Function] + # @PURPOSE: Returns default application configuration. + # @RETURN: AppConfig - Default configuration. + def _default_config(self) -> AppConfig: + return AppConfig( + environments=[], + settings=GlobalSettings(storage=StorageConfig()), + ) + # [/DEF:_default_config:Function] + + # [DEF:_load_from_legacy_file:Function] + # @PURPOSE: Loads legacy configuration from config.json for migration fallback. + # @RETURN: AppConfig - Loaded or default configuration. + def _load_from_legacy_file(self) -> AppConfig: + with belief_scope("_load_from_legacy_file"): + if not self.config_path.exists(): + logger.info("[_load_from_legacy_file][Action] Legacy config file not found, using defaults") + return self._default_config() + + try: + with open(self.config_path, "r", encoding="utf-8") as f: + data = json.load(f) + logger.info("[_load_from_legacy_file][Coherence:OK] Legacy configuration loaded") + return AppConfig(**data) + except Exception as e: + logger.error(f"[_load_from_legacy_file][Coherence:Failed] Error loading legacy config: {e}") + return self._default_config() + # [/DEF:_load_from_legacy_file:Function] + + # [DEF:_get_record:Function] + # @PURPOSE: Loads config record from DB. + # @PARAM: session (Session) - DB session. + # @RETURN: Optional[AppConfigRecord] - Existing record or None. + def _get_record(self, session: Session) -> Optional[AppConfigRecord]: + return session.query(AppConfigRecord).filter(AppConfigRecord.id == "global").first() + # [/DEF:_get_record:Function] + + # [DEF:_load_config:Function] + # @PURPOSE: Loads the configuration from DB or performs one-time migration from JSON file. + # @PRE: DB session factory is available. + # @POST: isinstance(return, AppConfig) + # @RETURN: AppConfig - Loaded configuration. + def _load_config(self) -> AppConfig: + with belief_scope("_load_config"): + session: Session = SessionLocal() + try: + record = self._get_record(session) + if record and record.payload: + logger.info("[_load_config][Coherence:OK] Configuration loaded from database") + return AppConfig(**record.payload) + + logger.info("[_load_config][Action] No database config found, migrating legacy config") + config = self._load_from_legacy_file() + self._save_config_to_db(config, session=session) + return config + except Exception as e: + logger.error(f"[_load_config][Coherence:Failed] Error loading config from DB: {e}") + return self._default_config() + finally: + session.close() + # [/DEF:_load_config:Function] + + # [DEF:_save_config_to_db:Function] + # @PURPOSE: Saves the provided configuration object to DB. + # @PRE: isinstance(config, AppConfig) + # @POST: Configuration saved to database. + # @PARAM: config (AppConfig) - The configuration to save. + # @PARAM: session (Optional[Session]) - Existing DB session for transactional reuse. + def _save_config_to_db(self, config: AppConfig, session: Optional[Session] = None): + with belief_scope("_save_config_to_db"): + assert isinstance(config, AppConfig), "config must be an instance of AppConfig" + + owns_session = session is None + db = session or SessionLocal() + try: + record = self._get_record(db) + payload = config.model_dump() + if record is None: + record = AppConfigRecord(id="global", payload=payload) + db.add(record) + else: + record.payload = payload + db.commit() + logger.info("[_save_config_to_db][Action] Configuration saved to database") + except Exception as e: + db.rollback() + logger.error(f"[_save_config_to_db][Coherence:Failed] Failed to save: {e}") + raise + finally: + if owns_session: + db.close() + # [/DEF:_save_config_to_db:Function] + + # [DEF:save:Function] + # @PURPOSE: Saves the current configuration state to DB. + # @PRE: self.config is set. + # @POST: self._save_config_to_db called. + def save(self): + with belief_scope("save"): + self._save_config_to_db(self.config) + # [/DEF:save:Function] + + # [DEF:get_config:Function] + # @PURPOSE: Returns the current configuration. + # @RETURN: AppConfig - The current configuration. + def get_config(self) -> AppConfig: + with belief_scope("get_config"): + return self.config + # [/DEF:get_config:Function] + + # [DEF:update_global_settings:Function] + # @PURPOSE: Updates the global settings and persists the change. + # @PRE: isinstance(settings, GlobalSettings) + # @POST: self.config.settings updated and saved. + # @PARAM: settings (GlobalSettings) - The new global settings. + def update_global_settings(self, settings: GlobalSettings): + with belief_scope("update_global_settings"): + logger.info("[update_global_settings][Entry] Updating settings") + + assert isinstance(settings, GlobalSettings), "settings must be an instance of GlobalSettings" + self.config.settings = settings + self.save() + configure_logger(settings.logging) + logger.info("[update_global_settings][Exit] Settings updated") + # [/DEF:update_global_settings:Function] + + # [DEF:validate_path:Function] + # @PURPOSE: Validates if a path exists and is writable. + # @PARAM: path (str) - The path to validate. + # @RETURN: tuple (bool, str) - (is_valid, message) + def validate_path(self, path: str) -> tuple[bool, str]: + with belief_scope("validate_path"): + p = os.path.abspath(path) + if not os.path.exists(p): + try: + os.makedirs(p, exist_ok=True) + except Exception as e: + return False, f"Path does not exist and could not be created: {e}" + + if not os.access(p, os.W_OK): + return False, "Path is not writable" + + return True, "Path is valid and writable" + # [/DEF:validate_path:Function] + + # [DEF:get_environments:Function] + # @PURPOSE: Returns the list of configured environments. + # @RETURN: List[Environment] - List of environments. + def get_environments(self) -> List[Environment]: + with belief_scope("get_environments"): + return self.config.environments + # [/DEF:get_environments:Function] + + # [DEF:has_environments:Function] + # @PURPOSE: Checks if at least one environment is configured. + # @RETURN: bool - True if at least one environment exists. + def has_environments(self) -> bool: + with belief_scope("has_environments"): + return len(self.config.environments) > 0 + # [/DEF:has_environments:Function] + + # [DEF:get_environment:Function] + # @PURPOSE: Returns a single environment by ID. + # @PARAM: env_id (str) - The ID of the environment to retrieve. + # @RETURN: Optional[Environment] - The environment with the given ID, or None. + def get_environment(self, env_id: str) -> Optional[Environment]: + with belief_scope("get_environment"): + for env in self.config.environments: + if env.id == env_id: + return env + return None + # [/DEF:get_environment:Function] + + # [DEF:add_environment:Function] + # @PURPOSE: Adds a new environment to the configuration. + # @PARAM: env (Environment) - The environment to add. + def add_environment(self, env: Environment): + with belief_scope("add_environment"): + logger.info(f"[add_environment][Entry] Adding environment {env.id}") + assert isinstance(env, Environment), "env must be an instance of Environment" + + self.config.environments = [e for e in self.config.environments if e.id != env.id] + self.config.environments.append(env) + self.save() + logger.info("[add_environment][Exit] Environment added") + # [/DEF:add_environment:Function] + + # [DEF:update_environment:Function] + # @PURPOSE: Updates an existing environment. + # @PARAM: env_id (str) - The ID of the environment to update. + # @PARAM: updated_env (Environment) - The updated environment data. + # @RETURN: bool - True if updated, False otherwise. + def update_environment(self, env_id: str, updated_env: Environment) -> bool: + with belief_scope("update_environment"): + logger.info(f"[update_environment][Entry] Updating {env_id}") + assert env_id and isinstance(env_id, str), "env_id must be a non-empty string" + assert isinstance(updated_env, Environment), "updated_env must be an instance of Environment" + + for i, env in enumerate(self.config.environments): + if env.id == env_id: + if updated_env.password == "********": + updated_env.password = env.password + + self.config.environments[i] = updated_env + self.save() + logger.info(f"[update_environment][Coherence:OK] Updated {env_id}") + return True + + logger.warning(f"[update_environment][Coherence:Failed] Environment {env_id} not found") + return False + # [/DEF:update_environment:Function] + + # [DEF:delete_environment:Function] + # @PURPOSE: Deletes an environment by ID. + # @PARAM: env_id (str) - The ID of the environment to delete. + def delete_environment(self, env_id: str): + with belief_scope("delete_environment"): + logger.info(f"[delete_environment][Entry] Deleting {env_id}") + assert env_id and isinstance(env_id, str), "env_id must be a non-empty string" + + original_count = len(self.config.environments) + self.config.environments = [e for e in self.config.environments if e.id != env_id] + + if len(self.config.environments) < original_count: + self.save() + logger.info(f"[delete_environment][Action] Deleted {env_id}") + else: + logger.warning(f"[delete_environment][Coherence:Failed] Environment {env_id} not found") + # [/DEF:delete_environment:Function] + + +# [/DEF:ConfigManager:Class] +# [/DEF:ConfigManagerModule:Module] diff --git a/backend/src/core/config_models.py b/backend/src/core/config_models.py index 5ac659e..8c7307d 100755 --- a/backend/src/core/config_models.py +++ b/backend/src/core/config_models.py @@ -3,7 +3,7 @@ # @SEMANTICS: config, models, pydantic # @PURPOSE: Defines the data models for application configuration using Pydantic. # @LAYER: Core -# @RELATION: READS_FROM -> config.json +# @RELATION: READS_FROM -> app_configurations (database) # @RELATION: USED_BY -> ConfigManager from pydantic import BaseModel, Field @@ -33,10 +33,10 @@ class Environment(BaseModel): # [DEF:LoggingConfig:DataClass] # @PURPOSE: Defines the configuration for the application's logging system. -class LoggingConfig(BaseModel): - level: str = "INFO" - task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR) - file_path: Optional[str] = "logs/app.log" +class LoggingConfig(BaseModel): + level: str = "INFO" + task_log_level: str = "INFO" # Minimum level for task-specific logs (DEBUG, INFO, WARNING, ERROR) + file_path: Optional[str] = None max_bytes: int = 10 * 1024 * 1024 backup_count: int = 5 enable_belief_state: bool = True diff --git a/backend/src/core/database.py b/backend/src/core/database.py index 8eae25c..837bb1e 100644 --- a/backend/src/core/database.py +++ b/backend/src/core/database.py @@ -1,7 +1,7 @@ # [DEF:backend.src.core.database:Module] # -# @SEMANTICS: database, sqlite, sqlalchemy, session, persistence -# @PURPOSE: Configures the SQLite database connection and session management. +# @SEMANTICS: database, postgresql, sqlalchemy, session, persistence +# @PURPOSE: Configures database connection and session management (PostgreSQL-first). # @LAYER: Core # @RELATION: DEPENDS_ON -> sqlalchemy # @RELATION: USES -> backend.src.models.mapping @@ -14,6 +14,9 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from ..models.mapping import Base # Import models to ensure they're registered with Base +from ..models import task as _task_models # noqa: F401 +from ..models import auth as _auth_models # noqa: F401 +from ..models import config as _config_models # noqa: F401 from .logger import belief_scope from .auth.config import auth_config import os @@ -21,44 +24,50 @@ from pathlib import Path # [/SECTION] # [DEF:BASE_DIR:Variable] -# @PURPOSE: Base directory for the backend (where .db files should reside). +# @PURPOSE: Base directory for the backend. BASE_DIR = Path(__file__).resolve().parent.parent.parent # [/DEF:BASE_DIR:Variable] # [DEF:DATABASE_URL:Constant] -# @PURPOSE: URL for the main mappings database. -DATABASE_URL = os.getenv("DATABASE_URL", f"sqlite:///{BASE_DIR}/mappings.db") +# @PURPOSE: URL for the main application database. +DEFAULT_POSTGRES_URL = os.getenv( + "POSTGRES_URL", + "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools", +) +DATABASE_URL = os.getenv("DATABASE_URL", DEFAULT_POSTGRES_URL) # [/DEF:DATABASE_URL:Constant] # [DEF:TASKS_DATABASE_URL:Constant] # @PURPOSE: URL for the tasks execution database. -TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", f"sqlite:///{BASE_DIR}/tasks.db") +# Defaults to DATABASE_URL to keep task logs in the same PostgreSQL instance. +TASKS_DATABASE_URL = os.getenv("TASKS_DATABASE_URL", DATABASE_URL) # [/DEF:TASKS_DATABASE_URL:Constant] # [DEF:AUTH_DATABASE_URL:Constant] # @PURPOSE: URL for the authentication database. AUTH_DATABASE_URL = os.getenv("AUTH_DATABASE_URL", auth_config.AUTH_DATABASE_URL) -# If it's a relative sqlite path starting with ./backend/, fix it to be absolute or relative to BASE_DIR -if AUTH_DATABASE_URL.startswith("sqlite:///./backend/"): - AUTH_DATABASE_URL = AUTH_DATABASE_URL.replace("sqlite:///./backend/", f"sqlite:///{BASE_DIR}/") -elif AUTH_DATABASE_URL.startswith("sqlite:///./") and not AUTH_DATABASE_URL.startswith("sqlite:///./backend/"): - # If it's just ./ but we are in backend, it's fine, but let's make it absolute for robustness - AUTH_DATABASE_URL = AUTH_DATABASE_URL.replace("sqlite:///./", f"sqlite:///{BASE_DIR}/") # [/DEF:AUTH_DATABASE_URL:Constant] # [DEF:engine:Variable] +def _build_engine(db_url: str): + with belief_scope("_build_engine"): + if db_url.startswith("sqlite"): + return create_engine(db_url, connect_args={"check_same_thread": False}) + return create_engine(db_url, pool_pre_ping=True) + + # @PURPOSE: SQLAlchemy engine for mappings database. -engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False}) +engine = _build_engine(DATABASE_URL) # [/DEF:engine:Variable] # [DEF:tasks_engine:Variable] # @PURPOSE: SQLAlchemy engine for tasks database. -tasks_engine = create_engine(TASKS_DATABASE_URL, connect_args={"check_same_thread": False}) +tasks_engine = _build_engine(TASKS_DATABASE_URL) # [/DEF:tasks_engine:Variable] # [DEF:auth_engine:Variable] # @PURPOSE: SQLAlchemy engine for authentication database. -auth_engine = create_engine(AUTH_DATABASE_URL, connect_args={"check_same_thread": False}) +auth_engine = _build_engine(AUTH_DATABASE_URL) # [/DEF:auth_engine:Variable] # [DEF:SessionLocal:Class] diff --git a/backend/src/dependencies.py b/backend/src/dependencies.py index c25db49..c49ba02 100755 --- a/backend/src/dependencies.py +++ b/backend/src/dependencies.py @@ -20,14 +20,14 @@ from .core.auth.jwt import decode_token from .core.auth.repository import AuthRepository from .models.auth import User -# Initialize singletons -# Use absolute path relative to this file to ensure plugins are found regardless of CWD -project_root = Path(__file__).parent.parent.parent -config_path = project_root / "config.json" -config_manager = ConfigManager(config_path=str(config_path)) - -# Initialize database before any other services that might use it -init_db() +# Initialize singletons +# Use absolute path relative to this file to ensure plugins are found regardless of CWD +project_root = Path(__file__).parent.parent.parent +config_path = project_root / "config.json" + +# Initialize database before services that use persisted configuration. +init_db() +config_manager = ConfigManager(config_path=str(config_path)) # [DEF:get_config_manager:Function] # @PURPOSE: Dependency injector for ConfigManager. diff --git a/backend/src/models/config.py b/backend/src/models/config.py new file mode 100644 index 0000000..2f7b1e2 --- /dev/null +++ b/backend/src/models/config.py @@ -0,0 +1,26 @@ +# [DEF:backend.src.models.config:Module] +# +# @TIER: STANDARD +# @SEMANTICS: database, config, settings, sqlalchemy +# @PURPOSE: Defines database schema for persisted application configuration. +# @LAYER: Domain +# @RELATION: DEPENDS_ON -> sqlalchemy + +from sqlalchemy import Column, String, DateTime, JSON +from sqlalchemy.sql import func + +from .mapping import Base + + +# [DEF:AppConfigRecord:Class] +# @PURPOSE: Stores the single source of truth for application configuration. +class AppConfigRecord(Base): + __tablename__ = "app_configurations" + + id = Column(String, primary_key=True) + payload = Column(JSON, nullable=False) + updated_at = Column(DateTime(timezone=True), server_default=func.now(), onupdate=func.now()) + + +# [/DEF:AppConfigRecord:Class] +# [/DEF:backend.src.models.config:Module] diff --git a/backend/src/models/storage.py b/backend/src/models/storage.py index 981ba89..cdf039e 100644 --- a/backend/src/models/storage.py +++ b/backend/src/models/storage.py @@ -22,6 +22,8 @@ class FileCategory(str, Enum): # @PURPOSE: Configuration model for the storage system, defining paths and naming patterns. class StorageConfig(BaseModel): root_path: str = Field(default="backups", description="Absolute path to the storage root directory.") + backup_path: str = Field(default="backups", description="Subpath for backups.") + repo_path: str = Field(default="repositorys", description="Subpath for repositories.") backup_structure_pattern: str = Field(default="{category}/", description="Pattern for backup directory structure.") repo_structure_pattern: str = Field(default="{category}/", description="Pattern for repository directory structure.") filename_pattern: str = Field(default="{name}_{timestamp}", description="Pattern for filenames.") diff --git a/backend/src/scripts/migrate_sqlite_to_postgres.py b/backend/src/scripts/migrate_sqlite_to_postgres.py new file mode 100644 index 0000000..d878368 --- /dev/null +++ b/backend/src/scripts/migrate_sqlite_to_postgres.py @@ -0,0 +1,350 @@ +# [DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module] +# +# @SEMANTICS: migration, sqlite, postgresql, config, task_logs, task_records +# @PURPOSE: Migrates legacy config and task history from SQLite/file storage to PostgreSQL. +# @LAYER: Scripts +# @RELATION: READS_FROM -> backend/tasks.db +# @RELATION: READS_FROM -> backend/config.json +# @RELATION: WRITES_TO -> postgresql.task_records +# @RELATION: WRITES_TO -> postgresql.task_logs +# @RELATION: WRITES_TO -> postgresql.app_configurations +# +# @INVARIANT: Script is idempotent for task_records and app_configurations. + +# [SECTION: IMPORTS] +import argparse +import json +import os +import sqlite3 +from pathlib import Path +from typing import Any, Dict, Iterable, Optional + +from sqlalchemy import create_engine, text +from sqlalchemy.exc import SQLAlchemyError + +from src.core.logger import belief_scope, logger +# [/SECTION] + + +# [DEF:Constants:Section] +DEFAULT_TARGET_URL = os.getenv( + "DATABASE_URL", + os.getenv("POSTGRES_URL", "postgresql+psycopg2://postgres:postgres@localhost:5432/ss_tools"), +) +# [/DEF:Constants:Section] + + +# [DEF:_json_load_if_needed:Function] +# @PURPOSE: Parses JSON-like values from SQLite TEXT/JSON columns to Python objects. +def _json_load_if_needed(value: Any) -> Any: + with belief_scope("_json_load_if_needed"): + if value is None: + return None + if isinstance(value, (dict, list)): + return value + if isinstance(value, str): + raw = value.strip() + if not raw: + return None + if raw[0] in "{[": + try: + return json.loads(raw) + except json.JSONDecodeError: + return value + return value + + +# [DEF:_find_legacy_config_path:Function] +# @PURPOSE: Resolves the existing legacy config.json path from candidates. +def _find_legacy_config_path(explicit_path: Optional[str]) -> Optional[Path]: + with belief_scope("_find_legacy_config_path"): + if explicit_path: + p = Path(explicit_path) + return p if p.exists() else None + + candidates = [ + Path("backend/config.json"), + Path("config.json"), + ] + for candidate in candidates: + if candidate.exists(): + return candidate + return None + + +# [DEF:_connect_sqlite:Function] +# @PURPOSE: Opens a SQLite connection with row factory. +def _connect_sqlite(path: Path) -> sqlite3.Connection: + with belief_scope("_connect_sqlite"): + conn = sqlite3.connect(str(path)) + conn.row_factory = sqlite3.Row + return conn + + +# [DEF:_ensure_target_schema:Function] +# @PURPOSE: Ensures required PostgreSQL tables exist before migration. +def _ensure_target_schema(engine) -> None: + with belief_scope("_ensure_target_schema"): + stmts: Iterable[str] = ( + """ + CREATE TABLE IF NOT EXISTS app_configurations ( + id TEXT PRIMARY KEY, + payload JSONB NOT NULL, + updated_at TIMESTAMPTZ DEFAULT NOW() + ) + """, + """ + CREATE TABLE IF NOT EXISTS task_records ( + id TEXT PRIMARY KEY, + type TEXT NOT NULL, + status TEXT NOT NULL, + environment_id TEXT NULL, + started_at TIMESTAMPTZ NULL, + finished_at TIMESTAMPTZ NULL, + logs JSONB NULL, + error TEXT NULL, + result JSONB NULL, + created_at TIMESTAMPTZ DEFAULT NOW(), + params JSONB NULL + ) + """, + """ + CREATE TABLE IF NOT EXISTS task_logs ( + id INTEGER PRIMARY KEY, + task_id TEXT NOT NULL, + timestamp TIMESTAMPTZ NOT NULL, + level VARCHAR(16) NOT NULL, + source VARCHAR(64) NOT NULL DEFAULT 'system', + message TEXT NOT NULL, + metadata_json TEXT NULL, + CONSTRAINT fk_task_logs_task + FOREIGN KEY(task_id) + REFERENCES task_records(id) + ON DELETE CASCADE + ) + """, + "CREATE INDEX IF NOT EXISTS ix_task_logs_task_timestamp ON task_logs (task_id, timestamp)", + "CREATE INDEX IF NOT EXISTS ix_task_logs_task_level ON task_logs (task_id, level)", + "CREATE INDEX IF NOT EXISTS ix_task_logs_task_source ON task_logs (task_id, source)", + """ + DO $$ + BEGIN + IF EXISTS ( + SELECT 1 FROM pg_class WHERE relkind = 'S' AND relname = 'task_logs_id_seq' + ) THEN + PERFORM 1; + ELSE + CREATE SEQUENCE task_logs_id_seq OWNED BY task_logs.id; + END IF; + END $$; + """, + "ALTER TABLE task_logs ALTER COLUMN id SET DEFAULT nextval('task_logs_id_seq')", + ) + with engine.begin() as conn: + for stmt in stmts: + conn.execute(text(stmt)) + + +# [DEF:_migrate_config:Function] +# @PURPOSE: Migrates legacy config.json into app_configurations(global). +def _migrate_config(engine, legacy_config_path: Optional[Path]) -> int: + with belief_scope("_migrate_config"): + if legacy_config_path is None: + logger.info("[_migrate_config][Action] No legacy config.json found, skipping") + return 0 + + payload = json.loads(legacy_config_path.read_text(encoding="utf-8")) + with engine.begin() as conn: + conn.execute( + text( + """ + INSERT INTO app_configurations (id, payload, updated_at) + VALUES ('global', CAST(:payload AS JSONB), NOW()) + ON CONFLICT (id) + DO UPDATE SET payload = EXCLUDED.payload, updated_at = NOW() + """ + ), + {"payload": json.dumps(payload, ensure_ascii=True)}, + ) + logger.info("[_migrate_config][Coherence:OK] Config migrated from %s", legacy_config_path) + return 1 + + +# [DEF:_migrate_tasks_and_logs:Function] +# @PURPOSE: Migrates task_records and task_logs from SQLite into PostgreSQL. +def _migrate_tasks_and_logs(engine, sqlite_conn: sqlite3.Connection) -> Dict[str, int]: + with belief_scope("_migrate_tasks_and_logs"): + stats = {"task_records_total": 0, "task_records_inserted": 0, "task_logs_total": 0, "task_logs_inserted": 0} + + rows = sqlite_conn.execute( + """ + SELECT id, type, status, environment_id, started_at, finished_at, logs, error, result, created_at, params + FROM task_records + ORDER BY created_at ASC + """ + ).fetchall() + stats["task_records_total"] = len(rows) + + with engine.begin() as conn: + existing_env_ids = { + row[0] + for row in conn.execute(text("SELECT id FROM environments")).fetchall() + } + for row in rows: + params_obj = _json_load_if_needed(row["params"]) + result_obj = _json_load_if_needed(row["result"]) + logs_obj = _json_load_if_needed(row["logs"]) + environment_id = row["environment_id"] + if environment_id and environment_id not in existing_env_ids: + # Legacy task may reference environments that were not migrated; keep task row and drop FK value. + environment_id = None + + res = conn.execute( + text( + """ + INSERT INTO task_records ( + id, type, status, environment_id, started_at, finished_at, + logs, error, result, created_at, params + ) VALUES ( + :id, :type, :status, :environment_id, :started_at, :finished_at, + CAST(:logs AS JSONB), :error, CAST(:result AS JSONB), :created_at, CAST(:params AS JSONB) + ) + ON CONFLICT (id) DO NOTHING + """ + ), + { + "id": row["id"], + "type": row["type"], + "status": row["status"], + "environment_id": environment_id, + "started_at": row["started_at"], + "finished_at": row["finished_at"], + "logs": json.dumps(logs_obj, ensure_ascii=True) if logs_obj is not None else None, + "error": row["error"], + "result": json.dumps(result_obj, ensure_ascii=True) if result_obj is not None else None, + "created_at": row["created_at"], + "params": json.dumps(params_obj, ensure_ascii=True) if params_obj is not None else None, + }, + ) + if res.rowcount and res.rowcount > 0: + stats["task_records_inserted"] += int(res.rowcount) + + log_rows = sqlite_conn.execute( + """ + SELECT id, task_id, timestamp, level, source, message, metadata_json + FROM task_logs + ORDER BY id ASC + """ + ).fetchall() + stats["task_logs_total"] = len(log_rows) + + with engine.begin() as conn: + for row in log_rows: + # Preserve original IDs to keep migration idempotent. + res = conn.execute( + text( + """ + INSERT INTO task_logs (id, task_id, timestamp, level, source, message, metadata_json) + VALUES (:id, :task_id, :timestamp, :level, :source, :message, :metadata_json) + ON CONFLICT (id) DO NOTHING + """ + ), + { + "id": row["id"], + "task_id": row["task_id"], + "timestamp": row["timestamp"], + "level": row["level"], + "source": row["source"] or "system", + "message": row["message"], + "metadata_json": row["metadata_json"], + }, + ) + if res.rowcount and res.rowcount > 0: + stats["task_logs_inserted"] += int(res.rowcount) + + # Ensure sequence is aligned after explicit id inserts. + conn.execute( + text( + """ + SELECT setval( + 'task_logs_id_seq', + COALESCE((SELECT MAX(id) FROM task_logs), 1), + TRUE + ) + """ + ) + ) + + logger.info( + "[_migrate_tasks_and_logs][Coherence:OK] task_records=%s/%s task_logs=%s/%s", + stats["task_records_inserted"], + stats["task_records_total"], + stats["task_logs_inserted"], + stats["task_logs_total"], + ) + return stats + + +# [DEF:run_migration:Function] +# @PURPOSE: Orchestrates migration from SQLite/file to PostgreSQL. +def run_migration(sqlite_path: Path, target_url: str, legacy_config_path: Optional[Path]) -> Dict[str, int]: + with belief_scope("run_migration"): + logger.info("[run_migration][Entry] sqlite=%s target=%s", sqlite_path, target_url) + if not sqlite_path.exists(): + raise FileNotFoundError(f"SQLite source not found: {sqlite_path}") + + sqlite_conn = _connect_sqlite(sqlite_path) + engine = create_engine(target_url, pool_pre_ping=True) + try: + _ensure_target_schema(engine) + config_upserted = _migrate_config(engine, legacy_config_path) + stats = _migrate_tasks_and_logs(engine, sqlite_conn) + stats["config_upserted"] = config_upserted + return stats + finally: + sqlite_conn.close() + + +# [DEF:main:Function] +# @PURPOSE: CLI entrypoint. +def main() -> int: + with belief_scope("main"): + parser = argparse.ArgumentParser( + description="Migrate legacy config.json and task logs from SQLite to PostgreSQL.", + ) + parser.add_argument( + "--sqlite-path", + default="backend/tasks.db", + help="Path to source SQLite DB with task_records/task_logs (default: backend/tasks.db).", + ) + parser.add_argument( + "--target-url", + default=DEFAULT_TARGET_URL, + help="Target PostgreSQL SQLAlchemy URL (default: DATABASE_URL/POSTGRES_URL env).", + ) + parser.add_argument( + "--config-path", + default=None, + help="Optional path to legacy config.json (auto-detected when omitted).", + ) + + args = parser.parse_args() + + sqlite_path = Path(args.sqlite_path) + legacy_config_path = _find_legacy_config_path(args.config_path) + try: + stats = run_migration(sqlite_path=sqlite_path, target_url=args.target_url, legacy_config_path=legacy_config_path) + print("Migration completed.") + print(json.dumps(stats, indent=2)) + return 0 + except (SQLAlchemyError, OSError, sqlite3.Error, ValueError) as e: + logger.error("[main][Coherence:Failed] Migration failed: %s", e) + print(f"Migration failed: {e}") + return 1 + + +if __name__ == "__main__": + raise SystemExit(main()) +# [/DEF:main:Function] + +# [/DEF:backend.src.scripts.migrate_sqlite_to_postgres:Module] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..dc14511 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,42 @@ +services: + db: + image: ${POSTGRES_IMAGE:-postgres:16-alpine} + container_name: ss_tools_db + restart: unless-stopped + environment: + POSTGRES_DB: ss_tools + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + ports: + - "${POSTGRES_HOST_PORT:-5432}:5432" + volumes: + - postgres_data:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d ss_tools"] + interval: 10s + timeout: 5s + retries: 10 + + app: + build: + context: . + dockerfile: Dockerfile + container_name: ss_tools_app + restart: unless-stopped + depends_on: + db: + condition: service_healthy + environment: + POSTGRES_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools + DATABASE_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools + TASKS_DATABASE_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools + AUTH_DATABASE_URL: postgresql+psycopg2://postgres:postgres@db:5432/ss_tools + BACKEND_PORT: 8000 + ports: + - "8000:8000" + volumes: + - ./backups:/app/backups + - ./backend/git_repos:/app/backend/git_repos + +volumes: + postgres_data: diff --git a/generate_semantic_map.py b/generate_semantic_map.py index baff717..d07e789 100644 --- a/generate_semantic_map.py +++ b/generate_semantic_map.py @@ -1,13 +1,14 @@ # [DEF:generate_semantic_map:Module] # # @TIER: CRITICAL -# @SEMANTICS: semantic_analysis, parser, map_generator, compliance_checker, tier_validation, svelte_props, data_flow -# @PURPOSE: Scans the codebase to generate a Semantic Map and Compliance Report based on the System Standard. +# @SEMANTICS: semantic_analysis, parser, map_generator, compliance_checker, tier_validation, svelte_props, data_flow, module_map +# @PURPOSE: Scans the codebase to generate a Semantic Map, Module Map, and Compliance Report based on the System Standard. # @LAYER: DevOps/Tooling # @INVARIANT: All DEF anchors must have matching closing anchors; TIER determines validation strictness. # @RELATION: READS -> FileSystem # @RELATION: PRODUCES -> semantics/semantic_map.json # @RELATION: PRODUCES -> .ai/PROJECT_MAP.md +# @RELATION: PRODUCES -> .ai/MODULE_MAP.md # @RELATION: PRODUCES -> semantics/reports/semantic_report_*.md # [SECTION: IMPORTS] @@ -83,6 +84,7 @@ IGNORE_FILES = { } OUTPUT_JSON = "semantics/semantic_map.json" OUTPUT_COMPRESSED_MD = ".ai/PROJECT_MAP.md" +OUTPUT_MODULE_MAP_MD = ".ai/MODULE_MAP.md" REPORTS_DIR = "semantics/reports" # Tier-based mandatory tags @@ -830,6 +832,7 @@ class SemanticMapGenerator: self._generate_report() self._generate_compressed_map() + self._generate_module_map() # [/DEF:_generate_artifacts:Function] # [DEF:_generate_report:Function] @@ -990,6 +993,163 @@ class SemanticMapGenerator: self._write_entity_md(f, child, level + 1) # [/DEF:_write_entity_md:Function] + # [DEF:_generate_module_map:Function] + # @TIER: CRITICAL + # @PURPOSE: Generates a module-centric map grouping entities by directory structure. + # @PRE: Entities have been processed. + # @POST: Markdown module map is written to .ai/MODULE_MAP.md. + def _generate_module_map(self): + with belief_scope("_generate_module_map"): + os.makedirs(os.path.dirname(OUTPUT_MODULE_MAP_MD), exist_ok=True) + + # Group entities by directory/module + modules: Dict[str, Dict[str, Any]] = {} + + # [DEF:_get_module_path:Function] + # @TIER: STANDARD + # @PURPOSE: Extracts the module path from a file path. + # @PRE: file_path is a valid relative path. + # @POST: Returns a module path string. + def _get_module_path(file_path: str) -> str: + # Convert file path to module-like path + parts = file_path.replace(os.sep, '/').split('/') + # Remove filename + if len(parts) > 1: + return '/'.join(parts[:-1]) + return 'root' + # [/DEF:_get_module_path:Function] + + # [DEF:_collect_all_entities:Function] + # @TIER: STANDARD + # @PURPOSE: Flattens entity tree for easier grouping. + # @PRE: entity list is valid. + # @POST: Returns flat list of all entities with their hierarchy. + def _collect_all_entities(entities: List[SemanticEntity], result: List[Tuple[str, SemanticEntity]]): + for e in entities: + result.append((_get_module_path(e.file_path), e)) + _collect_all_entities(e.children, result) + # [/DEF:_collect_all_entities:Function] + + # Collect all entities + all_entities: List[Tuple[str, SemanticEntity]] = [] + _collect_all_entities(self.entities, all_entities) + + # Group by module path + for module_path, entity in all_entities: + if module_path not in modules: + modules[module_path] = { + 'entities': [], + 'files': set(), + 'layers': set(), + 'tiers': {'CRITICAL': 0, 'STANDARD': 0, 'TRIVIAL': 0}, + 'relations': [] + } + modules[module_path]['entities'].append(entity) + modules[module_path]['files'].add(entity.file_path) + if entity.tags.get('LAYER'): + modules[module_path]['layers'].add(entity.tags.get('LAYER')) + tier = entity.get_tier().value + modules[module_path]['tiers'][tier] = modules[module_path]['tiers'].get(tier, 0) + 1 + for rel in entity.relations: + modules[module_path]['relations'].append(rel) + + # Write module map + with open(OUTPUT_MODULE_MAP_MD, 'w', encoding='utf-8') as f: + f.write("# Module Map\n\n") + f.write("> High-level module structure for AI Context. Generated automatically.\n\n") + f.write(f"**Generated:** {datetime.datetime.now().isoformat()}\n\n") + + # Summary statistics + total_modules = len(modules) + total_entities = len(all_entities) + f.write("## Summary\n\n") + f.write(f"- **Total Modules:** {total_modules}\n") + f.write(f"- **Total Entities:** {total_entities}\n\n") + + # Module hierarchy + f.write("## Module Hierarchy\n\n") + + # Sort modules by path for consistent output + sorted_modules = sorted(modules.items(), key=lambda x: x[0]) + + for module_path, data in sorted_modules: + # Calculate module depth for indentation + depth = module_path.count('/') + indent = " " * depth + + # Module header + module_name = module_path.split('/')[-1] if module_path != 'root' else 'root' + f.write(f"{indent}### 📁 `{module_name}/`\n\n") + + # Module metadata + if data['layers']: + layers_str = ", ".join(sorted(data['layers'])) + f.write(f"{indent}- 🏗️ **Layers:** {layers_str}\n") + + tiers_summary = [] + for tier_name, count in data['tiers'].items(): + if count > 0: + tiers_summary.append(f"{tier_name}: {count}") + if tiers_summary: + f.write(f"{indent}- 📊 **Tiers:** {', '.join(tiers_summary)}\n") + + f.write(f"{indent}- 📄 **Files:** {len(data['files'])}\n") + f.write(f"{indent}- 📦 **Entities:** {len(data['entities'])}\n") + + # List key entities (Modules, Classes, Components only) + key_entities = [e for e in data['entities'] if e.type in ['Module', 'Class', 'Component', 'Store']] + if key_entities: + f.write(f"\n{indent}**Key Entities:**\n\n") + for entity in sorted(key_entities, key=lambda x: (x.type, x.name))[:10]: + icon = "📦" if entity.type == "Module" else "ℂ" if entity.type == "Class" else "🧩" if entity.type == "Component" else "🗄️" + tier_badge = "" + if entity.get_tier() == Tier.CRITICAL: + tier_badge = " `[CRITICAL]`" + elif entity.get_tier() == Tier.TRIVIAL: + tier_badge = " `[TRIVIAL]`" + purpose = entity.tags.get('PURPOSE', '')[:60] + "..." if entity.tags.get('PURPOSE') and len(entity.tags.get('PURPOSE', '')) > 60 else entity.tags.get('PURPOSE', '') + f.write(f"{indent} - {icon} **{entity.name}** ({entity.type}){tier_badge}\n") + if purpose: + f.write(f"{indent} - {purpose}\n") + + # External relations + external_relations = [r for r in data['relations'] if r['type'] in ['DEPENDS_ON', 'IMPLEMENTS', 'INHERITS']] + if external_relations: + unique_deps = {} + for rel in external_relations: + key = f"{rel['type']} -> {rel['target']}" + unique_deps[key] = rel + f.write(f"\n{indent}**Dependencies:**\n\n") + for rel_str in sorted(unique_deps.keys())[:5]: + f.write(f"{indent} - 🔗 {rel_str}\n") + + f.write("\n") + + # Cross-module dependency graph + f.write("## Cross-Module Dependencies\n\n") + f.write("```mermaid\n") + f.write("graph TD\n") + + # Find inter-module dependencies + for module_path, data in sorted_modules: + module_name = module_path.split('/')[-1] if module_path != 'root' else 'root' + safe_name = module_name.replace('-', '_').replace('.', '_') + + for rel in data['relations']: + target = rel.get('target', '') + # Check if target references another module + for other_module in modules: + if other_module != module_path and other_module in target: + other_name = other_module.split('/')[-1] + safe_other = other_name.replace('-', '_').replace('.', '_') + f.write(f" {safe_name}-->|{rel['type']}|{safe_other}\n") + break + + f.write("```\n") + + print(f"Generated {OUTPUT_MODULE_MAP_MD}") + # [/DEF:_generate_module_map:Function] + # [/DEF:SemanticMapGenerator:Class] diff --git a/semantics/semantic_map.json b/semantics/semantic_map.json index 45191f8..e138ec7 100644 --- a/semantics/semantic_map.json +++ b/semantics/semantic_map.json @@ -1,17 +1,17 @@ { "project_root": ".", - "generated_at": "2026-02-19T22:53:21.494028", + "generated_at": "2026-02-20T11:30:24.246504", "modules": [ { "name": "generate_semantic_map", "type": "Module", "tier": "CRITICAL", "start_line": 1, - "end_line": 1000, + "end_line": 1160, "tags": { "TIER": "CRITICAL", - "SEMANTICS": "semantic_analysis, parser, map_generator, compliance_checker, tier_validation, svelte_props, data_flow", - "PURPOSE": "Scans the codebase to generate a Semantic Map and Compliance Report based on the System Standard.", + "SEMANTICS": "semantic_analysis, parser, map_generator, compliance_checker, tier_validation, svelte_props, data_flow, module_map", + "PURPOSE": "Scans the codebase to generate a Semantic Map, Module Map, and Compliance Report based on the System Standard.", "LAYER": "DevOps/Tooling", "INVARIANT": "All DEF anchors must have matching closing anchors; TIER determines validation strictness." }, @@ -28,6 +28,10 @@ "type": "PRODUCES", "target": ".ai/PROJECT_MAP.md" }, + { + "type": "PRODUCES", + "target": ".ai/MODULE_MAP.md" + }, { "type": "PRODUCES", "target": "semantics/reports/semantic_report_*.md" @@ -38,8 +42,8 @@ "name": "__init__", "type": "Function", "tier": "TRIVIAL", - "start_line": 25, - "end_line": 32, + "start_line": 26, + "end_line": 33, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Mock init for self-containment.", @@ -58,8 +62,8 @@ "name": "__enter__", "type": "Function", "tier": "TRIVIAL", - "start_line": 34, - "end_line": 41, + "start_line": 35, + "end_line": 42, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Mock enter.", @@ -78,8 +82,8 @@ "name": "__exit__", "type": "Function", "tier": "TRIVIAL", - "start_line": 43, - "end_line": 50, + "start_line": 44, + "end_line": 51, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Mock exit.", @@ -98,8 +102,8 @@ "name": "Tier", "type": "Class", "tier": "TRIVIAL", - "start_line": 57, - "end_line": 63, + "start_line": 58, + "end_line": 64, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Enumeration of semantic tiers defining validation strictness." @@ -116,8 +120,8 @@ "name": "Severity", "type": "Class", "tier": "TRIVIAL", - "start_line": 67, - "end_line": 73, + "start_line": 68, + "end_line": 74, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Severity levels for compliance issues." @@ -134,8 +138,8 @@ "name": "ComplianceIssue", "type": "Class", "tier": "TRIVIAL", - "start_line": 119, - "end_line": 134, + "start_line": 121, + "end_line": 136, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Represents a single compliance issue with severity." @@ -152,8 +156,8 @@ "name": "SemanticEntity", "type": "Class", "tier": "CRITICAL", - "start_line": 137, - "end_line": 313, + "start_line": 139, + "end_line": 315, "tags": { "TIER": "CRITICAL", "PURPOSE": "Represents a code entity (Module, Function, Component) found during parsing.", @@ -165,8 +169,8 @@ "name": "__init__", "type": "Function", "tier": "STANDARD", - "start_line": 142, - "end_line": 165, + "start_line": 144, + "end_line": 167, "tags": { "TIER": "STANDARD", "PURPOSE": "Initializes a new SemanticEntity instance.", @@ -185,8 +189,8 @@ "name": "get_tier", "type": "Function", "tier": "STANDARD", - "start_line": 167, - "end_line": 179, + "start_line": 169, + "end_line": 181, "tags": { "TIER": "STANDARD", "PURPOSE": "Returns the tier of the entity, defaulting to STANDARD.", @@ -205,8 +209,8 @@ "name": "to_dict", "type": "Function", "tier": "STANDARD", - "start_line": 181, - "end_line": 210, + "start_line": 183, + "end_line": 212, "tags": { "TIER": "STANDARD", "PURPOSE": "Serializes the entity to a dictionary for JSON output.", @@ -225,8 +229,8 @@ "name": "validate", "type": "Function", "tier": "CRITICAL", - "start_line": 212, - "end_line": 276, + "start_line": 214, + "end_line": 278, "tags": { "TIER": "CRITICAL", "PURPOSE": "Checks for semantic compliance based on TIER requirements.", @@ -246,8 +250,8 @@ "name": "get_score", "type": "Function", "tier": "STANDARD", - "start_line": 278, - "end_line": 312, + "start_line": 280, + "end_line": 314, "tags": { "TIER": "STANDARD", "PURPOSE": "Calculates a compliance score (0.0 to 1.0) based on tier requirements.", @@ -273,8 +277,8 @@ "name": "get_patterns", "type": "Function", "tier": "STANDARD", - "start_line": 316, - "end_line": 351, + "start_line": 318, + "end_line": 353, "tags": { "TIER": "STANDARD", "PURPOSE": "Returns regex patterns for a specific language.", @@ -294,8 +298,8 @@ "name": "extract_svelte_props", "type": "Function", "tier": "STANDARD", - "start_line": 354, - "end_line": 380, + "start_line": 356, + "end_line": 382, "tags": { "TIER": "STANDARD", "PURPOSE": "Extracts props from Svelte component script section.", @@ -314,8 +318,8 @@ "name": "extract_svelte_events", "type": "Function", "tier": "STANDARD", - "start_line": 383, - "end_line": 417, + "start_line": 385, + "end_line": 419, "tags": { "TIER": "STANDARD", "PURPOSE": "Extracts dispatched events from Svelte component.", @@ -334,8 +338,8 @@ "name": "extract_data_flow", "type": "Function", "tier": "STANDARD", - "start_line": 420, - "end_line": 470, + "start_line": 422, + "end_line": 472, "tags": { "TIER": "STANDARD", "PURPOSE": "Extracts store subscriptions and data flow from Svelte component.", @@ -354,8 +358,8 @@ "name": "parse_file", "type": "Function", "tier": "CRITICAL", - "start_line": 473, - "end_line": 660, + "start_line": 475, + "end_line": 662, "tags": { "TIER": "CRITICAL", "PURPOSE": "Parses a single file to extract semantic entities with tier awareness and enhanced Svelte analysis.", @@ -376,8 +380,8 @@ "name": "SemanticMapGenerator", "type": "Class", "tier": "CRITICAL", - "start_line": 663, - "end_line": 993, + "start_line": 665, + "end_line": 1153, "tags": { "TIER": "CRITICAL", "PURPOSE": "Orchestrates the mapping process with tier-based validation.", @@ -389,8 +393,8 @@ "name": "__init__", "type": "Function", "tier": "STANDARD", - "start_line": 668, - "end_line": 680, + "start_line": 670, + "end_line": 682, "tags": { "TIER": "STANDARD", "PURPOSE": "Initializes the generator with a root directory.", @@ -409,8 +413,8 @@ "name": "_load_gitignore", "type": "Function", "tier": "STANDARD", - "start_line": 682, - "end_line": 698, + "start_line": 684, + "end_line": 700, "tags": { "TIER": "STANDARD", "PURPOSE": "Loads patterns from .gitignore file.", @@ -429,8 +433,8 @@ "name": "_is_ignored", "type": "Function", "tier": "STANDARD", - "start_line": 700, - "end_line": 734, + "start_line": 702, + "end_line": 736, "tags": { "TIER": "STANDARD", "PURPOSE": "Checks if a path should be ignored based on .gitignore or hardcoded defaults.", @@ -449,8 +453,8 @@ "name": "run", "type": "Function", "tier": "CRITICAL", - "start_line": 736, - "end_line": 749, + "start_line": 738, + "end_line": 751, "tags": { "TIER": "CRITICAL", "PURPOSE": "Main execution flow.", @@ -478,8 +482,8 @@ "name": "_walk_and_parse", "type": "Function", "tier": "CRITICAL", - "start_line": 751, - "end_line": 780, + "start_line": 753, + "end_line": 782, "tags": { "TIER": "CRITICAL", "PURPOSE": "Recursively walks directories and triggers parsing.", @@ -498,8 +502,8 @@ "name": "_process_file_results", "type": "Function", "tier": "STANDARD", - "start_line": 782, - "end_line": 811, + "start_line": 784, + "end_line": 813, "tags": { "TIER": "STANDARD", "PURPOSE": "Validates entities and calculates file scores with tier awareness.", @@ -512,8 +516,8 @@ "name": "validate_recursive", "type": "Function", "tier": "STANDARD", - "start_line": 792, - "end_line": 805, + "start_line": 794, + "end_line": 807, "tags": { "TIER": "STANDARD", "PURPOSE": "Recursively validates a list of entities.", @@ -539,8 +543,8 @@ "name": "_generate_artifacts", "type": "Function", "tier": "CRITICAL", - "start_line": 813, - "end_line": 833, + "start_line": 815, + "end_line": 836, "tags": { "TIER": "CRITICAL", "PURPOSE": "Writes output files with tier-based compliance data.", @@ -559,8 +563,8 @@ "name": "_generate_report", "type": "Function", "tier": "CRITICAL", - "start_line": 835, - "end_line": 888, + "start_line": 838, + "end_line": 891, "tags": { "TIER": "CRITICAL", "PURPOSE": "Generates the Markdown compliance report with severity levels.", @@ -579,8 +583,8 @@ "name": "_collect_issues", "type": "Function", "tier": "STANDARD", - "start_line": 890, - "end_line": 902, + "start_line": 893, + "end_line": 905, "tags": { "TIER": "STANDARD", "PURPOSE": "Helper to collect issues for a specific file from the entity tree.", @@ -599,8 +603,8 @@ "name": "_generate_compressed_map", "type": "Function", "tier": "CRITICAL", - "start_line": 904, - "end_line": 921, + "start_line": 907, + "end_line": 924, "tags": { "TIER": "CRITICAL", "PURPOSE": "Generates the token-optimized project map with enhanced Svelte details.", @@ -619,8 +623,8 @@ "name": "_write_entity_md", "type": "Function", "tier": "CRITICAL", - "start_line": 923, - "end_line": 991, + "start_line": 926, + "end_line": 994, "tags": { "TIER": "CRITICAL", "PURPOSE": "Recursive helper to write entity tree to Markdown with tier badges and enhanced details.", @@ -634,6 +638,109 @@ "issues": [], "score": 1.0 } + }, + { + "name": "_generate_module_map", + "type": "Function", + "tier": "CRITICAL", + "start_line": 996, + "end_line": 1151, + "tags": { + "TIER": "CRITICAL", + "PURPOSE": "Generates a module-centric map grouping entities by directory structure.", + "PRE": "Entities have been processed.", + "POST": "Markdown module map is written to .ai/MODULE_MAP.md." + }, + "relations": [], + "children": [ + { + "name": "_get_module_path", + "type": "Function", + "tier": "STANDARD", + "start_line": 1008, + "end_line": 1020, + "tags": { + "TIER": "STANDARD", + "PURPOSE": "Extracts the module path from a file path.", + "PRE": "file_path is a valid relative path.", + "POST": "Returns a module path string." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1008 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1008 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1008 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1008 + } + ], + "score": 0.6 + } + }, + { + "name": "_collect_all_entities", + "type": "Function", + "tier": "STANDARD", + "start_line": 1022, + "end_line": 1031, + "tags": { + "TIER": "STANDARD", + "PURPOSE": "Flattens entity tree for easier grouping.", + "PRE": "entity list is valid.", + "POST": "Returns flat list of all entities with their hierarchy." + }, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1022 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1022 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1022 + }, + { + "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 1022 + } + ], + "score": 0.6 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } } ], "compliance": { @@ -646,8 +753,8 @@ "name": "to_dict", "type": "Function", "tier": "TRIVIAL", - "start_line": 128, - "end_line": 128, + "start_line": 130, + "end_line": 130, "tags": { "PURPOSE": "Auto-detected function (orphan)", "TIER": "TRIVIAL" @@ -667,19 +774,83 @@ "score": 1.0 } }, + { + "name": "TransactionCore", + "type": "Module", + "tier": "CRITICAL", + "start_line": 1, + "end_line": 79, + "tags": { + "TIER": "CRITICAL", + "SEMANTICS": "Finance, ACID, Transfer, Ledger", + "PURPOSE": "Core banking transaction processor with ACID guarantees.", + "LAYER": "Domain (Core)", + "INVARIANT": "Negative transfers are strictly forbidden.", + "TEST_DATA": "concurrency_lock -> {./fixtures/transactions.json#race_condition}" + }, + "relations": [ + { + "type": "DEPENDS_ON", + "target": "[DEF:Infra:PostgresDB]" + }, + { + "type": "DEPENDS_ON", + "target": "[DEF:Infra:AuditLog]" + } + ], + "children": [ + { + "name": "execute_transfer", + "type": "Function", + "tier": "STANDARD", + "start_line": 26, + "end_line": 77, + "tags": { + "PURPOSE": "Atomically move funds between accounts with audit trails.", + "PARAM": "amount (Decimal) - Positive amount to transfer.", + "PRE": "amount > 0; sender != receiver; sender_balance >= amount.", + "POST": "sender_balance -= amount; receiver_balance += amount; Audit Record Created.", + "SIDE_EFFECT": "Database mutation (Rows locked), Audit IO.", + "UX_STATE": "Error(System) -> 500 Internal -> UI shows \"Retry later\" toast.", + "UX_FEEDBACK": "Triggers specific UI flow for insufficient funds" + }, + "relations": [ + { + "type": "CALLS", + "target": "atomic_transaction" + } + ], + "children": [], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + } + ], + "compliance": { + "valid": true, + "issues": [], + "score": 1.0 + } + }, { "name": "PluginExampleShot", "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 67, + "end_line": 64, "tags": { - "PURPOSE": "Reference implementation of a plugin following GRACE standards." + "TIER": "STANDARD", + "SEMANTICS": "Plugin, Core, Extension", + "PURPOSE": "Reference implementation of a plugin following GRACE standards.", + "LAYER": "Domain (Business Logic)", + "INVARIANT": "get_schema must return valid JSON Schema." }, "relations": [ { - "type": "IMPLEMENTS", - "target": "[DEF:Std:Plugin]" + "type": "INHERITS", + "target": "PluginBase" } ], "children": [ @@ -687,10 +858,11 @@ "name": "get_schema", "type": "Function", "tier": "STANDARD", - "start_line": 26, - "end_line": 41, + "start_line": 18, + "end_line": 32, "tags": { - "PURPOSE": "Defines input validation schema." + "PURPOSE": "Defines input validation schema.", + "POST": "Returns dict compliant with JSON Schema draft 7." }, "relations": [], "children": [], @@ -700,48 +872,44 @@ { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 26 - }, - { - "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 26 + "line_number": 18 }, { "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", "severity": "WARNING", - "line_number": 26 + "line_number": 18 }, { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 26 - }, - { - "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 26 + "line_number": 18 }, { "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", "severity": "WARNING", - "line_number": 26 + "line_number": 18 } ], - "score": 0.26666666666666655 + "score": 0.5333333333333333 } }, { "name": "execute", "type": "Function", "tier": "STANDARD", - "start_line": 43, - "end_line": 65, + "start_line": 34, + "end_line": 62, "tags": { - "PURPOSE": "Core plugin logic with structured logging and progress reporting.", - "PARAM": "context (TaskContext) - Execution context with logging and progress tools." + "PURPOSE": "Core plugin logic with structured logging and scope isolation.", + "PARAM": "context (TaskContext) - Execution tools (log, progress).", + "SIDE_EFFECT": "Emits logs to centralized system." }, - "relations": [], + "relations": [ + { + "type": "BINDS_TO", + "target": "context.logger" + } + ], "children": [], "compliance": { "valid": true, @@ -749,59 +917,31 @@ { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 34 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 - }, - { - "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 43 + "line_number": 34 }, { "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 + "line_number": 34 }, { "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", "severity": "WARNING", - "line_number": 43 - }, - { - "message": "Missing Belief State Logging: Function should use belief_scope (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 43 + "line_number": 34 } ], - "score": 0.26666666666666655 + "score": 0.4666666666666666 } }, { "name": "id", "type": "Function", "tier": "TRIVIAL", - "start_line": 11, - "end_line": 11, - "tags": { - "PURPOSE": "Auto-detected function (orphan)", - "TIER": "TRIVIAL" - }, - "relations": [], - "children": [], - "compliance": { - "valid": true, - "issues": [], - "score": 1.0 - } - }, - { - "name": "name", - "type": "Function", - "tier": "TRIVIAL", "start_line": 15, "end_line": 15, "tags": { @@ -815,64 +955,12 @@ "issues": [], "score": 1.0 } - }, - { - "name": "description", - "type": "Function", - "tier": "TRIVIAL", - "start_line": 19, - "end_line": 19, - "tags": { - "PURPOSE": "Auto-detected function (orphan)", - "TIER": "TRIVIAL" - }, - "relations": [], - "children": [], - "compliance": { - "valid": true, - "issues": [], - "score": 1.0 - } - }, - { - "name": "version", - "type": "Function", - "tier": "TRIVIAL", - "start_line": 23, - "end_line": 23, - "tags": { - "PURPOSE": "Auto-detected function (orphan)", - "TIER": "TRIVIAL" - }, - "relations": [], - "children": [], - "compliance": { - "valid": true, - "issues": [], - "score": 1.0 - } } ], "compliance": { "valid": true, - "issues": [ - { - "message": "Missing Mandatory Tag: @LAYER (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - }, - { - "message": "Missing Mandatory Tag: @SEMANTICS (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - }, - { - "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - } - ], - "score": 0.5499999999999999 + "issues": [], + "score": 1.0 } }, { @@ -880,9 +968,13 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 57, + "end_line": 65, "tags": { - "PURPOSE": "Reference implementation of a task-based route using GRACE-Poly." + "TIER": "STANDARD", + "SEMANTICS": "Route, Task, API, Async", + "PURPOSE": "Reference implementation of a task-based route using GRACE-Poly.", + "LAYER": "Interface (API)", + "INVARIANT": "TaskManager must be available in dependency graph." }, "relations": [ { @@ -895,14 +987,15 @@ "name": "create_task", "type": "Function", "tier": "STANDARD", - "start_line": 20, - "end_line": 55, + "start_line": 24, + "end_line": 63, "tags": { "PURPOSE": "Create and start a new task using TaskManager. Non-blocking.", - "PARAM": "config (ConfigManager) - Centralized configuration.", - "PRE": "plugin_id must exist; config must be initialized.", + "PARAM": "task_manager (TaskManager) - Async task executor.", + "PRE": "plugin_id must match a registered plugin.", "POST": "A new task is spawned; Task ID returned immediately.", - "UX_STATE": "Error feedback to frontend" + "SIDE_EFFECT": "Writes to DB, Trigger background worker.", + "UX_STATE": "Error feedback -> 500 Internal Error" }, "relations": [ { @@ -920,44 +1013,77 @@ ], "compliance": { "valid": true, - "issues": [ - { - "message": "Missing Mandatory Tag: @LAYER (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - }, - { - "message": "Missing Mandatory Tag: @SEMANTICS (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - }, - { - "message": "Missing Mandatory Tag: @TIER (required for STANDARD tier)", - "severity": "WARNING", - "line_number": 1 - } - ], - "score": 0.5499999999999999 + "issues": [], + "score": 1.0 } }, { "name": "FrontendComponentShot", "type": "Component", - "tier": "STANDARD", + "tier": "CRITICAL", "start_line": 1, - "end_line": 59, + "end_line": 70, "tags": { - "TIER": "STANDARD", - "PURPOSE": "Reference implementation of a task-spawning component using", - "LAYER": "UI", - "SEMANTICS": "Task, Creation, Button", - "RELATION": "IMPLEMENTS -> [DEF:Std:UI_Svelte]", - "UX_STATE": "Loading -> Button disabled with spinner while postApi resolves.", - "UX_FEEDBACK": "toast.success on completion; toast.error on failure.", - "UX_TEST": "Idle -> {click: spawnTask, expected: loading state then success}" + "TIER": "CRITICAL", + "SEMANTICS": "Task, Button, Action, UX", + "PURPOSE": "Action button to spawn a new task with full UX feedback cycle.", + "LAYER": "UI (Presentation)", + "RELATION": "CALLS -> postApi", + "INVARIANT": "Must prevent double-submission while loading.", + "TEST_DATA": "loading_state -> {\"isLoading\": true}", + "UX_STATE": "Error -> Toast notification triggers.", + "UX_FEEDBACK": "Toast success/error.", + "UX_TEST": "Success -> {api_resolve: 200, expected: toast.success called}" }, "relations": [], - "children": [], + "children": [ + { + "name": "spawnTask", + "type": "Function", + "tier": "STANDARD", + "start_line": 31, + "end_line": 56, + "tags": {}, + "relations": [], + "children": [], + "compliance": { + "valid": true, + "issues": [ + { + "message": "Missing Mandatory Tag: @PURPOSE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @PURPOSE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @PRE (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + }, + { + "message": "Missing Mandatory Tag: @POST (required for STANDARD tier)", + "severity": "WARNING", + "line_number": 31 + } + ], + "score": 0.1999999999999999 + } + } + ], "compliance": { "valid": true, "issues": [], @@ -979,32 +1105,32 @@ { "store": "lib", "type": "READS_FROM", - "line": 15 + "line": 22 }, { "store": "lib", "type": "READS_FROM", - "line": 16 + "line": 23 }, { "store": "lib", "type": "READS_FROM", - "line": 17 + "line": 24 }, { "store": "t", "type": "READS_FROM", - "line": 35 + "line": 46 }, { "store": "t", "type": "READS_FROM", - "line": 39 + "line": 51 }, { "store": "t", "type": "READS_FROM", - "line": 57 + "line": 68 } ] }, @@ -8234,6 +8360,16 @@ "type": "WRITES_TO", "line": 12 }, + { + "store": "state", + "type": "WRITES_TO", + "line": 16 + }, + { + "store": "state", + "type": "WRITES_TO", + "line": 17 + }, { "store": "effect", "type": "READS_FROM", @@ -14643,7 +14779,7 @@ "type": "Module", "tier": "STANDARD", "start_line": 1, - "end_line": 284, + "end_line": 279, "tags": { "SEMANTICS": "config, manager, persistence, json", "PURPOSE": "Manages application configuration, including loading/saving to JSON and CRUD for environments.", @@ -14671,7 +14807,7 @@ "type": "Class", "tier": "STANDARD", "start_line": 22, - "end_line": 282, + "end_line": 277, "tags": { "PURPOSE": "A class to handle application configuration persistence and management." }, @@ -14707,7 +14843,7 @@ "type": "Function", "tier": "STANDARD", "start_line": 52, - "end_line": 88, + "end_line": 83, "tags": { "PURPOSE": "Loads the configuration from disk or creates a default one.", "PRE": "self.config_path is set.", @@ -14726,8 +14862,8 @@ "name": "_save_config_to_disk", "type": "Function", "tier": "STANDARD", - "start_line": 90, - "end_line": 109, + "start_line": 85, + "end_line": 104, "tags": { "PURPOSE": "Saves the provided configuration object to disk.", "PRE": "isinstance(config, AppConfig)", @@ -14746,8 +14882,8 @@ "name": "save", "type": "Function", "tier": "STANDARD", - "start_line": 111, - "end_line": 118, + "start_line": 106, + "end_line": 113, "tags": { "PURPOSE": "Saves the current configuration state to disk.", "PRE": "self.config is set.", @@ -14765,8 +14901,8 @@ "name": "get_config", "type": "Function", "tier": "STANDARD", - "start_line": 120, - "end_line": 128, + "start_line": 115, + "end_line": 123, "tags": { "PURPOSE": "Returns the current configuration.", "PRE": "self.config is set.", @@ -14785,8 +14921,8 @@ "name": "update_global_settings", "type": "Function", "tier": "STANDARD", - "start_line": 130, - "end_line": 150, + "start_line": 125, + "end_line": 145, "tags": { "PURPOSE": "Updates the global settings and persists the change.", "PRE": "isinstance(settings, GlobalSettings)", @@ -14805,8 +14941,8 @@ "name": "validate_path", "type": "Function", "tier": "STANDARD", - "start_line": 152, - "end_line": 171, + "start_line": 147, + "end_line": 166, "tags": { "PURPOSE": "Validates if a path exists and is writable.", "PRE": "path is a string.", @@ -14826,8 +14962,8 @@ "name": "get_environments", "type": "Function", "tier": "STANDARD", - "start_line": 173, - "end_line": 181, + "start_line": 168, + "end_line": 176, "tags": { "PURPOSE": "Returns the list of configured environments.", "PRE": "self.config is set.", @@ -14846,8 +14982,8 @@ "name": "has_environments", "type": "Function", "tier": "STANDARD", - "start_line": 183, - "end_line": 191, + "start_line": 178, + "end_line": 186, "tags": { "PURPOSE": "Checks if at least one environment is configured.", "PRE": "self.config is set.", @@ -14866,8 +15002,8 @@ "name": "get_environment", "type": "Function", "tier": "STANDARD", - "start_line": 193, - "end_line": 205, + "start_line": 188, + "end_line": 200, "tags": { "PURPOSE": "Returns a single environment by ID.", "PRE": "self.config is set and isinstance(env_id, str) and len(env_id) > 0.", @@ -14887,8 +15023,8 @@ "name": "add_environment", "type": "Function", "tier": "STANDARD", - "start_line": 207, - "end_line": 226, + "start_line": 202, + "end_line": 221, "tags": { "PURPOSE": "Adds a new environment to the configuration.", "PRE": "isinstance(env, Environment)", @@ -14907,8 +15043,8 @@ "name": "update_environment", "type": "Function", "tier": "STANDARD", - "start_line": 228, - "end_line": 257, + "start_line": 223, + "end_line": 252, "tags": { "PURPOSE": "Updates an existing environment.", "PRE": "isinstance(env_id, str) and len(env_id) > 0 and isinstance(updated_env, Environment)", @@ -14928,8 +15064,8 @@ "name": "delete_environment", "type": "Function", "tier": "STANDARD", - "start_line": 259, - "end_line": 280, + "start_line": 254, + "end_line": 275, "tags": { "PURPOSE": "Deletes an environment by ID.", "PRE": "isinstance(env_id, str) and len(env_id) > 0", @@ -25044,7 +25180,7 @@ "type": "Module", "tier": "TRIVIAL", "start_line": 1, - "end_line": 42, + "end_line": 44, "tags": { "TIER": "TRIVIAL", "SEMANTICS": "storage, file, model, pydantic", @@ -25076,7 +25212,7 @@ "type": "Class", "tier": "TRIVIAL", "start_line": 20, - "end_line": 28, + "end_line": 30, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Configuration model for the storage system, defining paths and naming patterns." @@ -25093,8 +25229,8 @@ "name": "StoredFile", "type": "Class", "tier": "TRIVIAL", - "start_line": 30, - "end_line": 40, + "start_line": 32, + "end_line": 42, "tags": { "TIER": "TRIVIAL", "PURPOSE": "Data model representing metadata for a file stored in the system." diff --git a/ut b/ut new file mode 100644 index 0000000..47c4c14 --- /dev/null +++ b/ut @@ -0,0 +1,15 @@ +Prepended http:// to './RealiTLScanner' +--2026-02-20 11:14:59-- http://./RealiTLScanner +Распознаётся . (.)… ошибка: С именем узла не связано ни одного адреса. +wget: не удаётся разрешить адрес ‘.’ +Prepended http:// to 'www.microsoft.com' +--2026-02-20 11:14:59-- http://www.microsoft.com/ +Распознаётся www.microsoft.com (www.microsoft.com)… 95.100.178.81 +Подключение к www.microsoft.com (www.microsoft.com)|95.100.178.81|:80... соединение установлено. +HTTP-запрос отправлен. Ожидание ответа… 403 Forbidden +2026-02-20 11:15:00 ОШИБКА 403: Forbidden. + +Prepended http:// to 'file.csv' +--2026-02-20 11:15:00-- http://file.csv/ +Распознаётся file.csv (file.csv)… ошибка: Неизвестное имя или служба. +wget: не удаётся разрешить адрес ‘file.csv’