diff --git a/backend/src/api/routes/dashboards.py b/backend/src/api/routes/dashboards.py index 79705c3..603322f 100644 --- a/backend/src/api/routes/dashboards.py +++ b/backend/src/api/routes/dashboards.py @@ -18,7 +18,7 @@ from ...dependencies import get_config_manager, get_task_manager, get_resource_s from ...core.logger import logger, belief_scope # [/SECTION] -router = APIRouter() +router = APIRouter(prefix="/api/dashboards", tags=["Dashboards"]) # [DEF:GitStatus:DataClass] class GitStatus(BaseModel): @@ -65,7 +65,7 @@ class DashboardsResponse(BaseModel): # @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) # @RETURN: DashboardsResponse - List of dashboards with status metadata # @RELATION: CALLS -> ResourceService.get_dashboards_with_status -@router.get("/api/dashboards", response_model=DashboardsResponse) +@router.get("", response_model=DashboardsResponse) async def get_dashboards( env_id: str, search: Optional[str] = None, @@ -157,7 +157,7 @@ class TaskResponse(BaseModel): # @RETURN: TaskResponse - Task ID for tracking # @RELATION: DISPATCHES -> MigrationPlugin # @RELATION: CALLS -> task_manager.create_task -@router.post("/api/dashboards/migrate", response_model=TaskResponse) +@router.post("/migrate", response_model=TaskResponse) async def migrate_dashboards( request: MigrateRequest, config_manager=Depends(get_config_manager), @@ -225,7 +225,7 @@ class BackupRequest(BaseModel): # @RETURN: TaskResponse - Task ID for tracking # @RELATION: DISPATCHES -> BackupPlugin # @RELATION: CALLS -> task_manager.create_task -@router.post("/api/dashboards/backup", response_model=TaskResponse) +@router.post("/backup", response_model=TaskResponse) async def backup_dashboards( request: BackupRequest, config_manager=Depends(get_config_manager), @@ -289,7 +289,7 @@ class DatabaseMappingsResponse(BaseModel): # @PARAM: target_env_id (str) - Target environment ID # @RETURN: DatabaseMappingsResponse - List of suggested mappings # @RELATION: CALLS -> MappingService.get_suggestions -@router.get("/api/dashboards/db-mappings", response_model=DatabaseMappingsResponse) +@router.get("/db-mappings", response_model=DatabaseMappingsResponse) async def get_database_mappings( source_env_id: str, target_env_id: str, diff --git a/backend/src/api/routes/datasets.py b/backend/src/api/routes/datasets.py index 89a0de4..1e8ea8c 100644 --- a/backend/src/api/routes/datasets.py +++ b/backend/src/api/routes/datasets.py @@ -19,7 +19,7 @@ from ...core.logger import logger, belief_scope from ...core.superset_client import SupersetClient # [/SECTION] -router = APIRouter() +router = APIRouter(prefix="/api/datasets", tags=["Datasets"]) # [DEF:MappedFields:DataClass] class MappedFields(BaseModel): @@ -63,8 +63,8 @@ class DatasetColumn(BaseModel): # [DEF:DatasetDetailResponse:DataClass] class DatasetDetailResponse(BaseModel): id: int - table_name: str - schema: str + table_name: Optional[str] = None + schema: Optional[str] = None database: str description: Optional[str] = None columns: List[DatasetColumn] @@ -99,7 +99,7 @@ class TaskResponse(BaseModel): # @PARAM: search (Optional[str]) - Filter by table name # @RETURN: List[int] - List of dataset IDs # @RELATION: CALLS -> ResourceService.get_datasets_with_status -@router.get("/api/datasets/ids") +@router.get("/ids") async def get_dataset_ids( env_id: str, search: Optional[str] = None, @@ -155,7 +155,7 @@ async def get_dataset_ids( # @PARAM: page_size (Optional[int]) - Items per page (default: 10, max: 100) # @RETURN: DatasetsResponse - List of datasets with status metadata # @RELATION: CALLS -> ResourceService.get_datasets_with_status -@router.get("/api/datasets", response_model=DatasetsResponse) +@router.get("", response_model=DatasetsResponse) async def get_datasets( env_id: str, search: Optional[str] = None, @@ -241,7 +241,7 @@ class MapColumnsRequest(BaseModel): # @RETURN: TaskResponse - Task ID for tracking # @RELATION: DISPATCHES -> MapperPlugin # @RELATION: CALLS -> task_manager.create_task -@router.post("/api/datasets/map-columns", response_model=TaskResponse) +@router.post("/map-columns", response_model=TaskResponse) async def map_columns( request: MapColumnsRequest, config_manager=Depends(get_config_manager), @@ -310,7 +310,7 @@ class GenerateDocsRequest(BaseModel): # @RETURN: TaskResponse - Task ID for tracking # @RELATION: DISPATCHES -> LLMAnalysisPlugin # @RELATION: CALLS -> task_manager.create_task -@router.post("/api/datasets/generate-docs", response_model=TaskResponse) +@router.post("/generate-docs", response_model=TaskResponse) async def generate_docs( request: GenerateDocsRequest, config_manager=Depends(get_config_manager), @@ -363,7 +363,7 @@ async def generate_docs( # @PARAM: dataset_id (int) - The dataset ID # @RETURN: DatasetDetailResponse - Detailed dataset information # @RELATION: CALLS -> SupersetClient.get_dataset_detail -@router.get("/api/datasets/{dataset_id}", response_model=DatasetDetailResponse) +@router.get("/{dataset_id}", response_model=DatasetDetailResponse) async def get_dataset_detail( env_id: str, dataset_id: int, diff --git a/backend/src/api/routes/environments.py b/backend/src/api/routes/environments.py index 3841517..b09adfd 100644 --- a/backend/src/api/routes/environments.py +++ b/backend/src/api/routes/environments.py @@ -18,7 +18,7 @@ from pydantic import BaseModel, Field from ...core.logger import belief_scope # [/SECTION] -router = APIRouter() +router = APIRouter(prefix="/api/environments", tags=["Environments"]) # [DEF:ScheduleSchema:DataClass] class ScheduleSchema(BaseModel): @@ -43,6 +43,8 @@ class DatabaseResponse(BaseModel): # [DEF:get_environments:Function] # @PURPOSE: List all configured environments. +# @LAYER: API +# @SEMANTICS: list, environments, config # @PRE: config_manager is injected via Depends. # @POST: Returns a list of EnvironmentResponse objects. # @RETURN: List[EnvironmentResponse] @@ -71,6 +73,8 @@ async def get_environments( # [DEF:update_environment_schedule:Function] # @PURPOSE: Update backup schedule for an environment. +# @LAYER: API +# @SEMANTICS: update, schedule, backup, environment # @PRE: Environment id exists, schedule is valid ScheduleSchema. # @POST: Backup schedule updated and scheduler reloaded. # @PARAM: id (str) - The environment ID. @@ -103,6 +107,8 @@ async def update_environment_schedule( # [DEF:get_environment_databases:Function] # @PURPOSE: Fetch the list of databases from a specific environment. +# @LAYER: API +# @SEMANTICS: fetch, databases, superset, environment # @PRE: Environment id exists. # @POST: Returns a list of database summaries from the environment. # @PARAM: id (str) - The environment ID. diff --git a/backend/src/api/routes/git.py b/backend/src/api/routes/git.py index fe3d342..9c9fef7 100644 --- a/backend/src/api/routes/git.py +++ b/backend/src/api/routes/git.py @@ -26,7 +26,7 @@ from src.api.routes.git_schemas import ( from src.services.git_service import GitService from src.core.logger import logger, belief_scope -router = APIRouter(prefix="/api/git", tags=["git"]) +router = APIRouter(tags=["git"]) git_service = GitService() # [DEF:get_git_configs:Function] diff --git a/backend/src/api/routes/llm.py b/backend/src/api/routes/llm.py index 43ea65c..0e57181 100644 --- a/backend/src/api/routes/llm.py +++ b/backend/src/api/routes/llm.py @@ -16,7 +16,7 @@ from sqlalchemy.orm import Session # [DEF:router:Global] # @PURPOSE: APIRouter instance for LLM routes. -router = APIRouter(prefix="/api/llm", tags=["LLM"]) +router = APIRouter(tags=["LLM"]) # [/DEF:router:Global] # [DEF:get_providers:Function] diff --git a/backend/src/api/routes/mappings.py b/backend/src/api/routes/mappings.py index 455620f..be0e5dd 100644 --- a/backend/src/api/routes/mappings.py +++ b/backend/src/api/routes/mappings.py @@ -21,7 +21,7 @@ from ...models.mapping import DatabaseMapping from pydantic import BaseModel # [/SECTION] -router = APIRouter(prefix="/api/mappings", tags=["mappings"]) +router = APIRouter(tags=["mappings"]) # [DEF:MappingCreate:DataClass] class MappingCreate(BaseModel): diff --git a/backend/src/api/routes/migration.py b/backend/src/api/routes/migration.py index 4202b63..b3d7264 100644 --- a/backend/src/api/routes/migration.py +++ b/backend/src/api/routes/migration.py @@ -44,7 +44,7 @@ async def get_dashboards( # @POST: Starts the migration task and returns the task ID. # @PARAM: selection (DashboardSelection) - The dashboards to migrate. # @RETURN: Dict - {"task_id": str, "message": str} -@router.post("/migration/execute") +@router.post("/execute") async def execute_migration( selection: DashboardSelection, config_manager=Depends(get_config_manager), diff --git a/backend/src/api/routes/settings.py b/backend/src/api/routes/settings.py index 0f400d5..fffb92d 100755 --- a/backend/src/api/routes/settings.py +++ b/backend/src/api/routes/settings.py @@ -283,6 +283,7 @@ class ConsolidatedSettingsResponse(BaseModel): environments: List[dict] connections: List[dict] llm: dict + llm_providers: List[dict] logging: dict storage: dict # [/DEF:ConsolidatedSettingsResponse:Class] @@ -302,13 +303,74 @@ async def get_consolidated_settings( config = config_manager.get_config() + from ...services.llm_provider import LLMProviderService + from ...core.database import SessionLocal + db = SessionLocal() + try: + llm_service = LLMProviderService(db) + providers = llm_service.get_all_providers() + llm_providers_list = [ + { + "id": p.id, + "provider_type": p.provider_type, + "name": p.name, + "base_url": p.base_url, + "api_key": "********", + "default_model": p.default_model, + "is_active": p.is_active + } for p in providers + ] + finally: + db.close() + return ConsolidatedSettingsResponse( environments=[env.dict() for env in config.environments], connections=config.settings.connections, llm=config.settings.llm, + llm_providers=llm_providers_list, logging=config.settings.logging.dict(), storage=config.settings.storage.dict() ) # [/DEF:get_consolidated_settings:Function] +# [DEF:update_consolidated_settings:Function] +# @PURPOSE: Bulk update application settings from the consolidated view. +# @PRE: User has admin permissions, config is valid. +# @POST: Settings are updated and saved via ConfigManager. +@router.patch("/consolidated") +async def update_consolidated_settings( + settings_patch: dict, + config_manager: ConfigManager = Depends(get_config_manager), + _ = Depends(has_permission("admin:settings", "WRITE")) +): + with belief_scope("update_consolidated_settings"): + logger.info("[update_consolidated_settings][Entry] Applying consolidated settings patch") + + current_config = config_manager.get_config() + current_settings = current_config.settings + + # Update connections if provided + if "connections" in settings_patch: + current_settings.connections = settings_patch["connections"] + + # Update LLM if provided + if "llm" in settings_patch: + current_settings.llm = settings_patch["llm"] + + # Update Logging if provided + if "logging" in settings_patch: + current_settings.logging = LoggingConfig(**settings_patch["logging"]) + + # Update Storage if provided + if "storage" in settings_patch: + new_storage = StorageConfig(**settings_patch["storage"]) + is_valid, message = config_manager.validate_path(new_storage.root_path) + if not is_valid: + raise HTTPException(status_code=400, detail=message) + current_settings.storage = new_storage + + config_manager.update_global_settings(current_settings) + return {"status": "success", "message": "Settings updated"} +# [/DEF:update_consolidated_settings:Function] + # [/DEF:SettingsRouter:Module] diff --git a/backend/src/app.py b/backend/src/app.py index 0793a36..15fe312 100755 --- a/backend/src/app.py +++ b/backend/src/app.py @@ -115,14 +115,21 @@ app.include_router(plugins.router, prefix="/api/plugins", tags=["Plugins"]) app.include_router(tasks.router, prefix="/api/tasks", tags=["Tasks"]) app.include_router(settings.router, prefix="/api/settings", tags=["Settings"]) app.include_router(connections.router, prefix="/api/settings/connections", tags=["Connections"]) -app.include_router(environments.router, prefix="/api/settings/environments", tags=["Environments"]) -app.include_router(mappings.router) +app.include_router(environments.router, tags=["Environments"]) +app.include_router(mappings.router, prefix="/api/mappings", tags=["Mappings"]) app.include_router(migration.router) -app.include_router(git.router) -app.include_router(llm.router) +app.include_router(git.router, prefix="/api/git", tags=["Git"]) +app.include_router(llm.router, prefix="/api/llm", tags=["LLM"]) app.include_router(storage.router, prefix="/api/storage", tags=["Storage"]) -app.include_router(dashboards.router, tags=["Dashboards"]) -app.include_router(datasets.router, tags=["Datasets"]) +app.include_router(dashboards.router) +app.include_router(datasets.router) + + +# [DEF:api.include_routers:Action] +# @PURPOSE: Registers all API routers with the FastAPI application. +# @LAYER: API +# @SEMANTICS: routes, registration, api +# [/DEF:api.include_routers:Action] # [DEF:websocket_endpoint:Function] # @PURPOSE: Provides a WebSocket endpoint for real-time log streaming of a task with server-side filtering. @@ -233,25 +240,20 @@ async def websocket_endpoint( frontend_path = project_root / "frontend" / "build" if frontend_path.exists(): app.mount("/_app", StaticFiles(directory=str(frontend_path / "_app")), name="static") - - # Serve other static files from the root of build directory - # [DEF:serve_spa:Function] - # @PURPOSE: Serves frontend static files or index.html for SPA routing. - # @PRE: file_path is requested by the client. - # @POST: Returns the requested file or index.html as a fallback. - @app.get("/{file_path:path}") + + @app.get("/{file_path:path}", include_in_schema=False) async def serve_spa(file_path: str): - with belief_scope("serve_spa", f"path={file_path}"): - # Don't serve SPA for API routes that fell through - if file_path.startswith("api/"): - logger.info(f"[DEBUG] API route fell through to serve_spa: {file_path}") - raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}") - - full_path = frontend_path / file_path - if full_path.is_file(): - return FileResponse(str(full_path)) - # Fallback to index.html for SPA routing - return FileResponse(str(frontend_path / "index.html")) + # Only serve SPA for non-API paths + # API routes are registered separately and should be matched by FastAPI first + if file_path and (file_path.startswith("api/") or file_path.startswith("/api/") or file_path == "api"): + # This should not happen if API routers are properly registered + # Return 404 instead of serving HTML + raise HTTPException(status_code=404, detail=f"API endpoint not found: {file_path}") + + full_path = frontend_path / file_path + if file_path and full_path.is_file(): + return FileResponse(str(full_path)) + return FileResponse(str(frontend_path / "index.html")) # [/DEF:serve_spa:Function] else: # [DEF:read_root:Function] diff --git a/backend/src/core/superset_client.py b/backend/src/core/superset_client.py index fe08433..102a574 100644 --- a/backend/src/core/superset_client.py +++ b/backend/src/core/superset_client.py @@ -247,8 +247,14 @@ class SupersetClient: def get_dataset_detail(self, dataset_id: int) -> Dict: with belief_scope("SupersetClient.get_dataset_detail", f"id={dataset_id}"): # Get base dataset info - dataset = self.get_dataset(dataset_id) + response = self.get_dataset(dataset_id) + # If the response is a dict and has a 'result' key, use that (standard Superset API) + if isinstance(response, dict) and 'result' in response: + dataset = response['result'] + else: + dataset = response + # Extract columns information columns = dataset.get("columns", []) column_info = [] diff --git a/backend/tasks.db b/backend/tasks.db index 3e64da2..43ffa31 100644 Binary files a/backend/tasks.db and b/backend/tasks.db differ diff --git a/backend/tests/test_dashboards_api.py b/backend/tests/test_dashboards_api.py index 44757ca..6f0be9c 100644 --- a/backend/tests/test_dashboards_api.py +++ b/backend/tests/test_dashboards_api.py @@ -1,6 +1,8 @@ # [DEF:backend.tests.test_dashboards_api:Module] # @TIER: STANDARD # @PURPOSE: Contract-driven tests for Dashboard Hub API +# @LAYER: Domain (Tests) +# @SEMANTICS: tests, dashboards, api, contract # @RELATION: TESTS -> backend.src.api.routes.dashboards from fastapi.testclient import TestClient @@ -48,6 +50,8 @@ def test_get_dashboards_success(): # Validate against Pydantic model DashboardsResponse(**data) +# [/DEF:test_get_dashboards_success:Function] + # [DEF:test_get_dashboards_env_not_found:Function] # @TEST: GET /api/dashboards returns 404 if env_id missing # @PRE: env_id does not exist @@ -64,4 +68,6 @@ def test_get_dashboards_env_not_found(): assert response.status_code == 404 assert "Environment not found" in response.json()["detail"] +# [/DEF:test_get_dashboards_env_not_found:Function] + # [/DEF:backend.tests.test_dashboards_api:Module] diff --git a/backend/tests/test_resource_service.py b/backend/tests/test_resource_service.py index 9e38576..a55292f 100644 --- a/backend/tests/test_resource_service.py +++ b/backend/tests/test_resource_service.py @@ -44,4 +44,6 @@ async def test_get_dashboards_with_status(): assert result[0]["last_task"]["task_id"] == "task-123" assert result[0]["last_task"]["status"] == "RUNNING" + # [/DEF:test_get_dashboards_with_status:Function] + # [/DEF:backend.tests.test_resource_service:Module] diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte index d5de61f..dc94880 100644 --- a/frontend/src/routes/+layout.svelte +++ b/frontend/src/routes/+layout.svelte @@ -1,3 +1,18 @@ + + + @@ -167,6 +255,12 @@ > {$t.settings?.environments || 'Environments'} + - - + + {#if !editingEnvId && !isAddingEnv} +
+ +
+ {/if} + + {#if editingEnvId || isAddingEnv} + +
+

{editingEnvId ? 'Edit' : 'Add'} Environment

+
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ + +
+
+ +

Backup Schedule

+
+
+ + +
+
+ + +

Example: 0 0 * * * (daily at midnight)

+
+
+ +
+ + +
+
+ {/if} + {#if settings.environments && settings.environments.length > 0} -
+
@@ -225,13 +387,21 @@ - +
{env.name} {env.url} {env.username}{env.is_default ? 'Yes' : 'No'} + {#if env.is_default} + + Yes + + {:else} + No + {/if} +
+ {:else if !isAddingEnv} +
+

Warning

+

No Superset environments configured. You must add at least one environment to perform backups or migrations.

+
{/if}
- {:else if activeTab === 'connections'} - -
-

{$t.settings?.connections || 'Database Connections'}

-

- {$t.settings?.connections_description || 'Configure database connections for data mapping.'} -

-
- {:else if activeTab === 'llm'} - -
-

{$t.settings?.llm || 'LLM Providers'}

-

- {$t.settings?.llm_description || 'Configure LLM providers for dataset documentation.'} -

-
- -
-
{:else if activeTab === 'logging'}
@@ -272,6 +426,76 @@

{$t.settings?.logging_description || 'Configure logging and task log levels.'}

+ +
+
+
+ + +
+
+ + +
+
+ +

Logs agent reasoning and internal state changes for debugging.

+
+
+ +
+ +
+
+
+ {:else if activeTab === 'connections'} + +
+

{$t.settings?.connections || 'Database Connections'}

+

+ {$t.settings?.connections_description || 'Configure database connections for data mapping.'} +

+ + {#if settings.connections && settings.connections.length > 0} + +

No additional connections configured. Superset database connections are used by default.

+ {:else} +
+

No external connections configured.

+ +
+ {/if} +
+ {:else if activeTab === 'llm'} + +
+

{$t.settings?.llm || 'LLM Providers'}

+

+ {$t.settings?.llm_description || 'Configure LLM providers for dataset documentation.'} +

+ +
{:else if activeTab === 'storage'} @@ -280,6 +504,32 @@

{$t.settings?.storage_description || 'Configure file storage paths and patterns.'}

+ +
+
+
+ + +
+
+ + +
+
+ + +
+
+ +
+ +
+
{/if} diff --git a/frontend/src/routes/storage/+page.svelte b/frontend/src/routes/storage/+page.svelte new file mode 100644 index 0000000..23cf814 --- /dev/null +++ b/frontend/src/routes/storage/+page.svelte @@ -0,0 +1,16 @@ + + + + diff --git a/frontend/src/routes/storage/backups/+page.svelte b/frontend/src/routes/storage/backups/+page.svelte new file mode 100644 index 0000000..cf7b396 --- /dev/null +++ b/frontend/src/routes/storage/backups/+page.svelte @@ -0,0 +1,35 @@ + + + + + + +
+ + +
+ +
+
+ + + diff --git a/frontend/src/routes/storage/repos/+page.svelte b/frontend/src/routes/storage/repos/+page.svelte new file mode 100644 index 0000000..d212bab --- /dev/null +++ b/frontend/src/routes/storage/repos/+page.svelte @@ -0,0 +1,110 @@ + + + + + + +
+ +
+