Файловое хранилище готово
This commit is contained in:
Submodule backend/backend/git_repos/12 updated: d592fa7ed5...f46772443a
@@ -1,269 +0,0 @@
|
||||
2025-12-20 19:55:11,325 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 19:55:11,325 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 19:55:11,327 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 43, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 21:01:49,905 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 21:01:49,906 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 21:01:49,988 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 21:01:49,990 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 22:42:32,538 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 22:42:32,538 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 22:42:32,583 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 22:42:32,587 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 22:54:29,770 - INFO - [BackupPlugin][Entry] Starting backup for .
|
||||
2025-12-20 22:54:29,771 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 22:54:29,831 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 22:54:29,833 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 22:54:34,078 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 22:54:34,078 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 22:54:34,079 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 22:54:34,079 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 22:59:25,060 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 22:59:25,060 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 22:59:25,114 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 22:59:25,117 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:00:31,156 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:00:31,156 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:00:31,157 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:00:31,162 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:00:34,710 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:00:34,710 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:00:34,710 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:00:34,711 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:01:43,894 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:01:43,894 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:01:43,895 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:01:43,895 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:04:07,731 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:04:07,731 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:04:07,732 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:04:07,732 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:06:39,641 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:06:39,642 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:06:39,687 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:06:39,689 - CRITICAL - [setup_clients][Failure] Critical error during client initialization: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
Traceback (most recent call last):
|
||||
File "/home/user/ss-tools/superset_tool/utils/init_clients.py", line 66, in setup_clients
|
||||
config = SupersetConfig(
|
||||
^^^^^^^^^^^^^^^
|
||||
File "/home/user/ss-tools/backend/.venv/lib/python3.12/site-packages/pydantic/main.py", line 250, in __init__
|
||||
validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self)
|
||||
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
|
||||
pydantic_core._pydantic_core.ValidationError: 1 validation error for SupersetConfig
|
||||
base_url
|
||||
Value error, Invalid URL format: https://superset.bebesh.ru. Must include '/api/v1'. [type=value_error, input_value='https://superset.bebesh.ru', input_type=str]
|
||||
For further information visit https://errors.pydantic.dev/2.12/v/value_error
|
||||
2025-12-20 23:30:36,090 - INFO - [BackupPlugin][Entry] Starting backup for superset.
|
||||
2025-12-20 23:30:36,093 - INFO - [setup_clients][Enter] Starting Superset clients initialization.
|
||||
2025-12-20 23:30:36,128 - INFO - [setup_clients][Action] Loading environments from ConfigManager
|
||||
2025-12-20 23:30:36,129 - INFO - [SupersetClient.__init__][Enter] Initializing SupersetClient.
|
||||
2025-12-20 23:30:36,129 - INFO - [APIClient.__init__][Entry] Initializing APIClient.
|
||||
2025-12-20 23:30:36,130 - WARNING - [_init_session][State] SSL verification disabled.
|
||||
2025-12-20 23:30:36,130 - INFO - [APIClient.__init__][Exit] APIClient initialized.
|
||||
2025-12-20 23:30:36,130 - INFO - [SupersetClient.__init__][Exit] SupersetClient initialized.
|
||||
2025-12-20 23:30:36,130 - INFO - [get_dashboards][Enter] Fetching dashboards.
|
||||
2025-12-20 23:30:36,131 - INFO - [authenticate][Enter] Authenticating to https://superset.bebesh.ru/api/v1
|
||||
2025-12-20 23:30:36,897 - INFO - [authenticate][Exit] Authenticated successfully.
|
||||
2025-12-20 23:30:37,527 - INFO - [get_dashboards][Exit] Found 11 dashboards.
|
||||
2025-12-20 23:30:37,527 - INFO - [BackupPlugin][Progress] Found 11 dashboards to export in superset.
|
||||
2025-12-20 23:30:37,529 - INFO - [export_dashboard][Enter] Exporting dashboard 11.
|
||||
2025-12-20 23:30:38,224 - INFO - [export_dashboard][Exit] Exported dashboard 11 to dashboard_export_20251220T203037.zip.
|
||||
2025-12-20 23:30:38,225 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:38,226 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/FCC New Coder Survey 2018/dashboard_export_20251220T203037.zip
|
||||
2025-12-20 23:30:38,227 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/FCC New Coder Survey 2018
|
||||
2025-12-20 23:30:38,230 - INFO - [export_dashboard][Enter] Exporting dashboard 10.
|
||||
2025-12-20 23:30:38,438 - INFO - [export_dashboard][Exit] Exported dashboard 10 to dashboard_export_20251220T203038.zip.
|
||||
2025-12-20 23:30:38,438 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:38,439 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/COVID Vaccine Dashboard/dashboard_export_20251220T203038.zip
|
||||
2025-12-20 23:30:38,439 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/COVID Vaccine Dashboard
|
||||
2025-12-20 23:30:38,440 - INFO - [export_dashboard][Enter] Exporting dashboard 9.
|
||||
2025-12-20 23:30:38,853 - INFO - [export_dashboard][Exit] Exported dashboard 9 to dashboard_export_20251220T203038.zip.
|
||||
2025-12-20 23:30:38,853 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:38,856 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Sales Dashboard/dashboard_export_20251220T203038.zip
|
||||
2025-12-20 23:30:38,856 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Sales Dashboard
|
||||
2025-12-20 23:30:38,858 - INFO - [export_dashboard][Enter] Exporting dashboard 8.
|
||||
2025-12-20 23:30:38,939 - INFO - [export_dashboard][Exit] Exported dashboard 8 to dashboard_export_20251220T203038.zip.
|
||||
2025-12-20 23:30:38,940 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:38,941 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Unicode Test/dashboard_export_20251220T203038.zip
|
||||
2025-12-20 23:30:38,941 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Unicode Test
|
||||
2025-12-20 23:30:38,942 - INFO - [export_dashboard][Enter] Exporting dashboard 7.
|
||||
2025-12-20 23:30:39,148 - INFO - [export_dashboard][Exit] Exported dashboard 7 to dashboard_export_20251220T203038.zip.
|
||||
2025-12-20 23:30:39,148 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:39,149 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Video Game Sales/dashboard_export_20251220T203038.zip
|
||||
2025-12-20 23:30:39,149 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Video Game Sales
|
||||
2025-12-20 23:30:39,150 - INFO - [export_dashboard][Enter] Exporting dashboard 6.
|
||||
2025-12-20 23:30:39,689 - INFO - [export_dashboard][Exit] Exported dashboard 6 to dashboard_export_20251220T203039.zip.
|
||||
2025-12-20 23:30:39,689 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:39,690 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Featured Charts/dashboard_export_20251220T203039.zip
|
||||
2025-12-20 23:30:39,691 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Featured Charts
|
||||
2025-12-20 23:30:39,692 - INFO - [export_dashboard][Enter] Exporting dashboard 5.
|
||||
2025-12-20 23:30:39,960 - INFO - [export_dashboard][Exit] Exported dashboard 5 to dashboard_export_20251220T203039.zip.
|
||||
2025-12-20 23:30:39,960 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:39,961 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Slack Dashboard/dashboard_export_20251220T203039.zip
|
||||
2025-12-20 23:30:39,961 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Slack Dashboard
|
||||
2025-12-20 23:30:39,962 - INFO - [export_dashboard][Enter] Exporting dashboard 4.
|
||||
2025-12-20 23:30:40,196 - INFO - [export_dashboard][Exit] Exported dashboard 4 to dashboard_export_20251220T203039.zip.
|
||||
2025-12-20 23:30:40,196 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:40,197 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/deck.gl Demo/dashboard_export_20251220T203039.zip
|
||||
2025-12-20 23:30:40,197 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/deck.gl Demo
|
||||
2025-12-20 23:30:40,198 - INFO - [export_dashboard][Enter] Exporting dashboard 3.
|
||||
2025-12-20 23:30:40,745 - INFO - [export_dashboard][Exit] Exported dashboard 3 to dashboard_export_20251220T203040.zip.
|
||||
2025-12-20 23:30:40,746 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:40,760 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/Misc Charts/dashboard_export_20251220T203040.zip
|
||||
2025-12-20 23:30:40,761 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/Misc Charts
|
||||
2025-12-20 23:30:40,762 - INFO - [export_dashboard][Enter] Exporting dashboard 2.
|
||||
2025-12-20 23:30:40,928 - INFO - [export_dashboard][Exit] Exported dashboard 2 to dashboard_export_20251220T203040.zip.
|
||||
2025-12-20 23:30:40,929 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:40,930 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/USA Births Names/dashboard_export_20251220T203040.zip
|
||||
2025-12-20 23:30:40,931 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/USA Births Names
|
||||
2025-12-20 23:30:40,932 - INFO - [export_dashboard][Enter] Exporting dashboard 1.
|
||||
2025-12-20 23:30:41,582 - INFO - [export_dashboard][Exit] Exported dashboard 1 to dashboard_export_20251220T203040.zip.
|
||||
2025-12-20 23:30:41,582 - INFO - [save_and_unpack_dashboard][Enter] Processing dashboard. Unpack: False
|
||||
2025-12-20 23:30:41,749 - INFO - [save_and_unpack_dashboard][State] Dashboard saved to: backups/SUPERSET/World Bank's Data/dashboard_export_20251220T203040.zip
|
||||
2025-12-20 23:30:41,750 - INFO - [archive_exports][Enter] Managing archive in backups/SUPERSET/World Bank's Data
|
||||
2025-12-20 23:30:41,752 - INFO - [consolidate_archive_folders][Enter] Consolidating archives in backups/SUPERSET
|
||||
2025-12-20 23:30:41,753 - INFO - [remove_empty_directories][Enter] Starting cleanup of empty directories in backups/SUPERSET
|
||||
2025-12-20 23:30:41,758 - INFO - [remove_empty_directories][Exit] Removed 0 empty directories.
|
||||
2025-12-20 23:30:41,758 - INFO - [BackupPlugin][CoherenceCheck:Passed] Backup logic completed for superset.
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -53,6 +53,7 @@ async def update_global_settings(
|
||||
):
|
||||
with belief_scope("update_global_settings"):
|
||||
logger.info("[update_global_settings][Entry] Updating global settings")
|
||||
|
||||
config_manager.update_global_settings(settings)
|
||||
return settings
|
||||
# [/DEF:update_global_settings:Function]
|
||||
@@ -207,30 +208,5 @@ async def test_environment_connection(
|
||||
return {"status": "error", "message": str(e)}
|
||||
# [/DEF:test_environment_connection:Function]
|
||||
|
||||
# [DEF:validate_backup_path:Function]
|
||||
# @PURPOSE: Validates if a backup path exists and is writable.
|
||||
# @PRE: Path is provided in path_data.
|
||||
# @POST: Returns success or error status.
|
||||
# @PARAM: path (str) - The path to validate.
|
||||
# @RETURN: dict - Validation result.
|
||||
@router.post("/validate-path")
|
||||
async def validate_backup_path(
|
||||
path_data: dict,
|
||||
config_manager: ConfigManager = Depends(get_config_manager)
|
||||
):
|
||||
with belief_scope("validate_backup_path"):
|
||||
path = path_data.get("path")
|
||||
if not path:
|
||||
raise HTTPException(status_code=400, detail="Path is required")
|
||||
|
||||
logger.info(f"[validate_backup_path][Entry] Validating path: {path}")
|
||||
|
||||
valid, message = config_manager.validate_path(path)
|
||||
|
||||
if not valid:
|
||||
return {"status": "error", "message": message}
|
||||
|
||||
return {"status": "success", "message": message}
|
||||
# [/DEF:validate_backup_path:Function]
|
||||
|
||||
# [/DEF:SettingsRouter:Module]
|
||||
|
||||
@@ -11,41 +11,47 @@
|
||||
from fastapi import APIRouter, Depends, UploadFile, File, Form, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
from typing import List, Optional
|
||||
from backend.src.models.storage import StoredFile, FileCategory
|
||||
from backend.src.dependencies import get_plugin_loader
|
||||
from backend.src.plugins.storage.plugin import StoragePlugin
|
||||
from backend.src.core.logger import belief_scope
|
||||
from ...models.storage import StoredFile, FileCategory
|
||||
from ...dependencies import get_plugin_loader
|
||||
from ...plugins.storage.plugin import StoragePlugin
|
||||
from ...core.logger import belief_scope
|
||||
# [/SECTION]
|
||||
|
||||
router = APIRouter(tags=["storage"])
|
||||
|
||||
# [DEF:list_files:Function]
|
||||
# @PURPOSE: List all files in the storage system, optionally filtered by category.
|
||||
# @PURPOSE: List all files and directories in the storage system.
|
||||
#
|
||||
# @PRE: None.
|
||||
# @POST: Returns a list of StoredFile objects.
|
||||
#
|
||||
# @PARAM: category (Optional[FileCategory]) - Filter by category.
|
||||
# @RETURN: List[StoredFile] - List of files.
|
||||
# @PARAM: path (Optional[str]) - Subpath within the category.
|
||||
# @RETURN: List[StoredFile] - List of files/directories.
|
||||
#
|
||||
# @RELATION: CALLS -> StoragePlugin.list_files
|
||||
@router.get("/files", response_model=List[StoredFile])
|
||||
async def list_files(category: Optional[FileCategory] = None, plugin_loader=Depends(get_plugin_loader)):
|
||||
async def list_files(
|
||||
category: Optional[FileCategory] = None,
|
||||
path: Optional[str] = None,
|
||||
plugin_loader=Depends(get_plugin_loader)
|
||||
):
|
||||
with belief_scope("list_files"):
|
||||
storage_plugin: StoragePlugin = plugin_loader.get_plugin("storage-manager")
|
||||
if not storage_plugin:
|
||||
raise HTTPException(status_code=500, detail="Storage plugin not loaded")
|
||||
return storage_plugin.list_files(category)
|
||||
return storage_plugin.list_files(category, path)
|
||||
# [/DEF:list_files:Function]
|
||||
|
||||
# [DEF:upload_file:Function]
|
||||
# @PURPOSE: Upload a file to the storage system under a specific category.
|
||||
# @PURPOSE: Upload a file to the storage system.
|
||||
#
|
||||
# @PRE: category must be a valid FileCategory.
|
||||
# @PRE: file must be a valid UploadFile.
|
||||
# @POST: Returns the StoredFile object of the uploaded file.
|
||||
#
|
||||
# @PARAM: category (FileCategory) - Target category.
|
||||
# @PARAM: path (Optional[str]) - Target subpath.
|
||||
# @PARAM: file (UploadFile) - The file content.
|
||||
# @RETURN: StoredFile - Metadata of the uploaded file.
|
||||
#
|
||||
@@ -55,6 +61,7 @@ async def list_files(category: Optional[FileCategory] = None, plugin_loader=Depe
|
||||
@router.post("/upload", response_model=StoredFile, status_code=201)
|
||||
async def upload_file(
|
||||
category: FileCategory = Form(...),
|
||||
path: Optional[str] = Form(None),
|
||||
file: UploadFile = File(...),
|
||||
plugin_loader=Depends(get_plugin_loader)
|
||||
):
|
||||
@@ -63,33 +70,32 @@ async def upload_file(
|
||||
if not storage_plugin:
|
||||
raise HTTPException(status_code=500, detail="Storage plugin not loaded")
|
||||
try:
|
||||
return await storage_plugin.save_file(file, category)
|
||||
return await storage_plugin.save_file(file, category, path)
|
||||
except ValueError as e:
|
||||
raise HTTPException(status_code=400, detail=str(e))
|
||||
# [/DEF:upload_file:Function]
|
||||
|
||||
# [DEF:delete_file:Function]
|
||||
# @PURPOSE: Delete a specific file from the storage system.
|
||||
# @PURPOSE: Delete a specific file or directory.
|
||||
#
|
||||
# @PRE: category must be a valid FileCategory.
|
||||
# @PRE: filename must not contain path separators.
|
||||
# @POST: File is removed from storage.
|
||||
# @POST: Item is removed from storage.
|
||||
#
|
||||
# @PARAM: category (FileCategory) - File category.
|
||||
# @PARAM: filename (str) - Name of the file.
|
||||
# @PARAM: path (str) - Relative path of the item.
|
||||
# @RETURN: None
|
||||
#
|
||||
# @SIDE_EFFECT: Deletes file from the filesystem.
|
||||
# @SIDE_EFFECT: Deletes item from the filesystem.
|
||||
#
|
||||
# @RELATION: CALLS -> StoragePlugin.delete_file
|
||||
@router.delete("/files/{category}/{filename}", status_code=204)
|
||||
async def delete_file(category: FileCategory, filename: str, plugin_loader=Depends(get_plugin_loader)):
|
||||
@router.delete("/files/{category}/{path:path}", status_code=204)
|
||||
async def delete_file(category: FileCategory, path: str, plugin_loader=Depends(get_plugin_loader)):
|
||||
with belief_scope("delete_file"):
|
||||
storage_plugin: StoragePlugin = plugin_loader.get_plugin("storage-manager")
|
||||
if not storage_plugin:
|
||||
raise HTTPException(status_code=500, detail="Storage plugin not loaded")
|
||||
try:
|
||||
storage_plugin.delete_file(category, filename)
|
||||
storage_plugin.delete_file(category, path)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
except ValueError as e:
|
||||
@@ -100,23 +106,23 @@ async def delete_file(category: FileCategory, filename: str, plugin_loader=Depen
|
||||
# @PURPOSE: Retrieve a file for download.
|
||||
#
|
||||
# @PRE: category must be a valid FileCategory.
|
||||
# @PRE: filename must exist in the specified category.
|
||||
# @POST: Returns a FileResponse.
|
||||
#
|
||||
# @PARAM: category (FileCategory) - File category.
|
||||
# @PARAM: filename (str) - Name of the file.
|
||||
# @PARAM: path (str) - Relative path of the file.
|
||||
# @RETURN: FileResponse - The file content.
|
||||
#
|
||||
# @RELATION: CALLS -> StoragePlugin.get_file_path
|
||||
@router.get("/download/{category}/{filename}")
|
||||
async def download_file(category: FileCategory, filename: str, plugin_loader=Depends(get_plugin_loader)):
|
||||
@router.get("/download/{category}/{path:path}")
|
||||
async def download_file(category: FileCategory, path: str, plugin_loader=Depends(get_plugin_loader)):
|
||||
with belief_scope("download_file"):
|
||||
storage_plugin: StoragePlugin = plugin_loader.get_plugin("storage-manager")
|
||||
if not storage_plugin:
|
||||
raise HTTPException(status_code=500, detail="Storage plugin not loaded")
|
||||
try:
|
||||
path = storage_plugin.get_file_path(category, filename)
|
||||
return FileResponse(path=path, filename=filename)
|
||||
abs_path = storage_plugin.get_file_path(category, path)
|
||||
filename = Path(path).name
|
||||
return FileResponse(path=abs_path, filename=filename)
|
||||
except FileNotFoundError:
|
||||
raise HTTPException(status_code=404, detail="File not found")
|
||||
except ValueError as e:
|
||||
|
||||
@@ -62,14 +62,18 @@ class ConfigManager:
|
||||
logger.info(f"[_load_config][Action] Config file not found. Creating default.")
|
||||
default_config = AppConfig(
|
||||
environments=[],
|
||||
settings=GlobalSettings(backup_path="backups")
|
||||
settings=GlobalSettings()
|
||||
)
|
||||
self._save_config_to_disk(default_config)
|
||||
return default_config
|
||||
|
||||
try:
|
||||
with open(self.config_path, "r") as f:
|
||||
data = json.load(f)
|
||||
|
||||
# Check for deprecated field
|
||||
if "settings" in data and "backup_path" in data["settings"]:
|
||||
del data["settings"]["backup_path"]
|
||||
|
||||
config = AppConfig(**data)
|
||||
logger.info(f"[_load_config][Coherence:OK] Configuration loaded")
|
||||
return config
|
||||
@@ -79,7 +83,7 @@ class ConfigManager:
|
||||
# For now, return default to be safe, but log the error prominently.
|
||||
return AppConfig(
|
||||
environments=[],
|
||||
settings=GlobalSettings(backup_path="backups")
|
||||
settings=GlobalSettings(storage=StorageConfig())
|
||||
)
|
||||
# [/DEF:_load_config:Function]
|
||||
|
||||
|
||||
@@ -43,7 +43,6 @@ class LoggingConfig(BaseModel):
|
||||
# [DEF:GlobalSettings:DataClass]
|
||||
# @PURPOSE: Represents global application settings.
|
||||
class GlobalSettings(BaseModel):
|
||||
backup_path: str
|
||||
storage: StorageConfig = Field(default_factory=StorageConfig)
|
||||
default_environment_id: Optional[str] = None
|
||||
logging: LoggingConfig = Field(default_factory=LoggingConfig)
|
||||
|
||||
@@ -5,13 +5,13 @@ from pydantic import BaseModel, Field
|
||||
|
||||
# [DEF:FileCategory:Class]
|
||||
class FileCategory(str, Enum):
|
||||
BACKUP = "backup"
|
||||
REPOSITORY = "repository"
|
||||
BACKUP = "backups"
|
||||
REPOSITORY = "repositorys"
|
||||
# [/DEF:FileCategory:Class]
|
||||
|
||||
# [DEF:StorageConfig:Class]
|
||||
class StorageConfig(BaseModel):
|
||||
root_path: str = Field(default="../ss-tools-storage", description="Absolute path to the storage root directory.")
|
||||
root_path: str = Field(default="backups", description="Absolute path to the storage root directory.")
|
||||
backup_structure_pattern: str = Field(default="{category}/", description="Pattern for backup directory structure.")
|
||||
repo_structure_pattern: str = Field(default="{category}/", description="Pattern for repository directory structure.")
|
||||
filename_pattern: str = Field(default="{name}_{timestamp}", description="Pattern for filenames.")
|
||||
|
||||
@@ -84,7 +84,7 @@ class BackupPlugin(PluginBase):
|
||||
with belief_scope("get_schema"):
|
||||
config_manager = get_config_manager()
|
||||
envs = [e.name for e in config_manager.get_environments()]
|
||||
default_path = config_manager.get_config().settings.backup_path
|
||||
default_path = config_manager.get_config().settings.storage.root_path
|
||||
|
||||
return {
|
||||
"type": "object",
|
||||
@@ -95,14 +95,8 @@ class BackupPlugin(PluginBase):
|
||||
"description": "The Superset environment to back up.",
|
||||
"enum": envs if envs else [],
|
||||
},
|
||||
"backup_path": {
|
||||
"type": "string",
|
||||
"title": "Backup Path",
|
||||
"description": "The root directory to save backups to.",
|
||||
"default": default_path
|
||||
}
|
||||
},
|
||||
"required": ["env", "backup_path"],
|
||||
"required": ["env"],
|
||||
}
|
||||
# [/DEF:get_schema:Function]
|
||||
|
||||
@@ -126,8 +120,9 @@ class BackupPlugin(PluginBase):
|
||||
if not env:
|
||||
raise KeyError("env")
|
||||
|
||||
backup_path_str = params.get("backup_path") or config_manager.get_config().settings.backup_path
|
||||
backup_path = Path(backup_path_str)
|
||||
storage_settings = config_manager.get_config().settings.storage
|
||||
# Use 'backups' subfolder within the storage root
|
||||
backup_path = Path(storage_settings.root_path) / "backups"
|
||||
|
||||
from ..core.logger import logger as app_logger
|
||||
app_logger.info(f"[BackupPlugin][Entry] Starting backup for {env}.")
|
||||
|
||||
@@ -98,14 +98,43 @@ class StoragePlugin(PluginBase):
|
||||
def get_storage_root(self) -> Path:
|
||||
with belief_scope("StoragePlugin:get_storage_root"):
|
||||
config_manager = get_config_manager()
|
||||
storage_config = config_manager.get_config().settings.storage
|
||||
root = Path(storage_config.root_path)
|
||||
global_settings = config_manager.get_config().settings
|
||||
|
||||
# Use storage.root_path as the source of truth for storage UI
|
||||
root = Path(global_settings.storage.root_path)
|
||||
|
||||
if not root.is_absolute():
|
||||
# Resolve relative to the workspace root (ss-tools)
|
||||
root = (Path(__file__).parents[4] / root).resolve()
|
||||
# Resolve relative to the backend directory
|
||||
# Path(__file__) is backend/src/plugins/storage/plugin.py
|
||||
# parents[3] is the project root (ss-tools)
|
||||
# We need to ensure it's relative to where backend/ is
|
||||
project_root = Path(__file__).parents[3]
|
||||
root = (project_root / root).resolve()
|
||||
return root
|
||||
# [/DEF:get_storage_root:Function]
|
||||
|
||||
# [DEF:resolve_path:Function]
|
||||
# @PURPOSE: Resolves a dynamic path pattern using provided variables.
|
||||
# @PARAM: pattern (str) - The path pattern to resolve.
|
||||
# @PARAM: variables (Dict[str, str]) - Variables to substitute in the pattern.
|
||||
# @RETURN: str - The resolved path.
|
||||
def resolve_path(self, pattern: str, variables: Dict[str, str]) -> str:
|
||||
with belief_scope("StoragePlugin:resolve_path"):
|
||||
# Add common variables
|
||||
vars_with_defaults = {
|
||||
"timestamp": datetime.now().strftime("%Y%m%dT%H%M%S"),
|
||||
**variables
|
||||
}
|
||||
try:
|
||||
resolved = pattern.format(**vars_with_defaults)
|
||||
# Clean up any double slashes or leading/trailing slashes for relative path
|
||||
return os.path.normpath(resolved).strip("/")
|
||||
except KeyError as e:
|
||||
logger.warning(f"[StoragePlugin][Coherence:Failed] Missing variable for path resolution: {e}")
|
||||
# Fallback to literal pattern if formatting fails partially (or handle as needed)
|
||||
return pattern.replace("{", "").replace("}", "")
|
||||
# [/DEF:resolve_path:Function]
|
||||
|
||||
# [DEF:ensure_directories:Function]
|
||||
# @PURPOSE: Creates the storage root and category subdirectories if they don't exist.
|
||||
# @SIDE_EFFECT: Creates directories on the filesystem.
|
||||
@@ -113,7 +142,8 @@ class StoragePlugin(PluginBase):
|
||||
with belief_scope("StoragePlugin:ensure_directories"):
|
||||
root = self.get_storage_root()
|
||||
for category in FileCategory:
|
||||
path = root / f"{category.value}s"
|
||||
# Use singular name for consistency with BackupPlugin and GitService
|
||||
path = root / category.value
|
||||
path.mkdir(parents=True, exist_ok=True)
|
||||
logger.debug(f"[StoragePlugin][Action] Ensured directory: {path}")
|
||||
# [/DEF:ensure_directories:Function]
|
||||
@@ -135,46 +165,68 @@ class StoragePlugin(PluginBase):
|
||||
# [/DEF:validate_path:Function]
|
||||
|
||||
# [DEF:list_files:Function]
|
||||
# @PURPOSE: Lists all files in a specific category.
|
||||
# @PURPOSE: Lists all files and directories in a specific category and subpath.
|
||||
# @PARAM: category (Optional[FileCategory]) - The category to list.
|
||||
# @RETURN: List[StoredFile] - List of file metadata objects.
|
||||
def list_files(self, category: Optional[FileCategory] = None) -> List[StoredFile]:
|
||||
# @PARAM: subpath (Optional[str]) - Nested path within the category.
|
||||
# @RETURN: List[StoredFile] - List of file and directory metadata objects.
|
||||
def list_files(self, category: Optional[FileCategory] = None, subpath: Optional[str] = None) -> List[StoredFile]:
|
||||
with belief_scope("StoragePlugin:list_files"):
|
||||
root = self.get_storage_root()
|
||||
logger.info(f"[StoragePlugin][Action] Listing files in root: {root}, category: {category}, subpath: {subpath}")
|
||||
files = []
|
||||
|
||||
categories = [category] if category else list(FileCategory)
|
||||
|
||||
for cat in categories:
|
||||
cat_dir = root / f"{cat.value}s"
|
||||
if not cat_dir.exists():
|
||||
# Scan the category subfolder + optional subpath
|
||||
base_dir = root / cat.value
|
||||
if subpath:
|
||||
target_dir = self.validate_path(base_dir / subpath)
|
||||
else:
|
||||
target_dir = base_dir
|
||||
|
||||
if not target_dir.exists():
|
||||
continue
|
||||
|
||||
for item in cat_dir.iterdir():
|
||||
if item.is_file():
|
||||
stat = item.stat()
|
||||
logger.debug(f"[StoragePlugin][Action] Scanning directory: {target_dir}")
|
||||
|
||||
# Use os.scandir for better performance and to distinguish files vs dirs
|
||||
with os.scandir(target_dir) as it:
|
||||
for entry in it:
|
||||
# Skip logs
|
||||
if "Logs" in entry.path:
|
||||
continue
|
||||
|
||||
stat = entry.stat()
|
||||
is_dir = entry.is_dir()
|
||||
|
||||
files.append(StoredFile(
|
||||
name=item.name,
|
||||
path=str(item.relative_to(root)),
|
||||
size=stat.st_size,
|
||||
name=entry.name,
|
||||
path=str(Path(entry.path).relative_to(root)),
|
||||
size=stat.st_size if not is_dir else 0,
|
||||
created_at=datetime.fromtimestamp(stat.st_ctime),
|
||||
category=cat,
|
||||
mime_type=None # Could use python-magic here if needed
|
||||
mime_type="directory" if is_dir else None
|
||||
))
|
||||
|
||||
return sorted(files, key=lambda x: x.created_at, reverse=True)
|
||||
# Sort: directories first, then by name
|
||||
return sorted(files, key=lambda x: (x.mime_type != "directory", x.name))
|
||||
# [/DEF:list_files:Function]
|
||||
|
||||
# [DEF:save_file:Function]
|
||||
# @PURPOSE: Saves an uploaded file to the specified category.
|
||||
# @PURPOSE: Saves an uploaded file to the specified category and optional subpath.
|
||||
# @PARAM: file (UploadFile) - The uploaded file.
|
||||
# @PARAM: category (FileCategory) - The target category.
|
||||
# @PARAM: subpath (Optional[str]) - The target subpath.
|
||||
# @RETURN: StoredFile - Metadata of the saved file.
|
||||
# @SIDE_EFFECT: Writes file to disk.
|
||||
async def save_file(self, file: UploadFile, category: FileCategory) -> StoredFile:
|
||||
async def save_file(self, file: UploadFile, category: FileCategory, subpath: Optional[str] = None) -> StoredFile:
|
||||
with belief_scope("StoragePlugin:save_file"):
|
||||
root = self.get_storage_root()
|
||||
dest_dir = root / f"{category.value}s"
|
||||
dest_dir = root / category.value
|
||||
if subpath:
|
||||
dest_dir = dest_dir / subpath
|
||||
|
||||
dest_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
dest_path = self.validate_path(dest_dir / file.filename)
|
||||
@@ -194,34 +246,44 @@ class StoragePlugin(PluginBase):
|
||||
# [/DEF:save_file:Function]
|
||||
|
||||
# [DEF:delete_file:Function]
|
||||
# @PURPOSE: Deletes a file from the specified category.
|
||||
# @PURPOSE: Deletes a file or directory from the specified category and path.
|
||||
# @PARAM: category (FileCategory) - The category.
|
||||
# @PARAM: filename (str) - The name of the file.
|
||||
# @SIDE_EFFECT: Removes file from disk.
|
||||
def delete_file(self, category: FileCategory, filename: str):
|
||||
# @PARAM: path (str) - The relative path of the file or directory.
|
||||
# @SIDE_EFFECT: Removes item from disk.
|
||||
def delete_file(self, category: FileCategory, path: str):
|
||||
with belief_scope("StoragePlugin:delete_file"):
|
||||
root = self.get_storage_root()
|
||||
file_path = self.validate_path(root / f"{category.value}s" / filename)
|
||||
# path is relative to root, but we ensure it starts with category
|
||||
full_path = self.validate_path(root / path)
|
||||
|
||||
if file_path.exists():
|
||||
file_path.unlink()
|
||||
logger.info(f"[StoragePlugin][Action] Deleted file: {file_path}")
|
||||
if not str(Path(path)).startswith(category.value):
|
||||
raise ValueError(f"Path {path} does not belong to category {category}")
|
||||
|
||||
if full_path.exists():
|
||||
if full_path.is_dir():
|
||||
shutil.rmtree(full_path)
|
||||
else:
|
||||
full_path.unlink()
|
||||
logger.info(f"[StoragePlugin][Action] Deleted: {full_path}")
|
||||
else:
|
||||
raise FileNotFoundError(f"File {filename} not found in {category.value}s")
|
||||
raise FileNotFoundError(f"Item {path} not found")
|
||||
# [/DEF:delete_file:Function]
|
||||
|
||||
# [DEF:get_file_path:Function]
|
||||
# @PURPOSE: Returns the absolute path of a file for download.
|
||||
# @PARAM: category (FileCategory) - The category.
|
||||
# @PARAM: filename (str) - The name of the file.
|
||||
# @PARAM: path (str) - The relative path of the file.
|
||||
# @RETURN: Path - Absolute path to the file.
|
||||
def get_file_path(self, category: FileCategory, filename: str) -> Path:
|
||||
def get_file_path(self, category: FileCategory, path: str) -> Path:
|
||||
with belief_scope("StoragePlugin:get_file_path"):
|
||||
root = self.get_storage_root()
|
||||
file_path = self.validate_path(root / f"{category.value}s" / filename)
|
||||
file_path = self.validate_path(root / path)
|
||||
|
||||
if not file_path.exists():
|
||||
raise FileNotFoundError(f"File {filename} not found in {category.value}s")
|
||||
if not str(Path(path)).startswith(category.value):
|
||||
raise ValueError(f"Path {path} does not belong to category {category}")
|
||||
|
||||
if not file_path.exists() or file_path.is_dir():
|
||||
raise FileNotFoundError(f"File {path} not found")
|
||||
|
||||
return file_path
|
||||
# [/DEF:get_file_path:Function]
|
||||
|
||||
@@ -31,9 +31,15 @@ class GitService:
|
||||
# @PARAM: base_path (str) - Root directory for all Git clones.
|
||||
# @PRE: base_path is a valid string path.
|
||||
# @POST: GitService is initialized; base_path directory exists.
|
||||
def __init__(self, base_path: str = "backend/git_repos"):
|
||||
def __init__(self, base_path: str = "git_repos"):
|
||||
with belief_scope("GitService.__init__"):
|
||||
self.base_path = base_path
|
||||
# Resolve relative to the backend directory
|
||||
# Path(__file__) is backend/src/services/git_service.py
|
||||
# parents[2] is backend/
|
||||
from pathlib import Path
|
||||
backend_root = Path(__file__).parents[2]
|
||||
|
||||
self.base_path = str((backend_root / base_path).resolve())
|
||||
if not os.path.exists(self.base_path):
|
||||
os.makedirs(self.base_path)
|
||||
# [/DEF:__init__:Function]
|
||||
|
||||
BIN
backend/tasks.db
BIN
backend/tasks.db
Binary file not shown.
Reference in New Issue
Block a user