busya/ss-tools
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

task panel

Browse Source
This commit is contained in:
busya
2026-02-19 09:43:01 +03:00
parent a3a9f0788d
commit d29bc511a2
15 changed files with 57271 additions and 59119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
backend/src/core/superset_client.py
View File

@@ -87,11 +87,11 @@ class SupersetClient:
if 'columns' not in validated_query:
validated_query['columns'] = ["slug", "id", "changed_on_utc", "dashboard_title", "published"]
total_count = self._fetch_total_object_count(endpoint="/dashboard/")
paginated_data = self._fetch_all_pages(
endpoint="/dashboard/",
pagination_options={"base_query": validated_query, "total_count": total_count, "results_field": "result"},
pagination_options={"base_query": validated_query, "results_field": "result"},
)
total_count = len(paginated_data)
app_logger.info("[get_dashboards][Exit] Found %d dashboards.", total_count)
return total_count, paginated_data
# [/DEF:get_dashboards:Function]
@@ -203,11 +203,11 @@ class SupersetClient:
app_logger.info("[get_datasets][Enter] Fetching datasets.")
validated_query = self._validate_query_params(query)
total_count = self._fetch_total_object_count(endpoint="/dataset/")
paginated_data = self._fetch_all_pages(
endpoint="/dataset/",
pagination_options={"base_query": validated_query, "total_count": total_count, "results_field": "result"},
pagination_options={"base_query": validated_query, "results_field": "result"},
)
total_count = len(paginated_data)
app_logger.info("[get_datasets][Exit] Found %d datasets.", total_count)
return total_count, paginated_data
# [/DEF:get_datasets:Function]
@@ -370,11 +370,12 @@ class SupersetClient:
validated_query = self._validate_query_params(query or {})
if 'columns' not in validated_query:
validated_query['columns'] = []
total_count = self._fetch_total_object_count(endpoint="/database/")
paginated_data = self._fetch_all_pages(
endpoint="/database/",
pagination_options={"base_query": validated_query, "total_count": total_count, "results_field": "result"},
pagination_options={"base_query": validated_query, "results_field": "result"},
)
total_count = len(paginated_data)
app_logger.info("[get_databases][Exit] Found %d databases.", total_count)
return total_count, paginated_data
# [/DEF:get_databases:Function]

30
backend/src/core/utils/network.py
View File

@@ -355,20 +355,40 @@ class APIClient:
# @PURPOSE: Автоматически собирает данные со всех страниц пагинированного эндпоинта.
# @PARAM: endpoint (str) - Эндпоинт.
# @PARAM: pagination_options (Dict[str, Any]) - Опции пагинации.
# @PRE: pagination_options must contain 'base_query', 'total_count', 'results_field'.
# @PRE: pagination_options must contain 'base_query', 'results_field'. 'total_count' is optional.
# @POST: Returns all items across all pages.
# @RETURN: List[Any] - Список данных.
def fetch_paginated_data(self, endpoint: str, pagination_options: Dict[str, Any]) -> List[Any]:
with belief_scope("fetch_paginated_data"):
base_query, total_count = pagination_options["base_query"], pagination_options["total_count"]
results_field, page_size = pagination_options["results_field"], base_query.get('page_size')
assert page_size and page_size > 0, "'page_size' must be a positive number."
base_query = pagination_options["base_query"]
total_count = pagination_options.get("total_count")
results_field = pagination_options["results_field"]
count_field = pagination_options.get("count_field", "count")
page_size = base_query.get('page_size', 1000)
assert page_size > 0, "'page_size' must be a positive number."
results = []
for page in range((total_count + page_size - 1) // page_size):
page = 0
# Fetch first page to get data and total count if not provided
query = {**base_query, 'page': page}
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
first_page_results = response_json.get(results_field, [])
results.extend(first_page_results)
if total_count is None:
total_count = response_json.get(count_field, len(first_page_results))
app_logger.debug(f"[fetch_paginated_data][State] Total count resolved from first page: {total_count}")
# Fetch remaining pages
total_pages = (total_count + page_size - 1) // page_size
for page in range(1, total_pages):
query = {**base_query, 'page': page}
response_json = cast(Dict[str, Any], self.request("GET", endpoint, params={"q": json.dumps(query)}))
results.extend(response_json.get(results_field, []))
return results
# [/DEF:fetch_paginated_data:Function]