From 253e44dba03a3081dc6545ec2ff1ffb0150c48d3 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Wed, 10 Jun 2026 17:40:03 +0800 Subject: [PATCH 01/17] feat: add last query --- swanlab/api/experiment.py | 154 ++++++++++++++++++++++++---------- swanlab/api/metric.py | 21 ++++- swanlab/api/typings/common.py | 18 ++-- 3 files changed, 142 insertions(+), 51 deletions(-) diff --git a/swanlab/api/experiment.py b/swanlab/api/experiment.py index 44a6e4cef..892afa92f 100644 --- a/swanlab/api/experiment.py +++ b/swanlab/api/experiment.py @@ -36,10 +36,12 @@ class Experiment(BaseEntity): """ - 表示一个 SwanLab 实验（完整信息，通过 POST /runs/shows 或单实验详情接口获取）。 + Represents a SwanLab experiment with lazy-loaded attributes and query / mutation methods. - 支持双模式：构造时传入 data，或 data=None（按需懒加载）。 - 构造时从 data 中提取 _cuid 缓存，避免 _ensure_data 与 id 属性的循环调用。 + Construct with ``data`` to eagerly cache experiment metadata, or ``data=None`` to lazily + load on first property access. Obtain instances via ``swanlab.Api``:: + + exp = api.run("username/project/run_id") """ def __init__( @@ -170,11 +172,11 @@ def column( column_type: Optional[ApiColumnDataTypeLiteral] = "FLOAT", ): """ - 获取实验下指定 key 的单个列。 + Get a single column by key under this experiment. - :param key: 列的 key，如 "loss"、"acc" - :param column_class: 列的分类，CUSTOM 或 SYSTEM - :param column_type: 列的数据类型，如 FLOAT、STRING、IMAGE 等 + :param key: Column key, e.g. ``"loss"``, ``"acc"`` + :param column_class: Column class, ``CUSTOM`` (default) or ``SYSTEM`` + :param column_type: Column data type, e.g. ``FLOAT``, ``STRING``, ``IMAGE`` """ from swanlab.api.column import Column @@ -200,17 +202,80 @@ def metrics( range_query: Optional[Union[Dict[str, Any], RangeQuery]] = None, ) -> Dict[str, Any]: """ - Fetch scalar metrics for the given keys. - - :param keys: Metric keys to fetch, e.g. ["loss", "acc"]. - :param sample: Max number of sampled data points (default 1500). Ignored when ``all`` or ``range_query`` is set. - :param ignore_timestamp: If True, omit timestamp fields from the response. - :param all: If True, fetch full-resolution data without sampling limit. - :param range_query: Precise step(default)-range filter — accepts a ``RangeQuery`` object or a plain dict - with keys ``start``, ``end``, ``head``, ``tail``. Only supported for SCALAR metrics. - Example: ``{"type": "step", "start": 100, "end": 500}`` or ``{"tail": 50}``. - For timestamp-based filtering: ``{"type": "timestamp", "start": 1715769600000, "end": 1715773200000}``. - ``start``/``end`` accept int or str timestamps; values shorter than millisecond precision are auto-padded. + Fetch scalar metrics (e.g. loss, acc) with three query modes: + + 1. **Sampled** (default) — server-side LTTB downsampling, up to ``sample`` data points + 2. **Full** — ``all=True``, no sampling limit + 3. **Range** — filter by step / timestamp / recent time window via ``range_query`` + + .. note:: + Modes 2 and 3 (``all`` / ``range_query``) download full-resolution CSV data and + perform range filtering client-side. Each metric point contains ``step``, ``value``, + and ``timestamp`` (if available). + + :param keys: Metric keys to fetch, e.g. ``["loss", "acc"]`` + :param sample: Max sampled data points (default 1500, max 1500). Ignored when ``all`` or ``range_query`` is set. + :param ignore_timestamp: If True, omit timestamp fields from the response + :param all: If True, fetch full-resolution data without sampling limit + :param range_query: Range filter — accepts a ``RangeQuery`` object or a plain dict. + Only supported for SCALAR metrics. + + --- + + **range_query fields** + + ========== ============ ================================================= + Field Type Description + ========== ============ ================================================= + ``type`` ``str`` Filter axis: ``"step"`` (default) or ``"timestamp"`` + ``start`` ``int`` Lower bound (inclusive); None = from beginning + ``end`` ``int`` Upper bound (inclusive); None = to end + ``last`` ``int`` Last N milliseconds (mutually exclusive with start/end) + ``head`` ``int`` First N data points (mutually exclusive with tail) + ``tail`` ``int`` Last N data points (mutually exclusive with head) + ========== ============ ================================================= + + **Mutual exclusivity** + + - ``head`` and ``tail`` are mutually exclusive + - ``last`` is mutually exclusive with ``start`` / ``end`` + - ``head`` / ``tail`` can be combined with ``last`` or ``start`` / ``end`` + + --- + + **Examples — progressive** + + 1. Default sampled query:: + + exp.metrics(keys=["loss", "acc"]) + + 2. Filter by step range:: + + exp.metrics(keys=["loss"], range_query={"start": 100, "end": 500}) + + 3. Step range + first 50 points:: + + exp.metrics(keys=["loss"], range_query={"start": 0, "end": 500, "head": 50}) + + 4. Filter by timestamp (Unix ms, auto-padded if < 13 digits):: + + exp.metrics(keys=["loss"], range_query={ + "type": "timestamp", + "start": 1715769600000, + "end": 1715773200000, + }) + + 5. Last 5 minutes:: + + exp.metrics(keys=["loss"], range_query={"last": 300_000}) + + 6. Last 5 minutes + first 20 points:: + + exp.metrics(keys=["loss"], range_query={"last": 300_000, "head": 20}) + + 7. Last 30 data points:: + + exp.metrics(keys=["loss"], range_query={"tail": 30}) """ run_id = self.run_id project_id = self.project_id @@ -244,9 +309,14 @@ def summary( keys: Optional[List[str]] = None, ) -> Dict[str, Any]: """ - 获取实验的标量指标概要统计数据。 + Get summary statistics for scalar metrics (latest, min, max, avg, median, etc.). + + :param keys: Scalar keys to query; None means all keys + + :: - :param keys: 需要查询的标量 key 列表，为 None 表示查询全量 keys + exp.summary() # all keys + exp.summary(keys=["loss"]) # specific keys """ run_id = self.run_id project_id = self.project_id @@ -313,10 +383,10 @@ def export_logs( rows: int = 500_000, ) -> ApiResponseType: """ - 导出实验日志为 .log 文件。 + Export experiment logs as a .log file. - :param start: 导出起始行号，0-based，默认 0 - :param rows: 导出行数，默认 500000，最大 500000 + :param start: Export start row (0-based), default 0 + :param rows: Number of rows to export, default 500_000, max 500_000 """ run_id = self.run_id project_id = self.project_id @@ -343,14 +413,14 @@ def columns( all: bool = False, ): """ - 获取实验下的列列表（分页查询，支持搜索）。 - - :param page: 起始页码，默认 1 - :param size: 每页数量，默认 20 - :param search: 搜索关键词，搜索的是列的 name - :param column_type: 列的类型，如 FLOAT、STRING、IMAGE 等 - :param column_class: 列的分类，CUSTOM 或 SYSTEM - :param all: 是否获取全部数据，默认 False + List columns under this experiment (paginated, with optional search). + + :param page: Page number, default 1 + :param size: Page size, default 20 + :param search: Search keyword (matches column name) + :param column_type: Column type filter, e.g. FLOAT, STRING, IMAGE + :param column_class: Column class filter, CUSTOM or SYSTEM + :param all: If True, fetch all pages, default False """ self._ensure_data() from swanlab.api.column import Columns @@ -370,7 +440,7 @@ def columns( ) def delete(self, commit: bool = False) -> bool: - """删除此实验。commit=False 时打印待删除信息，commit=True 时执行删除。""" + """Delete this experiment. ``commit=False`` prints the pending deletion; ``commit=True`` executes it.""" if not commit: name = self.name if self._errors: @@ -385,7 +455,7 @@ def json(self) -> Dict[str, Any]: def _flatten_runs(runs: Union[list, Dict]) -> list: - """展开分组后的实验数据，返回一个包含所有实验的列表。""" + """Flatten grouped experiment data into a flat list of experiments.""" if isinstance(runs, dict): return [item for v in runs.values() for item in _flatten_runs(v)] if isinstance(runs, list): @@ -395,19 +465,19 @@ def _flatten_runs(runs: Union[list, Dict]) -> list: class Experiments(BaseEntity): """ - 项目下实验集合的迭代器。 + Iterator over experiments in a project. - 支持两种模式： - - POST 模式（默认）：通过 /runs/shows 接口获取，支持复杂过滤，不支持分页 - - GET 模式：通过 /runs 接口获取，支持标准分页，返回精简信息 + Supports two modes: + - POST mode (default): fetches via ``/runs/shows``, supports complex filtering, no pagination + - GET mode: fetches via ``/runs``, supports standard pagination, returns summary info - 用法:: + Usage:: - # POST 复杂过滤 + # POST with complex filters for run in api.runs(path="username/project"): print(run.name) - # GET 分页 + # GET with pagination for run in api.list_runs_simple(path="username/project"): print(run.name, run.state) """ @@ -445,7 +515,7 @@ def __iter__(self) -> Iterator[Experiment]: yield from self._iter_filtered() def _iter_filtered(self) -> Iterator[Experiment]: - """POST /runs/shows 模式：复杂过滤，不支持分页。""" + """POST /runs/shows mode: complex filtering, no pagination.""" resp = self._post( f"/project/{self._proj_path}/runs/shows", data={ @@ -472,7 +542,7 @@ def _iter_filtered(self) -> Iterator[Experiment]: yield Experiment(self._ctx, path=full_path, data=run_data) def _iter_paginated(self) -> Iterator[Experiment]: - """GET /runs 模式：标准分页，返回精简信息。""" + """GET /runs mode: standard pagination, returns summary info.""" for item in self._paginate( f"/project/{self._proj_path}/runs", self._query, diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index 6696b8155..8fad74881 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -43,8 +43,9 @@ def _stream_csv_rows( rq: Optional[RangeQuery] = None, timeout: int = 30, ) -> Optional[List[Dict[str, Any]]]: - """流式下载 CSV 并逐行解析，复用 Client session（保留 proxy/retry 配置）。""" + """Stream-download CSV and parse rows, reusing the Client session (preserving proxy/retry config).""" import csv + import time from collections import deque resp = client._session.get(url, stream=True, timeout=timeout) @@ -57,6 +58,11 @@ def _stream_csv_rows( max_len = rq.tail if rq is not None and rq.tail is not None else None rows: Union[deque[Dict[str, Any]], List[Dict[str, Any]]] = deque(maxlen=max_len) if max_len is not None else [] + # Pre-compute timestamp lower bound for ``last`` mode + last_start_ts: Optional[int] = None + if rq is not None and rq.last is not None: + last_start_ts = int(time.time() * 1000) - rq.last + for row in csv.reader(lines): if not row or len(row) < 2: continue @@ -75,7 +81,16 @@ def _stream_csv_rows( pass if rq is not None: - if rq.type == "timestamp": + # --- ``last`` mode: filter by timestamp >= (now - last) --- + if last_start_ts is not None: + ts = item.get("timestamp") + if ts is None: + console.warning(f"CSV row missing timestamp column: {row}") + continue + if ts < last_start_ts: + continue + # --- timestamp range mode --- + elif rq.type == "timestamp": ts = item.get("timestamp") if ts is None: console.warning(f"CSV row missing timestamp column: {row}") @@ -84,11 +99,13 @@ def _stream_csv_rows( continue if rq.end is not None and ts > rq.end: break + # --- step range mode --- else: if rq.start is not None and step < rq.start: continue if rq.end is not None and step > rq.end: break + # --- head early-stop --- if rq.head is not None and len(rows) >= rq.head: break diff --git a/swanlab/api/typings/common.py b/swanlab/api/typings/common.py index af984f88d..e9b603195 100644 --- a/swanlab/api/typings/common.py +++ b/swanlab/api/typings/common.py @@ -145,20 +145,22 @@ class RangeQuery(BaseModel, frozen=True): """ - SCALAR 指标范围查询参数。 + Scalar metric range query parameters. - type: 维度，支持 "step" / "timestamp" - start: 起始边界（含），None 表示从头开始 - end: 结束边界（含），None 表示到最后 - head: 取前 N 条（与 tail 互斥） - tail: 取后 N 条（与 head 互斥） + type: Filter axis — ``"step"`` (default) or ``"timestamp"`` + start: Lower bound (inclusive); None = from beginning + end: Upper bound (inclusive); None = to end + last: Last N milliseconds (mutually exclusive with start/end; SDK auto-converts to timestamp filter) + head: First N data points (mutually exclusive with tail) + tail: Last N data points (mutually exclusive with head) - head/tail 与 start/end 可组合：先 start/end 过滤，再取 head/tail。 + head/tail can be combined with start/end/last: range filter first, then head/tail. """ type: Literal["step", "timestamp"] = "step" start: Optional[int] = Field(default=None, ge=0) end: Optional[int] = Field(default=None, ge=0) + last: Optional[int] = Field(default=None, gt=0) head: Optional[int] = Field(default=None, gt=0) tail: Optional[int] = Field(default=None, gt=0) @@ -168,6 +170,8 @@ def _validate_range_query(self) -> "RangeQuery": raise ValueError("head and tail are mutually exclusive") if self.start is not None and self.end is not None and self.start > self.end: raise ValueError(f"start must be <= end, got ({self.start}, {self.end})") + if self.last is not None and (self.start is not None or self.end is not None): + raise ValueError("last is mutually exclusive with start/end") return self From 4b101aaa602628825430f4a9dae3752bf64c403f Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Wed, 10 Jun 2026 19:37:35 +0800 Subject: [PATCH 02/17] feat: expand page_size --- swanlab/api/__init__.py | 45 +++++++++++++++++++-------------- swanlab/api/base.py | 4 --- swanlab/api/column.py | 47 +++++++++++++++++++++++++++-------- swanlab/api/experiment.py | 17 +++++++++---- swanlab/api/project.py | 4 +-- swanlab/api/selfhosted.py | 12 ++++----- swanlab/api/typings/common.py | 2 +- swanlab/api/workspace.py | 2 +- 8 files changed, 84 insertions(+), 49 deletions(-) diff --git a/swanlab/api/__init__.py b/swanlab/api/__init__.py index d071a4ad2..9306ceb6b 100644 --- a/swanlab/api/__init__.py +++ b/swanlab/api/__init__.py @@ -154,7 +154,7 @@ def projects( search: Optional[str] = None, detail: Optional[bool] = True, page: int = 1, - size: int = 20, + size: int = 100, all: bool = False, ) -> Projects: """ @@ -165,7 +165,7 @@ def projects( :param search: 搜索关键词 :param detail: 是否返回详细信息 :param page: 起始页码，默认 1 - :param size: 每页数量，默认 20 + :param size: 每页数量，默认 100 :param all: 是否获取全部数据，默认 False """ validate_api_path(path, segments=1, label="workspace") @@ -225,7 +225,7 @@ def runs_get( self, path: str, page: int = 1, - size: int = 20, + size: int = 100, all: bool = False, ) -> Experiments: """ @@ -233,7 +233,7 @@ def runs_get( :param path: 项目路径，格式为 'username/project' :param page: 起始页码，默认 1 - :param size: 每页数量，默认 20 + :param size: 每页数量，默认 100 :param all: 是否获取全部数据，默认 False """ validate_api_path(path, segments=2, label="project") @@ -247,22 +247,25 @@ def columns( self, path: str, page: int = 1, - size: int = 20, + size: int = 100, search: Optional[str] = None, column_class: ApiColumnClassLiteral = "CUSTOM", column_type: Optional[ApiColumnDataTypeLiteral] = None, all: bool = False, ) -> Columns: """ - 获取实验下的列列表（分页查询，支持搜索）。 - - :param path: 实验路径，格式为 'username/project/run_id' - :param page: 起始页码，默认 1 - :param size: 每页数量，默认 20 - :param search: 搜索关键词，搜索的是列的 name - :param column_class: 列的分类，CUSTOM 或 SYSTEM, 默认为 CUSTOM - :param column_type: 列的类型，如 FLOAT、STRING、IMAGE 等 - :param all: 是否获取全部数据，默认 False + List columns under an experiment (paginated, with optional fuzzy search). + + The ``search`` parameter performs **fuzzy matching** (case-insensitive ``contains``) + on the column ``name`` field. + + :param path: Experiment path, format: ``'username/project/run_id'`` + :param page: Page number, default 1 + :param size: Page size, default 100 + :param search: Fuzzy search keyword (matches column **name**, not key) + :param column_class: Column class, ``CUSTOM`` or ``SYSTEM``, default ``CUSTOM`` + :param column_type: Column data type, e.g. ``FLOAT``, ``STRING``, ``IMAGE`` + :param all: If True, fetch all pages, default False """ validate_api_path(path, segments=3, label="run") query = PaginatedQuery(page=page, size=size, search=search, all=all) @@ -282,12 +285,16 @@ def column( column_type: Optional[ApiColumnDataTypeLiteral] = None, ) -> Column: """ - 获取单个列（通过搜索 key 匹配）。 + Get a single column by key (fuzzy search, first match). - :param path: 实验路径，格式为 'username/project/run_id' - :param key: 列的键名, 输入不完整则模糊匹配 name 为首个 key. - :param column_class: 列的分类，CUSTOM 或 SYSTEM，默认 CUSTOM - :param column_type: 列的类型，如 FLOAT、STRING、IMAGE 等，默认为 None + Performs fuzzy search (``contains`` on ``name``) and returns the first matching + column. If multiple columns share a similar name, the first one (ordered by + ``id DESC``) is returned. + + :param path: Experiment path, format: ``'username/project/run_id'`` + :param key: Column key to search, e.g. ``"loss"``, ``"acc"`` + :param column_class: Column class, ``CUSTOM`` or ``SYSTEM``, default ``CUSTOM`` + :param column_type: Column data type, e.g. ``FLOAT``, ``STRING``, ``IMAGE`` """ validate_api_path(path, segments=3, label="run") validate_non_empty_string(key, label="column key") diff --git a/swanlab/api/base.py b/swanlab/api/base.py index 06a0d5c05..373ba526c 100644 --- a/swanlab/api/base.py +++ b/swanlab/api/base.py @@ -5,8 +5,6 @@ @description: 所有实体类的公共基类 """ -import random -import time from abc import ABC, abstractmethod from dataclasses import dataclass from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Sequence, Tuple @@ -141,8 +139,6 @@ def _paginate( if not items: break yield from items - # 随机休眠控制 qps - time.sleep(random.random()) if page >= body.get("pages", 1): break if not query.all: diff --git a/swanlab/api/column.py b/swanlab/api/column.py index daa575c0e..61a2423db 100644 --- a/swanlab/api/column.py +++ b/swanlab/api/column.py @@ -21,10 +21,14 @@ class Column(BaseEntity): """ - 表示一个 SwanLab 实验列。 + A single column in a SwanLab experiment. - 支持双模式：构造时传入 data（列表迭代注入），或 data=None（按需懒加载）。 - 注意：列不支持单个获取 API，只能通过列表接口获取。 + Supports two construction modes: pass ``data`` for eager initialization (e.g. from + list iteration), or ``data=None`` for lazy loading on first property access. + + ``_ensure_data()`` performs fuzzy search (``contains`` on ``name``) and returns the + first matching column. This is a soft lookup — if multiple columns share a similar + name, the first one (ordered by ``id DESC``) is returned. """ @staticmethod @@ -203,19 +207,22 @@ def json(self) -> Dict[str, Any]: class Columns(BaseEntity): """ - 实验下列集合的分页迭代器。 + Paginated iterator over columns in an experiment. + + The ``search`` parameter performs **fuzzy matching** (case-insensitive ``contains``) + on the column ``name`` field via the backend ``GET /experiment/:id/column`` endpoint. - 用法:: + Usage:: - # 获取所有列 + # All columns for column in experiment.columns(): - print(column.name, column.data_type) + print(column.name, column.column_type) - # 分页获取列（支持搜索） - for column in experiment.columns(page=1, size=20, search="loss"): + # Fuzzy search by name + for column in experiment.columns(search="loss"): print(column.name) - # 获取全部列（自动翻页） + # Fetch all (auto-paginate) for column in experiment.columns(all=True): print(column.name) """ @@ -271,7 +278,13 @@ def _ensure_project_id(self) -> str: return self._project_id def __iter__(self) -> Iterator[Column]: - """迭代分页获取列。""" + """ + Iterate columns with automatic pagination. + + When ``all=True``, the backend skips ``search`` / ``type`` / ``class`` filters, + so we apply **client-side filtering** to ensure ``search + all`` returns only + matching columns. + """ extra: Dict[str, Any] = {} run_id = self._ensure_run_id() if self._column_type: @@ -279,12 +292,24 @@ def __iter__(self) -> Iterator[Column]: if self._column_class: extra["class"] = self._column_class + # In all=True mode the backend skips search/type/class filters; + # apply client-side filtering instead so search + all works correctly. + _client_filter = self._query.all + _search_lower = (self._query.search or "").lower() if self._query.search else None + for item in self._paginate( f"/experiment/{run_id}/column", self._query, page_info=self._page_info, extra=extra, ): + if _client_filter: + if _search_lower and _search_lower not in (item.get("name") or "").lower(): + continue + if self._column_type and item.get("type") != self._column_type: + continue + if self._column_class and item.get("class") != self._column_class: + continue data = {**item, "run_id": run_id} yield Column( self._ctx, diff --git a/swanlab/api/experiment.py b/swanlab/api/experiment.py index 892afa92f..e1b87bc24 100644 --- a/swanlab/api/experiment.py +++ b/swanlab/api/experiment.py @@ -172,7 +172,11 @@ def column( column_type: Optional[ApiColumnDataTypeLiteral] = "FLOAT", ): """ - Get a single column by key under this experiment. + Get a single column by key under this experiment (fuzzy search, first match). + + Performs fuzzy search (``contains`` on ``name``) and returns the first matching + column. If multiple columns share a similar name, the first one (ordered by + ``id DESC``) is returned. :param key: Column key, e.g. ``"loss"``, ``"acc"`` :param column_class: Column class, ``CUSTOM`` (default) or ``SYSTEM`` @@ -406,18 +410,21 @@ def export_logs( def columns( self, page: int = 1, - size: int = 20, + size: int = 100, search: Optional[str] = None, column_type: Optional[ApiColumnDataTypeLiteral] = None, column_class: Optional[ApiColumnClassLiteral] = None, all: bool = False, ): """ - List columns under this experiment (paginated, with optional search). + List columns under this experiment (paginated, with optional fuzzy search). + + The ``search`` parameter performs **fuzzy matching** (case-insensitive ``contains``) + on the column ``name`` field. :param page: Page number, default 1 - :param size: Page size, default 20 - :param search: Search keyword (matches column name) + :param size: Page size, default 100 + :param search: Fuzzy search keyword (matches column **name**, not key) :param column_type: Column type filter, e.g. FLOAT, STRING, IMAGE :param column_class: Column class filter, CUSTOM or SYSTEM :param all: If True, fetch all pages, default False diff --git a/swanlab/api/project.py b/swanlab/api/project.py index 7d1379f9b..745003923 100644 --- a/swanlab/api/project.py +++ b/swanlab/api/project.py @@ -98,14 +98,14 @@ def runs( def runs_get( self, page: int = 1, - size: int = 20, + size: int = 100, all: bool = False, ): """ 获取项目下的实验列表（GET 模式，标准分页，返回精简信息）。 :param page: 起始页码，默认 1 - :param size: 每页数量，默认 20 + :param size: 每页数量，默认 100 :param all: 是否获取全部数据，默认 False """ from swanlab.api.experiment import Experiments diff --git a/swanlab/api/selfhosted.py b/swanlab/api/selfhosted.py index 76fa8070e..f662fc9eb 100644 --- a/swanlab/api/selfhosted.py +++ b/swanlab/api/selfhosted.py @@ -87,12 +87,12 @@ def create_user(self, username: str, password: str) -> ApiResponseType: data = {"users": [{"username": username, "password": password}]} return self._post("/self_hosted/users", data=data) - def get_users(self, page: int = 1, size: int = 20, all: bool = False) -> Iterator[dict]: + def get_users(self, page: int = 1, size: int = 100, all: bool = False) -> Iterator[dict]: """ 分页获取用户（管理员限定）。 :param page: 起始页码，默认 1 - :param size: 每页大小，默认 20 + :param size: 每页大小，默认 100 :param all: 是否获取全部数据，默认 False """ SelfHosted.validate_root(self._ensure_data()) @@ -103,7 +103,7 @@ def get_users(self, page: int = 1, size: int = 20, all: bool = False) -> Iterato def get_projects( self, page: int = 1, - size: int = 20, + size: int = 100, search: Optional[str] = None, sort: Optional[str] = None, state: Optional[str] = None, @@ -115,7 +115,7 @@ def get_projects( 分页获取所有项目（管理员限定）。 :param page: 起始页码，默认 1 - :param size: 每页大小，默认 20 + :param size: 每页大小，默认 100 :param search: 搜索关键词 :param sort: 排序字段，update(默认) / create / name :param state: 实验状态过滤，RUNNING / FINISHED @@ -136,7 +136,7 @@ def get_projects( def get_groups( self, page: int = 1, - size: int = 20, + size: int = 100, search: Optional[str] = None, type: Optional[str] = None, sort: Optional[str] = None, @@ -146,7 +146,7 @@ def get_groups( 分页获取所有空间（管理员限定）。 :param page: 起始页码，默认 1 - :param size: 每页大小，默认 20 + :param size: 每页大小，默认 100 :param search: 搜索关键词 :param type: 空间类型过滤，PERSON / TEAM :param sort: 排序字段，update(默认) / create / name diff --git a/swanlab/api/typings/common.py b/swanlab/api/typings/common.py index e9b603195..1494e5cca 100644 --- a/swanlab/api/typings/common.py +++ b/swanlab/api/typings/common.py @@ -188,7 +188,7 @@ class PaginatedQuery: """ page: int = 1 - size: int = 20 + size: int = 100 search: Optional[str] = None sort: Optional[str] = None all: bool = False diff --git a/swanlab/api/workspace.py b/swanlab/api/workspace.py index 0c5dbddc0..0ef891d54 100644 --- a/swanlab/api/workspace.py +++ b/swanlab/api/workspace.py @@ -69,7 +69,7 @@ def projects( search: Optional[str] = None, detail: Optional[bool] = True, page: int = 1, - size: int = 20, + size: int = 100, all: bool = False, ): from swanlab.api.project import Projects From b1d077565c2d0440263ebac70a19caf00a043cc0 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Wed, 10 Jun 2026 19:39:21 +0800 Subject: [PATCH 03/17] fix: utils validation error --- tests/unit/api/test_utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/api/test_utils.py b/tests/unit/api/test_utils.py index 98a8f5f5a..7ca32e256 100644 --- a/tests/unit/api/test_utils.py +++ b/tests/unit/api/test_utils.py @@ -137,7 +137,7 @@ def test_invalid_order(self): class TestPaginatedQuery: def test_valid_defaults(self): q = PaginatedQuery() - assert q.page == 1 and q.size == 20 + assert q.page == 1 and q.size == 100 def test_page_less_than_1(self): with pytest.raises(ValueError, match="page must be >= 1"): From 325474a54bcd2308da9d9a0e3a0a9a9e898c8ad3 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Wed, 10 Jun 2026 21:04:41 +0800 Subject: [PATCH 04/17] fix: column search --- swanlab/api/column.py | 42 +++++++++++++-------------------- swanlab/api/typings/__init__.py | 3 +-- swanlab/api/typings/column.py | 11 +-------- swanlab/api/typings/common.py | 2 +- 4 files changed, 20 insertions(+), 38 deletions(-) diff --git a/swanlab/api/column.py b/swanlab/api/column.py index 61a2423db..38f6a4a4a 100644 --- a/swanlab/api/column.py +++ b/swanlab/api/column.py @@ -5,6 +5,7 @@ @description: Column 实体类 — 实验列的查询与操作 """ +import dataclasses from typing import Any, Callable, Dict, Iterator, Optional, cast from swanlab.api.base import ApiClientContext, BaseEntity @@ -154,11 +155,6 @@ def created_at(self) -> int: """列的创建时间戳。""" return self._ensure_data().get("createdAt", 0) - @property - def error(self) -> Optional[Dict[str, Any]]: - """列的错误信息。""" - return self._ensure_data().get("error", {}) - def metric( self, sample: int = 1500, @@ -278,13 +274,7 @@ def _ensure_project_id(self) -> str: return self._project_id def __iter__(self) -> Iterator[Column]: - """ - Iterate columns with automatic pagination. - - When ``all=True``, the backend skips ``search`` / ``type`` / ``class`` filters, - so we apply **client-side filtering** to ensure ``search + all`` returns only - matching columns. - """ + """Iterate columns with automatic pagination.""" extra: Dict[str, Any] = {} run_id = self._ensure_run_id() if self._column_type: @@ -292,24 +282,20 @@ def __iter__(self) -> Iterator[Column]: if self._column_class: extra["class"] = self._column_class - # In all=True mode the backend skips search/type/class filters; - # apply client-side filtering instead so search + all works correctly. - _client_filter = self._query.all - _search_lower = (self._query.search or "").lower() if self._query.search else None + # all=True + 有过滤条件时：后端 all=True 模式无视 search/type/class 仅做全量返回， + # 所以改为 all=False 走分页过滤 + 自动翻页收集全部结果 + use_auto_paginate = self._query.all and (self._query.search or self._column_type or self._column_class) + query = dataclasses.replace(self._query, all=use_auto_paginate) if use_auto_paginate else self._query + if use_auto_paginate: + if self._query.search: + extra["search"] = self._query.search for item in self._paginate( f"/experiment/{run_id}/column", - self._query, + query, page_info=self._page_info, extra=extra, ): - if _client_filter: - if _search_lower and _search_lower not in (item.get("name") or "").lower(): - continue - if self._column_type and item.get("type") != self._column_type: - continue - if self._column_class and item.get("class") != self._column_class: - continue data = {**item, "run_id": run_id} yield Column( self._ctx, @@ -334,5 +320,11 @@ def total(self) -> int: return self._page_info["total"] def json(self) -> Dict[str, Any]: - self._page_info["list"] = [c.json() for c in self] + items = [c.json() for c in self] + self._page_info["list"] = items + # 自动翻页后 total/size/pages 应与实际 list 对齐 + if self._query.all: + self._page_info["total"] = len(items) + self._page_info["size"] = len(items) + self._page_info["pages"] = 1 return self._page_info diff --git a/swanlab/api/typings/__init__.py b/swanlab/api/typings/__init__.py index 5b60c1c9e..0bc885ad8 100644 --- a/swanlab/api/typings/__init__.py +++ b/swanlab/api/typings/__init__.py @@ -5,7 +5,7 @@ @description: SwanLab OpenAPI 类型提示, 以 Api 前缀区分 """ -from .column import ApiColumnCsvExportType, ApiColumnErrorType, ApiColumnType +from .column import ApiColumnCsvExportType, ApiColumnType from .common import ( ApiIdentityLiteral, ApiLicensePlanLiteral, @@ -65,7 +65,6 @@ "ApiApiKeyType", "ApiSelfHostedInfoType", # Column - "ApiColumnErrorType", "ApiColumnType", "ApiColumnCsvExportType", # Metric diff --git a/swanlab/api/typings/column.py b/swanlab/api/typings/column.py index f119a3696..1bf1abeda 100644 --- a/swanlab/api/typings/column.py +++ b/swanlab/api/typings/column.py @@ -5,18 +5,11 @@ @description: 公共查询 API 实验列类型定义 """ -from typing import Any, Dict, Optional, TypedDict +from typing import TypedDict from .common import ApiColumnClassLiteral, ApiColumnDataTypeLiteral -class ApiColumnErrorType(TypedDict, total=False): - """列错误信息""" - - message: str - code: str - - class ApiColumnType(TypedDict, total=False): """ 实验列数据类型 @@ -40,8 +33,6 @@ class ApiColumnType(TypedDict, total=False): name: str # 创建时间戳 createdAt: int - # 错误信息 - error: Optional[Dict[str, Any]] class ApiColumnCsvExportType(TypedDict): diff --git a/swanlab/api/typings/common.py b/swanlab/api/typings/common.py index 1494e5cca..546e2f5f1 100644 --- a/swanlab/api/typings/common.py +++ b/swanlab/api/typings/common.py @@ -139,7 +139,7 @@ # 后端允许的每页条数 VALID_PAGE_SIZES = (10, 12, 15, 20, 24, 27, 50, 100) -# asyncio 并发请求数 +# 并发请求数 MAX_CONCURRENT_COUNT: int = 4 From ecd04985558249ac1a66514ee26135cb86007a3d Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 10:51:15 +0800 Subject: [PATCH 05/17] feat: add cli range query params --- swanlab/cli/api/experiment.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/swanlab/cli/api/experiment.py b/swanlab/cli/api/experiment.py index 01a254a8d..f7faeed74 100644 --- a/swanlab/cli/api/experiment.py +++ b/swanlab/cli/api/experiment.py @@ -140,6 +140,12 @@ def filter_experiments(project_path: str, filter_query: str, save_name: str, api type=PAGE_SIZE_TYPE, help="Page size.", ) +@click.option( + "--search", + default=None, + type=str, + help="Fuzzy search keyword (matches column name).", +) @click.option( "--class", "column_class", @@ -167,6 +173,7 @@ def list_experiment_columns( path: str, page_num: int, page_size: str, + search: Optional[str], column_class: str, column_type: str, fetch_all: bool, @@ -183,6 +190,7 @@ def list_experiment_columns( path=path, page=page_num, size=int(page_size), + search=search, column_class=column_class.upper(), # type: ignore column_type=column_type.upper() if column_type else None, # type: ignore all=fetch_all, @@ -239,6 +247,13 @@ def list_experiment_columns( ) @click.option("--range-head", "range_head", default=None, type=click.IntRange(min=1), help="First N data points.") @click.option("--range-tail", "range_tail", default=None, type=click.IntRange(min=1), help="Last N data points.") +@click.option( + "--range-last", + "range_last", + default=None, + type=click.IntRange(min=1), + help="Last N milliseconds of data (mutually exclusive with --range-start/--range-end).", +) @click.option( "--save", "save_name", @@ -258,6 +273,7 @@ def get_experiment_metrics( range_end: Optional[int], range_head: Optional[int], range_tail: Optional[int], + range_last: Optional[int], save_name: str, api: Api, ): @@ -267,11 +283,13 @@ def get_experiment_metrics( """ if range_head is not None and range_tail is not None: raise click.BadParameter("--range-head and --range-tail are mutually exclusive.") + if range_last is not None and (range_start is not None or range_end is not None): + raise click.BadParameter("--range-last is mutually exclusive with --range-start/--range-end.") if range_start is not None and range_end is not None and range_start > range_end: raise click.BadParameter(f"--range-start must be <= --range-end, got ({range_start}, {range_end}).") range_query = None - has_range = any(v is not None for v in (range_type, range_start, range_end, range_head, range_tail)) + has_range = any(v is not None for v in (range_type, range_start, range_end, range_head, range_tail, range_last)) if has_range: range_query = { "type": (range_type or "step"), @@ -279,6 +297,7 @@ def get_experiment_metrics( "end": range_end, "head": range_head, "tail": range_tail, + "last": range_last, } key_list = parse_keys(keys) From 3f408e8b6c384610259926a31a7d6553e7a5fd1d Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 11:09:47 +0800 Subject: [PATCH 06/17] fix: cli path format --- swanlab/cli/api/experiment.py | 42 +++++++++++------------------------ 1 file changed, 13 insertions(+), 29 deletions(-) diff --git a/swanlab/cli/api/experiment.py b/swanlab/cli/api/experiment.py index f7faeed74..f92624fae 100644 --- a/swanlab/cli/api/experiment.py +++ b/swanlab/cli/api/experiment.py @@ -60,13 +60,7 @@ def get_experiment(path: str, save_name: str, api: Api): type=PAGE_SIZE_TYPE, help="Page size.", ) -@click.option( - "--project_path", - "-p", - required=True, - type=str, - help="Project path (e.g. username/project_name).", -) +@click.argument("project_path", required=True) @click.option("--all", "fetch_all", is_flag=True, default=False, help="Fetch all pages.") @click.option( "--save", @@ -81,22 +75,14 @@ def list_experiments(page_num: int, page_size: str, project_path: str, fetch_all PROJECT_PATH format: username/project_name """ - resp = ApiResponseType( - ok=True, data=api.runs_get(path=project_path, page=page_num, size=int(page_size), all=fetch_all) - ) + resp = api.runs_get(path=project_path, page=page_num, size=int(page_size), all=fetch_all).wrapper() payload = format_output(resp) if payload["ok"] and save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @run_cli.command("filter") -@click.option( - "--project_path", - "-p", - required=True, - type=str, - help="Project path (e.g. username/project_name).", -) +@click.argument("project_path", required=True) @click.option( "--filter_query", "-f", @@ -118,7 +104,7 @@ def filter_experiments(project_path: str, filter_query: str, save_name: str, api PROJECT_PATH format: username/project_name """ filters = validate_filter_query(filter_query) - resp = ApiResponseType(ok=True, data=api.runs(path=project_path, filters=filters)) + resp = api.runs(path=project_path, filters=filters).wrapper() payload = format_output(resp) if payload["ok"] and save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -184,18 +170,16 @@ def list_experiment_columns( PATH format: username/project_name/run_id """ - resp = ApiResponseType( - ok=True, - data=api.columns( - path=path, - page=page_num, - size=int(page_size), - search=search, - column_class=column_class.upper(), # type: ignore - column_type=column_type.upper() if column_type else None, # type: ignore - all=fetch_all, - ), + columns = api.columns( + path=path, + page=page_num, + size=int(page_size), + search=search, + column_class=column_class.upper(), # type: ignore + column_type=column_type.upper() if column_type else None, # type: ignore + all=fetch_all, ) + resp = columns.wrapper() payload = format_output(resp) if payload["ok"] and save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) From e502aa8f56fc248dbe91f9d1f26b3bfb073b4fc4 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 11:30:06 +0800 Subject: [PATCH 07/17] fix: support save for error resp --- swanlab/cli/api/experiment.py | 98 ++++++++++++++++++++++++----------- swanlab/cli/api/project.py | 6 +-- swanlab/cli/api/selfhosted.py | 42 +++++++++------ swanlab/cli/api/user.py | 2 +- swanlab/cli/api/workspace.py | 2 +- 5 files changed, 101 insertions(+), 49 deletions(-) diff --git a/swanlab/cli/api/experiment.py b/swanlab/cli/api/experiment.py index f92624fae..179b79ef8 100644 --- a/swanlab/cli/api/experiment.py +++ b/swanlab/cli/api/experiment.py @@ -4,7 +4,9 @@ import orjson from swanlab.api import Api -from swanlab.api.typings.common import ApiResponseType +from swanlab.api.metric import Metric, Metrics +from swanlab.api.summary import Summary +from swanlab.api.typings.common import RangeQuery from swanlab.cli.api.helper import ( COLUMN_CLASS_TYPE, COLUMN_DATA_TYPE, @@ -41,7 +43,7 @@ def get_experiment(path: str, save_name: str, api: Api): """ resp = api.run(path).wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -77,7 +79,7 @@ def list_experiments(page_num: int, page_size: str, project_path: str, fetch_all """ resp = api.runs_get(path=project_path, page=page_num, size=int(page_size), all=fetch_all).wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -106,7 +108,7 @@ def filter_experiments(project_path: str, filter_query: str, save_name: str, api filters = validate_filter_query(filter_query) resp = api.runs(path=project_path, filters=filters).wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -181,7 +183,7 @@ def list_experiment_columns( ) resp = columns.wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -272,29 +274,36 @@ def get_experiment_metrics( if range_start is not None and range_end is not None and range_start > range_end: raise click.BadParameter(f"--range-start must be <= --range-end, got ({range_start}, {range_end}).") - range_query = None + rq = None has_range = any(v is not None for v in (range_type, range_start, range_end, range_head, range_tail, range_last)) if has_range: - range_query = { - "type": (range_type or "step"), - "start": range_start, - "end": range_end, - "head": range_head, - "tail": range_tail, - "last": range_last, - } + rq = RangeQuery( + type=(range_type or "step"), # type: ignore[arg-type] + start=range_start, + end=range_end, + head=range_head, + tail=range_tail, + last=range_last, + ) key_list = parse_keys(keys) experiment = api.run(path) - data = experiment.metrics( + metrics = Metrics( + ctx=api._ctx, + project_id=experiment.project_id, + run_id=experiment.run_id, keys=key_list, sample=sample, + metric_type="SCALAR", ignore_timestamp=ignore_timestamp, all=fetch_all, - range_query=range_query, + range_query=rq, + root_pro_id=experiment.root_pro_id, + root_exp_id=experiment.root_exp_id, ) - payload = format_output(ApiResponseType(ok=True, data=data)) - if payload["ok"] and save_name is not None: + resp = metrics.wrapper() + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -321,9 +330,17 @@ def get_experiment_summary(path: str, keys: Optional[str], save_name: str, api: """ key_list = parse_keys(keys) if keys else None experiment = api.run(path) - data = experiment.summary(keys=key_list) - payload = format_output(ApiResponseType(ok=True, data=data)) - if payload["ok"] and save_name is not None: + summary = Summary( + ctx=api._ctx, + project_id=experiment.project_id, + experiment_id=experiment.run_id, + keys=key_list, + root_pro_id=experiment.root_pro_id, + root_exp_id=experiment.root_exp_id, + ) + resp = summary.wrapper() + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -377,7 +394,7 @@ def get_experiment_column( ) resp = col.wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -419,9 +436,20 @@ def get_experiment_medias( """ key_list = parse_keys(keys) experiment = api.run(path) - data = experiment.medias(keys=key_list, step=step, all=fetch_all) - payload = format_output(ApiResponseType(ok=True, data=data)) - if payload["ok"] and save_name is not None: + metrics = Metrics( + ctx=api._ctx, + project_id=experiment.project_id, + run_id=experiment.run_id, + keys=key_list, + metric_type="MEDIA", + media_step=step, + all=fetch_all, + root_pro_id=experiment.root_pro_id, + root_exp_id=experiment.root_exp_id, + ) + resp = metrics.wrapper() + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -469,9 +497,21 @@ def get_experiment_logs( PATH format: username/project_name/run_id """ experiment = api.run(path) - data = experiment.logs(offset=offset, level=level.upper(), ignore_timestamp=ignore_timestamp) # type: ignore - payload = format_output(ApiResponseType(ok=True, data=data)) - if payload["ok"] and save_name is not None: + metric = Metric( + ctx=api._ctx, + project_id=experiment.project_id, + run_id=experiment.run_id, + key="LOG", + log_offset=offset, + log_level=level.upper(), # type: ignore + metric_type="LOG", + ignore_timestamp=ignore_timestamp, + root_pro_id=experiment.root_pro_id, + root_exp_id=experiment.root_exp_id, + ) + resp = metric.wrapper() + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -506,5 +546,5 @@ def export_experiment_logs(path: str, start: int, rows: int, save_name: str, api experiment = api.run(path) resp = experiment.export_logs(start=start, rows=rows) payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) diff --git a/swanlab/cli/api/project.py b/swanlab/cli/api/project.py index 0299e94df..a4730e9c3 100644 --- a/swanlab/cli/api/project.py +++ b/swanlab/cli/api/project.py @@ -29,7 +29,7 @@ def get_project(path: str, save_name: str, api: Api): """ resp = api.project(path).wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -70,7 +70,7 @@ def list_projects(page_num: int, page_size: str, workspace: str, fetch_all: bool ok=True, data=api.projects(path=workspace, page=page_num, size=int(page_size), all=fetch_all) ) payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -119,5 +119,5 @@ def create_project(name: str, visibility: str, description: str, workspace: str, return resp = project.wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) diff --git a/swanlab/cli/api/selfhosted.py b/swanlab/cli/api/selfhosted.py index 8734910c0..6effec09f 100644 --- a/swanlab/cli/api/selfhosted.py +++ b/swanlab/cli/api/selfhosted.py @@ -29,7 +29,7 @@ def get_info(save_name: str, api: Api): """Show self-hosted instance info.""" resp = api.self_hosted().wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -51,7 +51,7 @@ def create_user(username: str, password: str, save_name: str, api: Api): except ValueError as e: resp = ApiResponseType(ok=False, errmsg=str(e)) payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -69,14 +69,18 @@ def create_user(username: str, password: str, save_name: str, api: Api): @with_custom_host def list_users(page_num: int, page_size: int, fetch_all: bool, save_name: str, api: Api): """List users in the self-hosted instance.""" + sh = api.self_hosted() try: - users = list(api.self_hosted().get_users(page=page_num, size=page_size, all=fetch_all)) + users = list(sh.get_users(page=page_num, size=page_size, all=fetch_all)) except ValueError as e: payload = format_output(ApiResponseType(ok=False, errmsg=str(e))) else: - resp = ApiResponseType(ok=True, data={"list": users}) - payload = format_output(resp) - if payload["ok"] and save_name is not None: + if sh._errors: + payload = format_output(ApiResponseType(ok=False, errmsg="; ".join(sh._errors))) + else: + resp = ApiResponseType(ok=True, data={"list": users}) + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -106,9 +110,10 @@ def list_projects( api: Api, ): """List all projects in the self-hosted instance.""" + sh = api.self_hosted() try: projects = list( - api.self_hosted().get_projects( + sh.get_projects( page=page_num, size=page_size, all=fetch_all, @@ -120,9 +125,12 @@ def list_projects( except ValueError as e: payload = format_output(ApiResponseType(ok=False, errmsg=str(e))) else: - resp = ApiResponseType(ok=True, data={"list": projects}) - payload = format_output(resp) - if payload["ok"] and save_name is not None: + if sh._errors: + payload = format_output(ApiResponseType(ok=False, errmsg="; ".join(sh._errors))) + else: + resp = ApiResponseType(ok=True, data={"list": projects}) + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -142,7 +150,7 @@ def get_summary(save_name: str, api: Api): except ValueError as e: resp = ApiResponseType(ok=False, errmsg=str(e)) payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) @@ -161,9 +169,10 @@ def get_summary(save_name: str, api: Api): @with_custom_host def list_groups(page_num: int, page_size: int, fetch_all: bool, search: Optional[str], save_name: str, api: Api): """List all workspaces in the self-hosted instance.""" + sh = api.self_hosted() try: workspaces = list( - api.self_hosted().get_groups( + sh.get_groups( page=page_num, size=page_size, all=fetch_all, @@ -173,7 +182,10 @@ def list_groups(page_num: int, page_size: int, fetch_all: bool, search: Optional except ValueError as e: payload = format_output(ApiResponseType(ok=False, errmsg=str(e))) else: - resp = ApiResponseType(ok=True, data={"list": workspaces}) - payload = format_output(resp) - if payload["ok"] and save_name is not None: + if sh._errors: + payload = format_output(ApiResponseType(ok=False, errmsg="; ".join(sh._errors))) + else: + resp = ApiResponseType(ok=True, data={"list": workspaces}) + payload = format_output(resp) + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) diff --git a/swanlab/cli/api/user.py b/swanlab/cli/api/user.py index 2215c424c..ad8d5edaa 100644 --- a/swanlab/cli/api/user.py +++ b/swanlab/cli/api/user.py @@ -24,5 +24,5 @@ def get_user(save_name: str, api: Api): """Get current User info""" resp = api.user().wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) diff --git a/swanlab/cli/api/workspace.py b/swanlab/cli/api/workspace.py index 63b6fd53a..0852d769a 100644 --- a/swanlab/cli/api/workspace.py +++ b/swanlab/cli/api/workspace.py @@ -26,5 +26,5 @@ def get_workspace(username: str, save_name: str, api: Api): """Get Workspace info.""" resp = api.workspace(username).wrapper() payload = format_output(resp) - if payload["ok"] and save_name is not None: + if save_name is not None: save_output(orjson.dumps(payload, option=orjson.OPT_INDENT_2), name=save_name) From 61e47f294224a578a104deb17acf111051bd547f Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 14:24:13 +0800 Subject: [PATCH 08/17] feat: optimize metrics batch request --- swanlab/api/metric.py | 345 ++++++++++++++++++++++------------ swanlab/api/typings/common.py | 3 + 2 files changed, 226 insertions(+), 122 deletions(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index 8fad74881..5d2c4e6b9 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -7,11 +7,16 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any, Dict, Iterator, List, Optional, Union +from typing import TYPE_CHECKING, Any, Callable, Dict, Iterator, List, Optional, Union from swanlab.api.base import ApiClientContext, BaseEntity from swanlab.api.typings import ApiColumnCsvExportType, ApiResponseType -from swanlab.api.typings.common import ApiMetricColumnTypeLiteral, ApiMetricLogLevelLiteral, RangeQuery +from swanlab.api.typings.common import ( + MAX_METRIC_KEY_BATCH_SIZE, + ApiMetricColumnTypeLiteral, + ApiMetricLogLevelLiteral, + RangeQuery, +) from swanlab.api.typings.metric import ( ApiLogSeriesType, ApiMediaItemDataType, @@ -20,6 +25,7 @@ ) from swanlab.api.utils import get_properties, validate_metric_keys, validate_metric_log_level, validate_metric_type from swanlab.sdk.internal.pkg import console, safe +from swanlab.sdk.internal.pkg.executor import SafeThreadPoolExecutor if TYPE_CHECKING: from swanlab.sdk.internal.pkg.client import Client @@ -28,6 +34,29 @@ _METRIC_SHARED_KEYS = frozenset({"project_id", "run_id", "metric_type"}) +def _merge_value_stats( + step_list: List[Dict[str, Any]], + time_list: List[Dict[str, Any]], + keys: List[str], +) -> List[Dict[str, Any]]: + """合并 step/time 两种 x_type 的 value stat 响应为 per-key 统计字典。""" + merged: List[Dict[str, Any]] = [] + for i in range(len(keys)): + entry: Dict[str, Any] = {} + for field in _SCALAR_STATISTIC_FIELDS: + step_val = step_list[i].get(field) if i < len(step_list) else None + time_val = time_list[i].get(field) if i < len(time_list) else None + if step_val is not None: + stat = dict(step_val) + if time_val is not None and time_val.get("index") is not None: + stat["timestamp"] = time_val["index"] + entry[field] = stat + elif time_val is not None: + entry[field] = dict(time_val) + merged.append(entry) + return merged + + def _extract_csv_url(data: Any) -> str: if isinstance(data, list) and data: return data[0].get("url", "") @@ -519,12 +548,18 @@ class Metrics(BaseEntity): 一次 metrics 查询只支持一种 metric_type（SCALAR 或 MEDIA），不支持 LOG。 通过 payload 的 columns 数组一次性传递多个 key，减少网络请求。 + 内部通过 ``_ensure_batch()`` 缓存结果，避免 ``__iter__`` 与 ``json()`` 重复请求。 + 当 key 数量超过 ``_BATCH_SIZE``（默认 4）时自动分批，多批并发执行。 + 用法:: for m in experiment.metrics(keys=["loss", "acc"], metric_type="SCALAR"): print(m.key, m.metrics) """ + # 每批最大 key 数量，超出后自动拆分为多批并发 + _BATCH_SIZE = MAX_METRIC_KEY_BATCH_SIZE + def __init__( self, ctx: ApiClientContext, @@ -552,7 +587,6 @@ def __init__( self._run_id = run_id self._keys = keys self._metric_type = metric_type - self._ignore_timestamp = ignore_timestamp self._media_step = media_step self._all = all @@ -570,20 +604,67 @@ def __init__( if sample > 1500: console.warning(f"Get sample = [{sample}], expected <= 1500, will be constrainted automatically..") self._sample = 1500 + # 批量结果缓存：避免 __iter__ 与 json() 重复请求 + self._cached_list: Optional[List[Metric]] = None + + # ------------------------------------------------------------------ + # 公开接口 + # ------------------------------------------------------------------ + + def _ensure_batch(self) -> List[Metric]: + """Fetch (if needed) and cache batch Metric objects.""" + if self._cached_list is not None: + return self._cached_list + self._cached_list = list(self._fetch_batch()) + return self._cached_list def __iter__(self) -> Iterator[Metric]: + yield from self._ensure_batch() + + def json(self) -> Dict[str, Any]: + self._page_info["list"] = [ + {k: v for k, v in m.json().items() if k not in _METRIC_SHARED_KEYS} for m in self._ensure_batch() + ] + return self._page_info + + # ------------------------------------------------------------------ + # Batch dispatch: 分批限流 + # ------------------------------------------------------------------ + + def _fetch_batch(self) -> Iterator[Metric]: + """根据 metric_type 和模式分发到具体的获取方法。""" if self._metric_type == "SCALAR": - if self._range_query is not None: - yield from self._fetch_scalars_range() - elif self._all: - yield from self._fetch_scalars_all() + if self._range_query is not None or self._all: + data_list = self._batch_keys(self._fetch_scalar_csv) else: - yield from self._fetch_scalars() + data_list = self._batch_keys(self._fetch_scalar_lines) else: + # media 后端已支持 columns 批量，无需分批 if self._all: - yield from self._fetch_medias_all() + data_list = self._fetch_media_all() else: - yield from self._fetch_medias() + data_list = self._fetch_media_data() + + for data in data_list: + yield self._build_metric(data.get("key", ""), data) + + def _batch_keys(self, fetch_fn: Callable[[List[str]], List[Dict[str, Any]]]) -> List[Dict[str, Any]]: + """将 keys 按 ``_BATCH_SIZE`` 分批；单批直接执行，多批并发。""" + keys = self._keys + if len(keys) <= self._BATCH_SIZE: + return fetch_fn(keys) + + chunks = [keys[i : i + self._BATCH_SIZE] for i in range(0, len(keys), self._BATCH_SIZE)] + with SafeThreadPoolExecutor(max_workers=min(len(chunks), self._BATCH_SIZE)) as pool: + futures = [pool.submit(fetch_fn, chunk) for chunk in chunks] + results: List[Dict[str, Any]] = [] + for f in futures: + results.extend(f.result()) + return results + + # ------------------------------------------------------------------ + # Metric 对象构建 + # ------------------------------------------------------------------ def _build_metric(self, key: str, data: Dict[str, Any]) -> Metric: return Metric( @@ -601,10 +682,18 @@ def _build_metric(self, key: str, data: Dict[str, Any]) -> Metric: root_exp_id=self._root_exp_id, ) - def _fetch_value_stats_dual(self, keys: List[str], sample: int) -> List[Dict[str, Any]]: - """并发获取 step/time 两种 x_type 的统计值，合并为扁平结构。""" + # ------------------------------------------------------------------ + # 共享辅助方法 + # ------------------------------------------------------------------ + + def _empty_scalar_results(self, keys: List[str]) -> List[Dict[str, Any]]: + """返回 per-key 空结果列表（用于 early return）。""" + return [{"projectId": self._project_id, "experimentId": self._run_id, "key": k, "metrics": []} for k in keys] + + def _build_value_stats_requests(self, keys: List[str]) -> List[tuple]: + """构建 step/time value stats 的并发请求列表（2 路并发）。""" value_path = "/house/metrics/scalar/value" - requests = [ + return [ ( self._post, value_path, @@ -613,7 +702,7 @@ def _fetch_value_stats_dual(self, keys: List[str], sample: int) -> List[Dict[str self._project_id, self._run_id, keys, - sample, + self._sample, x_type=x_type, root_pro_id=self._root_pro_id, root_exp_id=self._root_exp_id, @@ -623,124 +712,135 @@ def _fetch_value_stats_dual(self, keys: List[str], sample: int) -> List[Dict[str for x_type in ("step", "timestamp") ] - step_resp, time_resp = self._concurrent_request(requests) + @staticmethod + def _extract_value_stats( + step_resp: ApiResponseType, + time_resp: ApiResponseType, + keys: List[str], + ) -> List[Dict[str, Any]]: + """从并发的 step/time 响应中提取并合并 value stats。""" + step_list = step_resp.data if step_resp.ok and isinstance(step_resp.data, list) else [] + time_list = time_resp.data if time_resp.ok and isinstance(time_resp.data, list) else [] + return _merge_value_stats(step_list, time_list, keys) - step_list: List[Dict[str, Any]] = step_resp.data if step_resp.ok and isinstance(step_resp.data, list) else [] - time_list: List[Dict[str, Any]] = time_resp.data if time_resp.ok and isinstance(time_resp.data, list) else [] + # ------------------------------------------------------------------ + # Scalar: 折线数据 + 统计值 (后端 columns 批量) + # ------------------------------------------------------------------ - merged: List[Dict[str, Any]] = [] - for i in range(len(keys)): - entry: Dict[str, Any] = {} - for field in _SCALAR_STATISTIC_FIELDS: - step_val = step_list[i].get(field) if i < len(step_list) else None - time_val = time_list[i].get(field) if i < len(time_list) else None - if step_val is not None: - stat = dict(step_val) - if time_val is not None and time_val.get("index") is not None: - stat["timestamp"] = time_val["index"] - entry[field] = stat - elif time_val is not None: - entry[field] = dict(time_val) - merged.append(entry) - return merged - - def _fetch_scalars_range(self) -> Iterator[Metric]: - """CSV 全量下载 + 客户端 range_query 过滤（仅 SCALAR）。""" - assert self._range_query is not None - rq = self._range_query - - # 1. 并发获取统计值（step + time） - value_list = self._fetch_value_stats_dual(self._keys, self._sample) - - # 2. 逐 key 下载 CSV 并过滤 - for i, key in enumerate(self._keys): - data: Dict[str, Any] = { - "projectId": self._project_id, - "experimentId": self._run_id, - "key": key, - "metrics": [], - } + def _fetch_scalar_lines(self, keys: List[str]) -> List[Dict[str, Any]]: + """获取标量折线数据 + step/time 统计值，3 路并发。 + + 后端 ``POST /house/metrics/scalar`` 和 ``/scalar/value`` 的 ``columns`` + 数组天然支持多 key，此处将 keys 打包为一个批量请求。 + """ + # 3 路并发：折线数据 + step 统计 + time 统计 + requests: List[tuple] = [ + ( + self._post, + "/house/metrics/scalar", + { + "data": Metric._build_scalar_payload( + self._project_id, + self._run_id, + keys, + self._sample, + root_pro_id=self._root_pro_id, + root_exp_id=self._root_exp_id, + ) + }, + ), + ] + requests.extend(self._build_value_stats_requests(keys)) - # 获取 presigned URL - resp = self._get(f"/experiment/{self._run_id}/column/csv", params={"key": key}) - if not resp.ok or not resp.data: - if i < len(value_list): - data.update(value_list[i]) - yield self._build_metric(key, data) - continue - - url = _extract_csv_url(resp.data) - if not url: - if i < len(value_list): - data.update(value_list[i]) - yield self._build_metric(key, data) - continue - - # 下载 CSV 并流式解析 - rows = _stream_csv_rows(self._ctx.client, url, rq) - data["metrics"] = rows or [] + scalar_resp, step_resp, time_resp = self._concurrent_request(requests) - if i < len(value_list): - data.update(value_list[i]) - yield self._build_metric(key, data) + scalar_list = scalar_resp.data if scalar_resp.ok and isinstance(scalar_resp.data, list) else [] + # scalar_list 元素是 dict，提取 .get("metrics", []) 统一为 List[List] + metrics_per_key = [entry.get("metrics", []) for entry in scalar_list] + value_list = self._extract_value_stats(step_resp, time_resp, keys) - def _fetch_scalars(self) -> Iterator[Metric]: - payload = Metric._build_scalar_payload( - self._project_id, - self._run_id, - self._keys, - self._sample, - root_pro_id=self._root_pro_id, - root_exp_id=self._root_exp_id, - ) + return self._build_scalar_results(keys, metrics_per_key, value_list) - # 1. 获取折线数据 - scalar_resp = self._post("/house/metrics/scalar", data=payload) - scalar_list: List[Dict[str, Any]] = ( - scalar_resp.data if scalar_resp.ok and isinstance(scalar_resp.data, list) else [] - ) + # ------------------------------------------------------------------ + # Scalar: CSV 全量下载 + 统计值 (range_query 或 all 模式) + # ------------------------------------------------------------------ - # 2. 并发获取统计值（step + time） - value_list = self._fetch_value_stats_dual(self._keys, self._sample) + def _fetch_scalar_csv(self, keys: List[str]) -> List[Dict[str, Any]]: + """CSV 全量下载 + value stats，URL 获取与下载均并发。 - for i, key in enumerate(self._keys): - data: Dict[str, Any] = { - "projectId": self._project_id, - "experimentId": self._run_id, - "key": key, - "metrics": [], - } - if i < len(scalar_list): - data["metrics"] = scalar_list[i].get("metrics", []) - if i < len(value_list): - data.update(value_list[i]) - yield self._build_metric(key, data) + value stats 通过后端 ``columns`` 批量获取； + CSV presigned URL 通过 ``GET /experiment/{run_id}/column/csv`` per-key 获取， + 多 key 时并发拉取 URL + 并发下载 CSV。 + """ + # 并发：step 统计 + time 统计 + per-key CSV URL + requests: List[tuple] = self._build_value_stats_requests(keys) + for key in keys: + requests.append((self._get, f"/experiment/{self._run_id}/column/csv", {"params": {"key": key}})) - def _fetch_scalars_all(self) -> Iterator[Metric]: - csv_data: Dict[str, List[Dict[str, Any]]] = {} - for key in self._keys: - resp = self._get(f"/experiment/{self._run_id}/column/csv", params={"key": key}) + all_resps = self._concurrent_request(requests) + step_resp, time_resp = all_resps[0], all_resps[1] + csv_resps = all_resps[2:] + + value_list = self._extract_value_stats(step_resp, time_resp, keys) + + # 提取 presigned CSV URL + urls: List[Optional[str]] = [] + for resp in csv_resps: + url = "" if resp.ok and resp.data: url = _extract_csv_url(resp.data) - if url: - rows = _stream_csv_rows(self._ctx.client, url) - csv_data[key] = rows or [] + urls.append(url or None) - # 并发获取统计值（step + time） - value_list = self._fetch_value_stats_dual(self._keys, self._sample) + # 并发下载 CSV 并解析 + csv_data = self._download_csvs(urls) - for i, key in enumerate(self._keys): + return self._build_scalar_results(keys, csv_data, value_list) + + # ------------------------------------------------------------------ + # Scalar results 构建 + # ------------------------------------------------------------------ + + def _build_scalar_results( + self, + keys: List[str], + metrics_data: List[Any], + value_list: List[Dict[str, Any]], + ) -> List[Dict[str, Any]]: + """将折线/CSV 数据与 value stats 合并为 per-key 结果字典。""" + results: List[Dict[str, Any]] = [] + for i, key in enumerate(keys): data: Dict[str, Any] = { "projectId": self._project_id, "experimentId": self._run_id, "key": key, - "metrics": csv_data.get(key, []), + "metrics": metrics_data[i] if i < len(metrics_data) else [], } if i < len(value_list): data.update(value_list[i]) - yield self._build_metric(key, data) + results.append(data) + return results - def _fetch_medias(self) -> Iterator[Metric]: + # ------------------------------------------------------------------ + # CSV 并发下载 + # ------------------------------------------------------------------ + + def _download_csvs(self, urls: List[Optional[str]]) -> List[List[Dict[str, Any]]]: + """通过线程池并发下载并解析 CSV 文件。""" + if not any(urls): + return [[] for _ in urls] + + with SafeThreadPoolExecutor(max_workers=min(len(urls), self._BATCH_SIZE)) as pool: + futures = [ + pool.submit(_stream_csv_rows, self._ctx.client, url, self._range_query) if url else None for url in urls + ] + return [(f.result() or []) if f is not None else [] for f in futures] + + # ------------------------------------------------------------------ + # Media fetch（后端 columns 批量，单次请求，无需分批） + # ------------------------------------------------------------------ + + def _fetch_media_data(self) -> List[Dict[str, Any]]: + """获取媒体数据（单步），后端 columns 批量一次返回。""" payload = Metric._build_media_payload( self._project_id, self._run_id, @@ -751,10 +851,10 @@ def _fetch_medias(self) -> Iterator[Metric]: ) raw_resp = self._post("/house/metrics/media", data=payload) if not raw_resp.ok or not raw_resp.data: - return + return self._empty_scalar_results(self._keys) resp_data = raw_resp.data if not isinstance(resp_data, dict): - return + return self._empty_scalar_results(self._keys) steps = resp_data.get("steps", []) current_step = resp_data.get("step") @@ -769,6 +869,7 @@ def _fetch_medias(self) -> Iterator[Metric]: ) key_to_entry: Dict[str, Dict[str, Any]] = {e.get("key", ""): e for e in metrics_raw} + results: List[Dict[str, Any]] = [] for key in self._keys: data: Dict[str, Any] = { "projectId": self._project_id, @@ -782,9 +883,11 @@ def _fetch_medias(self) -> Iterator[Metric]: if entry: items = Metric._build_media_items(entry, url_map) data["metrics"] = [{"index": current_step or 0, "items": items}] - yield self._build_metric(key, data) + results.append(data) + return results - def _fetch_medias_all(self) -> Iterator[Metric]: + def _fetch_media_all(self) -> List[Dict[str, Any]]: + """获取全部媒体数据，后端 columns 批量一次返回。""" payload = Metric._build_media_payload( self._project_id, self._run_id, @@ -794,10 +897,10 @@ def _fetch_medias_all(self) -> Iterator[Metric]: ) raw_resp = self._post("/house/metrics/f_media", data=payload) if not raw_resp.ok or not raw_resp.data: - return + return self._empty_scalar_results(self._keys) raw_list = raw_resp.data if not isinstance(raw_list, list): - return + return self._empty_scalar_results(self._keys) prefix = f"{self._project_id}/{self._run_id}" all_paths = [p for entry in raw_list for m in entry.get("metrics", []) for p in m.get("data", [])] @@ -808,6 +911,7 @@ def _fetch_medias_all(self) -> Iterator[Metric]: ) key_to_entry: Dict[str, Dict[str, Any]] = {e.get("key", ""): e for e in raw_list} + results: List[Dict[str, Any]] = [] for key in self._keys: data: Dict[str, Any] = { "projectId": self._project_id, @@ -822,8 +926,5 @@ def _fetch_medias_all(self) -> Iterator[Metric]: items = Metric._build_media_items(m, url_map) metrics_list.append({"index": m.get("index", 0), "items": items}) data["metrics"] = metrics_list - yield self._build_metric(key, data) - - def json(self) -> Dict[str, Any]: - self._page_info["list"] = [{k: v for k, v in m.json().items() if k not in _METRIC_SHARED_KEYS} for m in self] - return self._page_info + results.append(data) + return results diff --git a/swanlab/api/typings/common.py b/swanlab/api/typings/common.py index 546e2f5f1..1ceea1b0f 100644 --- a/swanlab/api/typings/common.py +++ b/swanlab/api/typings/common.py @@ -142,6 +142,9 @@ # 并发请求数 MAX_CONCURRENT_COUNT: int = 4 +# CH 每次查询 key 数量上限 +MAX_METRIC_KEY_BATCH_SIZE: int = 4 + class RangeQuery(BaseModel, frozen=True): """ From 432095e9dfb345e5fddb02121edf59a7ea36f690 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 14:48:25 +0800 Subject: [PATCH 09/17] fix: deduplicate keys --- swanlab/api/metric.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index 5d2c4e6b9..7ad8ba9f0 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -585,7 +585,8 @@ def __init__( raise ValueError("range_query is only supported for SCALAR metric_type") self._project_id = project_id self._run_id = run_id - self._keys = keys + # 去重，保持插入顺序 + self._keys = list(dict.fromkeys(keys)) self._metric_type = metric_type self._ignore_timestamp = ignore_timestamp self._media_step = media_step From 0cc43f67f24bf1b5c4d6680e00ccc0503932fc60 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 15:10:35 +0800 Subject: [PATCH 10/17] fix: metric key order --- swanlab/api/metric.py | 39 +++++++++++++++++++++++++++++---------- 1 file changed, 29 insertions(+), 10 deletions(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index 7ad8ba9f0..ba51251c4 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -34,24 +34,40 @@ _METRIC_SHARED_KEYS = frozenset({"project_id", "run_id", "metric_type"}) +def _index_entries_by_key(entries: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + """Build a key-indexed response map; House may omit columns or return them out of order.""" + indexed: Dict[str, Dict[str, Any]] = {} + for entry in entries: + if not isinstance(entry, dict): + continue + key = entry.get("key") + if isinstance(key, str) and key: + indexed[key] = entry + return indexed + + def _merge_value_stats( step_list: List[Dict[str, Any]], time_list: List[Dict[str, Any]], keys: List[str], ) -> List[Dict[str, Any]]: """合并 step/time 两种 x_type 的 value stat 响应为 per-key 统计字典。""" + step_by_key = _index_entries_by_key(step_list) + time_by_key = _index_entries_by_key(time_list) merged: List[Dict[str, Any]] = [] - for i in range(len(keys)): + for key in keys: + step_entry = step_by_key.get(key, {}) + time_entry = time_by_key.get(key, {}) entry: Dict[str, Any] = {} for field in _SCALAR_STATISTIC_FIELDS: - step_val = step_list[i].get(field) if i < len(step_list) else None - time_val = time_list[i].get(field) if i < len(time_list) else None - if step_val is not None: + step_val = step_entry.get(field) + time_val = time_entry.get(field) + if isinstance(step_val, dict): stat = dict(step_val) - if time_val is not None and time_val.get("index") is not None: + if isinstance(time_val, dict) and time_val.get("index") is not None: stat["timestamp"] = time_val["index"] entry[field] = stat - elif time_val is not None: + elif isinstance(time_val, dict): entry[field] = dict(time_val) merged.append(entry) return merged @@ -59,9 +75,12 @@ def _merge_value_stats( def _extract_csv_url(data: Any) -> str: if isinstance(data, list) and data: - return data[0].get("url", "") + return _extract_csv_url(data[0]) if isinstance(data, dict): - return data.get("url", "") + url = data.get("url", "") + if isinstance(url, str) and url: + return url + return _extract_csv_url(data.get("data")) return "" @@ -756,8 +775,8 @@ def _fetch_scalar_lines(self, keys: List[str]) -> List[Dict[str, Any]]: scalar_resp, step_resp, time_resp = self._concurrent_request(requests) scalar_list = scalar_resp.data if scalar_resp.ok and isinstance(scalar_resp.data, list) else [] - # scalar_list 元素是 dict，提取 .get("metrics", []) 统一为 List[List] - metrics_per_key = [entry.get("metrics", []) for entry in scalar_list] + scalar_by_key = _index_entries_by_key(scalar_list) + metrics_per_key = [scalar_by_key.get(key, {}).get("metrics", []) for key in keys] value_list = self._extract_value_stats(step_resp, time_resp, keys) return self._build_scalar_results(keys, metrics_per_key, value_list) From 1cd5e3a634f640f059cb72fb7714e4bb42ceb71e Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 15:16:35 +0800 Subject: [PATCH 11/17] fix: column commens --- swanlab/api/column.py | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/swanlab/api/column.py b/swanlab/api/column.py index 38f6a4a4a..9472a0543 100644 --- a/swanlab/api/column.py +++ b/swanlab/api/column.py @@ -5,7 +5,6 @@ @description: Column 实体类 — 实验列的查询与操作 """ -import dataclasses from typing import Any, Callable, Dict, Iterator, Optional, cast from swanlab.api.base import ApiClientContext, BaseEntity @@ -282,17 +281,11 @@ def __iter__(self) -> Iterator[Column]: if self._column_class: extra["class"] = self._column_class - # all=True + 有过滤条件时：后端 all=True 模式无视 search/type/class 仅做全量返回， - # 所以改为 all=False 走分页过滤 + 自动翻页收集全部结果 - use_auto_paginate = self._query.all and (self._query.search or self._column_type or self._column_class) - query = dataclasses.replace(self._query, all=use_auto_paginate) if use_auto_paginate else self._query - if use_auto_paginate: - if self._query.search: - extra["search"] = self._query.search - for item in self._paginate( f"/experiment/{run_id}/column", - query, + # PaginatedQuery.all is client-side only: to_params() does not send it + # to the backend, so filtered all=True requests still use paged search. + self._query, page_info=self._page_info, extra=extra, ): From 6bac0e307b34adac661c4f149eb1bd250a335f31 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 15:39:04 +0800 Subject: [PATCH 12/17] fix: rename comments --- swanlab/api/metric.py | 100 ++++++++++++++++++++++++++++-------------- 1 file changed, 68 insertions(+), 32 deletions(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index ba51251c4..f56e72d23 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -34,8 +34,8 @@ _METRIC_SHARED_KEYS = frozenset({"project_id", "run_id", "metric_type"}) -def _index_entries_by_key(entries: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: - """Build a key-indexed response map; House may omit columns or return them out of order.""" +def _align_entries_by_key(entries: List[Dict[str, Any]]) -> Dict[str, Dict[str, Any]]: + """将后端返回的列表按 ``key`` 字段映射为 dict，应对后端可能省略列或乱序返回。""" indexed: Dict[str, Dict[str, Any]] = {} for entry in entries: if not isinstance(entry, dict): @@ -52,8 +52,8 @@ def _merge_value_stats( keys: List[str], ) -> List[Dict[str, Any]]: """合并 step/time 两种 x_type 的 value stat 响应为 per-key 统计字典。""" - step_by_key = _index_entries_by_key(step_list) - time_by_key = _index_entries_by_key(time_list) + step_by_key = _align_entries_by_key(step_list) + time_by_key = _align_entries_by_key(time_list) merged: List[Dict[str, Any]] = [] for key in keys: step_entry = step_by_key.get(key, {}) @@ -242,6 +242,13 @@ def metric_type(self) -> str: @property def metrics(self) -> List[Any]: + """指标数据列表。SCALAR 类型为采样后的折线点（含 index/data/timestamp）， + MEDIA 类型为 ``[{index, items}]`` 结构。 + + .. note:: + 当通过 ``Metrics`` 批量查询并使用 ``range_query`` 的 ``head`` / ``tail`` 时， + ``head`` / ``tail`` 是 post-sampling 操作——在采样/下载完成后截取，而非对原始全量数据截取后再采样。 + """ return self._ensure_data().get("metrics", []) @property @@ -354,18 +361,31 @@ def _fetch_scalar(self) -> ApiScalarSeriesType: root_exp_id=self._root_exp_id, ) - # 1. 获取折线数据 - raw_data = self._extract_first(self._post("/house/metrics/scalar", data=payload)) - if raw_data is None: + # 1. 获取折线数据 — 使用 key-indexed lookup 保证对齐 + scalar_resp = self._post("/house/metrics/scalar", data=payload) + if scalar_resp.ok and isinstance(scalar_resp.data, list): + scalar_by_key = _align_entries_by_key(scalar_resp.data) + res["metrics"] = scalar_by_key.get(self.key, {}).get("metrics", []) + if not res.get("metrics"): return res - res["metrics"] = raw_data.get("metrics", []) - # 2. 获取统计值 - stat_data = self._extract_first(self._post("/house/metrics/scalar/value", data=payload)) - if stat_data is None: - return res - for field in _SCALAR_STATISTIC_FIELDS: - res[field] = stat_data.get(field, {}) + # 2. 获取统计值 — step/time 并发，key-indexed 合并 + step_payload = {**payload, "xType": "step"} + time_payload = {**payload, "xType": "timestamp"} + step_resp, time_resp = self._concurrent_request( + [ + (self._post, "/house/metrics/scalar/value", {"data": step_payload}), + (self._post, "/house/metrics/scalar/value", {"data": time_payload}), + ] + ) + step_list = step_resp.data if step_resp.ok and isinstance(step_resp.data, list) else [] + time_list = time_resp.data if time_resp.ok and isinstance(time_resp.data, list) else [] + value_list = _merge_value_stats(step_list, time_list, [self.key]) + if value_list: + for field in _SCALAR_STATISTIC_FIELDS: + val = value_list[0].get(field) + if val: + res[field] = val return res @staticmethod @@ -570,6 +590,12 @@ class Metrics(BaseEntity): 内部通过 ``_ensure_batch()`` 缓存结果，避免 ``__iter__`` 与 ``json()`` 重复请求。 当 key 数量超过 ``_BATCH_SIZE``（默认 4）时自动分批，多批并发执行。 + .. note:: + ``range_query`` 的 ``head`` 和 ``tail`` 参数是 post-sampling 操作： + 在 sampled 模式下，服务端先做 LTTB 降采样，再对采样结果截取 head/tail； + 在 CSV 全量下载模式（``all`` 或 ``range_query``）下，先下载完整数据并做范围过滤， + 再对过滤后的结果截取 head/tail。因此 head/tail 不等同于对原始全量数据截取后再采样。 + 用法:: for m in experiment.metrics(keys=["loss", "acc"], metric_type="SCALAR"): @@ -775,11 +801,12 @@ def _fetch_scalar_lines(self, keys: List[str]) -> List[Dict[str, Any]]: scalar_resp, step_resp, time_resp = self._concurrent_request(requests) scalar_list = scalar_resp.data if scalar_resp.ok and isinstance(scalar_resp.data, list) else [] - scalar_by_key = _index_entries_by_key(scalar_list) - metrics_per_key = [scalar_by_key.get(key, {}).get("metrics", []) for key in keys] + scalar_by_key = _align_entries_by_key(scalar_list) + metrics_by_key: Dict[str, Any] = {key: scalar_by_key.get(key, {}).get("metrics", []) for key in keys} value_list = self._extract_value_stats(step_resp, time_resp, keys) + value_by_key: Dict[str, Dict[str, Any]] = {keys[i]: v for i, v in enumerate(value_list)} - return self._build_scalar_results(keys, metrics_per_key, value_list) + return self._build_scalar_results(keys, metrics_by_key, value_by_key) # ------------------------------------------------------------------ # Scalar: CSV 全量下载 + 统计值 (range_query 或 all 模式) @@ -802,19 +829,24 @@ def _fetch_scalar_csv(self, keys: List[str]) -> List[Dict[str, Any]]: csv_resps = all_resps[2:] value_list = self._extract_value_stats(step_resp, time_resp, keys) + value_by_key: Dict[str, Dict[str, Any]] = {keys[i]: v for i, v in enumerate(value_list)} - # 提取 presigned CSV URL - urls: List[Optional[str]] = [] - for resp in csv_resps: + # 提取 presigned CSV URL（按 key 索引） + url_by_key: Dict[str, Optional[str]] = {} + for i, key in enumerate(keys): url = "" - if resp.ok and resp.data: - url = _extract_csv_url(resp.data) - urls.append(url or None) + if i < len(csv_resps) and csv_resps[i].ok and csv_resps[i].data: + url = _extract_csv_url(csv_resps[i].data) + url_by_key[key] = url or None # 并发下载 CSV 并解析 - csv_data = self._download_csvs(urls) + urls_ordered = [url_by_key.get(key) for key in keys] + csv_rows_list = self._download_csvs(urls_ordered) + metrics_by_key: Dict[str, Any] = { + keys[i]: csv_rows_list[i] if i < len(csv_rows_list) else [] for i, key in enumerate(keys) + } - return self._build_scalar_results(keys, csv_data, value_list) + return self._build_scalar_results(keys, metrics_by_key, value_by_key) # ------------------------------------------------------------------ # Scalar results 构建 @@ -823,20 +855,24 @@ def _fetch_scalar_csv(self, keys: List[str]) -> List[Dict[str, Any]]: def _build_scalar_results( self, keys: List[str], - metrics_data: List[Any], - value_list: List[Dict[str, Any]], + metrics_by_key: Dict[str, Any], + value_by_key: Dict[str, Dict[str, Any]], ) -> List[Dict[str, Any]]: - """将折线/CSV 数据与 value stats 合并为 per-key 结果字典。""" + """将折线/CSV 数据与 value stats 合并为 per-key 结果字典。 + + 所有数据通过 key 索引查找，保证与请求 keys 顺序对齐，不依赖后端返回顺序。 + """ results: List[Dict[str, Any]] = [] - for i, key in enumerate(keys): + for key in keys: data: Dict[str, Any] = { "projectId": self._project_id, "experimentId": self._run_id, "key": key, - "metrics": metrics_data[i] if i < len(metrics_data) else [], + "metrics": metrics_by_key.get(key, []), } - if i < len(value_list): - data.update(value_list[i]) + stats = value_by_key.get(key, {}) + if stats: + data.update(stats) results.append(data) return results From 75a5ccca8121bdbfaab35f37e4510768e3d76a02 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 16:01:56 +0800 Subject: [PATCH 13/17] fix: column page_size --- swanlab/api/column.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/swanlab/api/column.py b/swanlab/api/column.py index 9472a0543..a9d5e4fc1 100644 --- a/swanlab/api/column.py +++ b/swanlab/api/column.py @@ -315,9 +315,6 @@ def total(self) -> int: def json(self) -> Dict[str, Any]: items = [c.json() for c in self] self._page_info["list"] = items - # 自动翻页后 total/size/pages 应与实际 list 对齐 - if self._query.all: - self._page_info["total"] = len(items) - self._page_info["size"] = len(items) - self._page_info["pages"] = 1 + # total/pages 来自后端第一页响应，始终为匹配 search 的真实总数； + # size 保持用户请求的分页大小，不因 all=True 而覆盖。 return self._page_info From fc784379c91e7f0d06fd50b5d9e13d0012f59588 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 16:20:07 +0800 Subject: [PATCH 14/17] fix: warning at most once --- swanlab/api/metric.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index f56e72d23..29fd14ff4 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -111,6 +111,8 @@ def _stream_csv_rows( if rq is not None and rq.last is not None: last_start_ts = int(time.time() * 1000) - rq.last + _warned_missing_ts = False + for row in csv.reader(lines): if not row or len(row) < 2: continue @@ -133,7 +135,9 @@ def _stream_csv_rows( if last_start_ts is not None: ts = item.get("timestamp") if ts is None: - console.warning(f"CSV row missing timestamp column: {row}") + if not _warned_missing_ts: + console.warning("CSV row missing `timestamp` column.") + _warned_missing_ts = True continue if ts < last_start_ts: continue @@ -141,7 +145,9 @@ def _stream_csv_rows( elif rq.type == "timestamp": ts = item.get("timestamp") if ts is None: - console.warning(f"CSV row missing timestamp column: {row}") + if not _warned_missing_ts: + console.warning("CSV row missing `timestamp` column.") + _warned_missing_ts = True continue if rq.start is not None and ts < rq.start: continue From 42729ccd63b644cafc56b938de18ebe60309a14a Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 16:32:04 +0800 Subject: [PATCH 15/17] fix: download without concurrent --- swanlab/api/metric.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/swanlab/api/metric.py b/swanlab/api/metric.py index 29fd14ff4..315d2caad 100644 --- a/swanlab/api/metric.py +++ b/swanlab/api/metric.py @@ -819,11 +819,11 @@ def _fetch_scalar_lines(self, keys: List[str]) -> List[Dict[str, Any]]: # ------------------------------------------------------------------ def _fetch_scalar_csv(self, keys: List[str]) -> List[Dict[str, Any]]: - """CSV 全量下载 + value stats，URL 获取与下载均并发。 + """CSV 全量下载 + value stats，URL 获取并发，CSV 顺序下载。 value stats 通过后端 ``columns`` 批量获取； CSV presigned URL 通过 ``GET /experiment/{run_id}/column/csv`` per-key 获取， - 多 key 时并发拉取 URL + 并发下载 CSV。 + 多 key 时并发拉取 URL，但 CSV 下载为顺序执行（每批最多 4 个 key）。 """ # 并发：step 统计 + time 统计 + per-key CSV URL requests: List[tuple] = self._build_value_stats_requests(keys) @@ -845,7 +845,7 @@ def _fetch_scalar_csv(self, keys: List[str]) -> List[Dict[str, Any]]: url = _extract_csv_url(csv_resps[i].data) url_by_key[key] = url or None - # 并发下载 CSV 并解析 + # 下载 CSV 并解析（每批最多 _BATCH_SIZE 个 key，穿行执行防止并发膨胀） urls_ordered = [url_by_key.get(key) for key in keys] csv_rows_list = self._download_csvs(urls_ordered) metrics_by_key: Dict[str, Any] = { @@ -883,19 +883,19 @@ def _build_scalar_results( return results # ------------------------------------------------------------------ - # CSV 并发下载 + # CSV 后处理 # ------------------------------------------------------------------ def _download_csvs(self, urls: List[Optional[str]]) -> List[List[Dict[str, Any]]]: - """通过线程池并发下载并解析 CSV 文件。""" - if not any(urls): - return [[] for _ in urls] - - with SafeThreadPoolExecutor(max_workers=min(len(urls), self._BATCH_SIZE)) as pool: - futures = [ - pool.submit(_stream_csv_rows, self._ctx.client, url, self._range_query) if url else None for url in urls - ] - return [(f.result() or []) if f is not None else [] for f in futures] + """顺序下载并解析 CSV 文件（每批最多 ``_BATCH_SIZE`` 个 key，无需并发）。""" + results: List[List[Dict[str, Any]]] = [] + for url in urls: + if url: + rows = _stream_csv_rows(self._ctx.client, url, self._range_query) + results.append(rows or []) + else: + results.append([]) + return results # ------------------------------------------------------------------ # Media fetch（后端 columns 批量，单次请求，无需分批） From 20c4b9ecf6199fe151f46aae44fb4c3fb017eef3 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 18:51:33 +0800 Subject: [PATCH 16/17] chore: rename self_hosted file --- swanlab/api/__init__.py | 2 +- swanlab/api/{selfhosted.py => self_hosted.py} | 0 swanlab/cli/api/__init__.py | 2 +- swanlab/cli/api/{selfhosted.py => self_hosted.py} | 0 tests/unit/api/test_api.py | 2 +- tests/unit/api/test_utils.py | 2 +- 6 files changed, 4 insertions(+), 4 deletions(-) rename swanlab/api/{selfhosted.py => self_hosted.py} (100%) rename swanlab/cli/api/{selfhosted.py => self_hosted.py} (100%) diff --git a/swanlab/api/__init__.py b/swanlab/api/__init__.py index 9306ceb6b..919fd4eb3 100644 --- a/swanlab/api/__init__.py +++ b/swanlab/api/__init__.py @@ -17,7 +17,7 @@ from .column import Column, Columns from .experiment import Experiment, Experiments from .project import Project, Projects -from .selfhosted import SelfHosted +from .self_hosted import SelfHosted from .typings.common import ApiColumnClassLiteral, ApiColumnDataTypeLiteral, ApiVisibilityLiteral, PaginatedQuery from .user import User from .utils import validate_api_path, validate_non_empty_string diff --git a/swanlab/api/selfhosted.py b/swanlab/api/self_hosted.py similarity index 100% rename from swanlab/api/selfhosted.py rename to swanlab/api/self_hosted.py diff --git a/swanlab/cli/api/__init__.py b/swanlab/cli/api/__init__.py index b3c2583a6..4757542ce 100644 --- a/swanlab/cli/api/__init__.py +++ b/swanlab/cli/api/__init__.py @@ -9,7 +9,7 @@ from .experiment import run_cli from .project import project_cli -from .selfhosted import selfhosted_cli +from .self_hosted import selfhosted_cli from .user import user_cli from .workspace import workspace_cli diff --git a/swanlab/cli/api/selfhosted.py b/swanlab/cli/api/self_hosted.py similarity index 100% rename from swanlab/cli/api/selfhosted.py rename to swanlab/cli/api/self_hosted.py diff --git a/tests/unit/api/test_api.py b/tests/unit/api/test_api.py index fccfa5b25..b3ae12c19 100644 --- a/tests/unit/api/test_api.py +++ b/tests/unit/api/test_api.py @@ -18,7 +18,7 @@ from swanlab.api.experiment import Experiment, Experiments from swanlab.api.metric import Metric, Metrics from swanlab.api.project import Project -from swanlab.api.selfhosted import SelfHosted +from swanlab.api.self_hosted import SelfHosted from swanlab.api.typings.common import PaginatedQuery from swanlab.api.typings.selfhosted import ApiSelfHostedInfoType from swanlab.api.workspace import Workspace diff --git a/tests/unit/api/test_utils.py b/tests/unit/api/test_utils.py index 7ca32e256..e93d2c75b 100644 --- a/tests/unit/api/test_utils.py +++ b/tests/unit/api/test_utils.py @@ -8,7 +8,7 @@ import pytest -from swanlab.api.selfhosted import SelfHosted +from swanlab.api.self_hosted import SelfHosted from swanlab.api.typings.common import PaginatedQuery from swanlab.api.typings.selfhosted import ApiSelfHostedInfoType from swanlab.api.utils import ( From 286b89cdcf626f740e32bf9bb3e8ece547f21bf0 Mon Sep 17 00:00:00 2001 From: Nexisato <978452096@qq.com> Date: Thu, 11 Jun 2026 19:18:59 +0800 Subject: [PATCH 17/17] feat(skill): upgrade skill instruction --- skills/swanlab-skill/SKILL.md | 35 +++++++++++++++---- .../swanlab-skill/references/CLI_REFERENCE.md | 19 ++++++++-- .../references/SWANLAB_CONCEPTS.md | 10 ++++-- 3 files changed, 53 insertions(+), 11 deletions(-) diff --git a/skills/swanlab-skill/SKILL.md b/skills/swanlab-skill/SKILL.md index 3073f316c..e164680c6 100644 --- a/skills/swanlab-skill/SKILL.md +++ b/skills/swanlab-skill/SKILL.md @@ -1,7 +1,7 @@ --- name: swanlab-skill metadata: - version: "0.1.0" + version: "0.1.1" description: > Interact with SwanLab — both writing tracking code (init/log/finish/multimedia) and querying experiment data via CLI (`swanlab api`). Use this skill when the user wants to write training @@ -95,12 +95,17 @@ CLI commands use `username/project_name` (project) or `username/project_name/run | "console output" | Captured logs | `CLI_REFERENCE.md > run logs` | | "what columns are tracked" | Metric definitions | `CLI_REFERENCE.md > run columns` | | "check connectivity" / "can I reach swanlab" | Environment check | `swanlab ping` | +| "check login status" / "am I logged in" | Verify credentials | `swanlab verify` | --- ## Environment Connectivity -Before writing tracking code or running CLI queries, especially in `online` mode, use `swanlab ping` to verify that the current environment can reach the SwanLab server: +Before writing tracking code or running CLI queries, especially in `online` mode, run these two checks to confirm the environment is ready: + +### 1. `swanlab ping` — Test network reachability + +The fastest way to diagnose connectivity issues. Run it first when a user reports upload failures, login problems, or unknown mode fallbacks. ```bash swanlab ping @@ -108,7 +113,25 @@ swanlab ping # If ping fails, check SWANLAB_API_HOST / network proxy / firewall settings. ``` -This is the fastest way to diagnose connectivity issues — run it first when a user reports upload failures, login problems, or unknown mode fallbacks. +### 2. `swanlab verify` — Validate login credentials + +After confirming the server is reachable, use `swanlab verify` to check that stored credentials are valid and have not expired. This reads the API key and host from the local `.netrc` file (created by `swanlab login`). + +```bash +swanlab verify +# Validates stored API key against the server. +# Reports: which host (default https://swanlab.cn) and the logged-in username. +# Fails if: not logged in, API key is invalid, or key has expired. + +swanlab verify --local +# Check local login status (.swanlab in current directory) instead of the global one. +``` + +**Recommended pre-flight sequence**: + +1. `swanlab ping` → confirm the server is reachable +2. `swanlab verify` → confirm credentials are valid +3. Proceed with `swanlab api` queries or SDK code --- @@ -116,6 +139,6 @@ This is the fastest way to diagnose connectivity issues — run it first when a See `CLI_REFERENCE.md > Behavioral Constraints` for the full list. Key rules: -- **Never use `--all` unless the user explicitly asks for it.** -- **Always ask for specific column keys before running `run metrics`, `run medias`, or `run column`.** -- **Never dump large metric JSON directly into conversation.** Use `--save` to persist to file, then visualize with `scripts/plot_metrics.py --data file.json` or use `run summary` for aggregate stats. +- **Use `--all` only when the user explicitly asks for it** (e.g. "fetch all", "get everything", "complete list"). For paginated list commands, always use default pagination (`--page_num` / `--page_size`). +- **Always ask for specific column keys before running `run metrics`, `run medias`, or `run column`.** If the user doesn't know the key names, first run `run columns PATH` to discover them. +- **Always persist large metric data to file via `--save`**, then visualize with `scripts/plot_metrics.py --data file.json` or use `run summary` for aggregate stats. diff --git a/skills/swanlab-skill/references/CLI_REFERENCE.md b/skills/swanlab-skill/references/CLI_REFERENCE.md index 18a9aefce..07628c341 100644 --- a/skills/swanlab-skill/references/CLI_REFERENCE.md +++ b/skills/swanlab-skill/references/CLI_REFERENCE.md @@ -179,11 +179,17 @@ swanlab api run columns PATH [OPTIONS] |--------|-------|---------|-------------| | `--page_num` | `-n` | 1 | Page number (>= 1) | | `--page_size` | `-s` | 20 | Page size. One of: 10, 12, 15, 20, 24, 27, 50, 100 | +| `--search` | | none | Fuzzy search keyword (matches column **name**, case-insensitive) | | `--class` | | CUSTOM | Column class: `CUSTOM` or `SYSTEM` (see `SWANLAB_CONCEPTS.md > Column Classes`) | | `--type` | | all | Data type filter (see `SWANLAB_CONCEPTS.md > Column Data Types` for all valid values) | | `--all` | | false | Fetch all pages | | `--save` | | off | Save output to file | +> **Notes on columns**: +> - Columns are **paginated** — use `--page_num` / `--page_size` to iterate in order to fetch all pages. +> - To query system metrics (CPU, GPU, memory, etc.), you must explicitly pass `--class SYSTEM`. The default is `CUSTOM` (user-defined metrics only). +> - **Resumed experiments have no system columns.** When an experiment is resumed via `swanlab.init(resume=...)`, system metrics (hardware monitoring) are not re-collected, so `--class SYSTEM` will return an empty list. + #### `swanlab api run column PATH --key KEY` Get a single column by key name. @@ -228,11 +234,14 @@ swanlab api run metrics PATH --keys KEYS [OPTIONS] | `--range-end` | | none | Range end value (inclusive). Same type as `--range-start`. | | `--range-head` | | none | Return only the first N data points (int >= 1). Mutually exclusive with `--range-tail`. | | `--range-tail` | | none | Return only the last N data points (int >= 1). Mutually exclusive with `--range-head`. | +| `--range-last` | | none | Last N milliseconds of data (int >= 1). Mutually exclusive with `--range-start`/`--range-end`. Can combine with `--range-head`/`--range-tail`. | | `--save` | | off | Save output to file | **Range query constraints:** - `--range-head` and `--range-tail` are mutually exclusive. +- `--range-last` is mutually exclusive with `--range-start`/`--range-end`. - `--range-start` must be ≤ `--range-end`. +- `--range-head`/`--range-tail` can be combined with `--range-last` or `--range-start`/`--range-end`. - Range query is only supported for SCALAR metrics. It downloads CSV data and applies client-side filtering. - When using `--range-type timestamp`, each CSV row must have a timestamp column. Rows missing timestamps are skipped with a warning. @@ -247,6 +256,12 @@ swanlab api run metrics PATH --keys loss --range-head 50 # Get data by timestamp range (Unix milliseconds) swanlab api run metrics PATH --keys loss --range-type timestamp --range-start 1714368000000 --range-end 1714454400000 + +# Get data from the last 5 minutes +swanlab api run metrics PATH --keys loss --range-last 300000 + +# Get last 30 data points +swanlab api run metrics PATH --keys loss --range-tail 30 ``` > **Tip**: For large metric data, use `--save` to write JSON to file, then plot with `scripts/plot_metrics.py --data file.json -k loss`. For quick stats, prefer `run summary` over full metrics. @@ -409,8 +424,8 @@ swanlab api self-hosted summary [--save [FILENAME]] [--host HOST] [--api-key KEY ## Behavioral Constraints -- **Never use `--all` unless the user explicitly asks for it.** This flag bypasses pagination and fetches the entire dataset in one go, which puts heavy load on the database. For paginated list commands, always use default pagination (`--page_num` / `--page_size`). Only add `--all` if the user says something like "fetch all", "get everything", or "I want the complete list". -- **Always ask the user for specific column keys before running `run metrics`, `run medias`, or `run column`.** These commands accept a `--keys` or `--key` parameter. Querying without a specific key forces the server to scan and return data for all columns, which is expensive. If the user doesn't know the key names, first run `run columns PATH` to list available columns, then use the returned keys to make targeted queries. Never guess or fabricate key names — always discover them first. +- **Use `--all` only when the user explicitly asks for it.** This flag bypasses pagination and fetches the entire dataset in one go, which puts heavy load on the database. For paginated list commands, always use default pagination (`--page_num` / `--page_size`). Only add `--all` when the user says something like "fetch all", "get everything", or "I want the complete list". +- **Always ask the user for specific column keys before running `run metrics`, `run medias`, or `run column`.** These commands accept a `--keys` or `--key` parameter. Querying without a specific key forces the server to scan and return data for all columns, which is expensive. If the user doesn't know the key names, first run `run columns PATH` to list available columns, then use the returned keys for targeted queries. Always discover key names from `run columns` output before querying specific metrics. - **Always add `--ignore-timestamp` for `run metrics` and `run logs`.** Unless the user specifically asks to keep timestamps, include this flag by default. It produces cleaner, more readable output by removing Unix timestamps from every data point. - All output is JSON to stdout. Pipe to `jq` or similar tools for further processing. - `--save` without a filename auto-generates `swanlab-YYYYMMDD_HHMMSS-xxxx.json` in the current directory. diff --git a/skills/swanlab-skill/references/SWANLAB_CONCEPTS.md b/skills/swanlab-skill/references/SWANLAB_CONCEPTS.md index 27ee72744..cf931f16d 100644 --- a/skills/swanlab-skill/references/SWANLAB_CONCEPTS.md +++ b/skills/swanlab-skill/references/SWANLAB_CONCEPTS.md @@ -134,7 +134,11 @@ Structure per data point: | `--range-end` | none | End value, inclusive (int ≥ 0) | | `--range-head` | none | First N data points (int ≥ 1). Mutually exclusive with `--range-tail` | | `--range-tail` | none | Last N data points (int ≥ 1). Mutually exclusive with `--range-head` | +| `--range-last` | none | Last N milliseconds of data (int > 0). Mutually exclusive with `--range-start`/`--range-end`. Can combine with `--range-head`/`--range-tail`. | +- `--range-head` and `--range-tail` are mutually exclusive, acted as post-sampling. +- `--range-last` is mutually exclusive with `--range-start`/`--range-end`. +- `--range-head`/`--range-tail` can be combined with `--range-last` or `--range-start`/`--range-end`. - `--range-start` must be ≤ `--range-end`. - When `--range-type timestamp` is used, rows missing a timestamp column are skipped. - Range query bypasses the sampling API entirely — it streams and filters the CSV export directly. Statistics (min/max/avg/median/latest) are still fetched via the sampling API and are **not** affected by the range filter. @@ -235,16 +239,16 @@ Each experiment carries a `profile` object containing metadata about the run: ## Self-Hosted Instance -> **Self-hosted commands are only available for self-hosted (private) deployments.** If the resolved host contains `swanlab.cn`, these commands will fail — do not attempt them. +> **Self-hosted commands are only available for self-hosted (private) deployments.** Only proceed with these commands when the resolved host points to a self-hosted deployment. -SwanLab can be deployed as a self-hosted instance (private deployment). **Only use `swanlab api self-hosted` subcommands when the target is a self-hosted server, never on the public SwanLab cloud (`swanlab.cn`).** +SwanLab can be deployed as a self-hosted instance (private deployment). **Use `swanlab api self-hosted` subcommands exclusively on self-hosted deployments. Always verify the host points to a private server before invoking these commands.** **Host detection rule**: Before using any self-hosted command, check where requests will be sent: 1. If the user passes `--host` explicitly → use that value. 2. Otherwise, check `SWANLAB_API_HOST` / `SWANLAB_WEB_HOST` environment variables. 3. Otherwise, check `.netrc` or SwanLab config (`~/.swanlab`). -If the resolved host contains `swanlab.cn`, self-hosted commands will fail and should not be attempted. Only proceed when the host points to a self-hosted deployment. +Self-hosted commands only succeed on self-hosted deployments. Always confirm the resolved host points to a private server before proceeding. Self-hosted-specific features: