Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
27 commits
Select commit Hold shift + click to select a range
594b78c
feat: add skip_store setting
Nexisato Sep 8, 2026
c063ec4
feat: integration datastore writer
Nexisato Sep 9, 2026
407fe58
feat: add payload field for skip_store mode
Nexisato Sep 9, 2026
d0400f7
feat: support transform media
Nexisato Sep 9, 2026
7d3d04c
feat: set skip_store setting to events bus
Nexisato Sep 10, 2026
61dff82
fix(test): path error on windows system
Nexisato Sep 10, 2026
64a6339
feat: support skip in core_python & probe
Nexisato Sep 10, 2026
1e8fbf0
feat: support sender skip
Nexisato Sep 10, 2026
e7cd08a
chore: update greeting string
Nexisato Sep 10, 2026
e6569da
feat: skip debug dir
Nexisato Sep 10, 2026
1c80426
chore: update probe run_dir comments
Nexisato Sep 10, 2026
374b939
fix: support empty bytes content
Nexisato Sep 10, 2026
da7c9d6
chore: adjust text color
Nexisato Sep 11, 2026
8b0f331
refactor: skip parser error files
Nexisato Sep 11, 2026
ca6e42d
test(benchmark): add skip test
Nexisato Sep 11, 2026
1a36b99
chore: add media and save files
Nexisato Sep 11, 2026
f7f6b21
test(becnmark): add e2e log test
Nexisato Sep 11, 2026
8a26891
chore: remove unused parameter
Nexisato Sep 11, 2026
0468272
feat(proto): add optional payload
Nexisato Sep 12, 2026
5daa90e
refactor: simplify writer code
Nexisato Sep 12, 2026
742283c
fix: add on_moved event listener
Nexisato Sep 14, 2026
742f85d
fix: debounce writer
Nexisato Sep 14, 2026
885f4cf
chore: update default debounce delay
Nexisato Sep 14, 2026
48441fc
refactor: sender save helper
Nexisato Sep 14, 2026
db2d9ca
Merge branch 'main' into feat/cloud-only
SAKURA-CAT Oct 5, 2026
7780290
Refactor init directory handling and regenerate Go proto
SAKURA-CAT Oct 5, 2026
216df7e
fix(watcher): include st_ino in file signature
Nexisato Oct 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 20 additions & 7 deletions core/proto/swanlab/metric/data/v1/data.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

44 changes: 32 additions & 12 deletions core/proto/swanlab/save/v1/save.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

20 changes: 16 additions & 4 deletions core/proto/swanlab/settings/core/v1/core.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

16 changes: 13 additions & 3 deletions core/proto/swanlab/settings/probe/v1/probe.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 2 additions & 0 deletions protos/swanlab/metric/data/v1/data.proto
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ message MediaItem {
string sha256 = 2;
int64 size = 3; // 文件大小(bytes)
string caption = 4;
// 媒体内容,仅 online + skip_store=true 时写入
optional bytes payload = 5;
}

message ScalarRecord {
Expand Down
7 changes: 6 additions & 1 deletion protos/swanlab/save/v1/save.proto
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,17 @@ enum SaveType {
}

// 文件保存记录,由 swanlab.save() API 产生。
// 每条记录对应一个用户文件的保存操作。
// size/md5/mime_type 不在此处存储,由 Core 在持久化/上传时按需计算。
//
// payload 仅用于 online + skip_store 下的四类内部 save
// (CONFIG/METADATA/REQUIREMENTS/CONDA):此时 source_path/target_path 为空,
// Core 直接从 payload 解析并上传;CUSTOM 的 payload 必须为空,只使用 source_path;
// 未启用 skip_store 时 payload 为空,Core 按 source_path 处理。
message SaveRecord {
string name = 1; // 文件标识(相对于 base_path 的路径)
string source_path = 2; // 源文件绝对路径
string target_path = 3; // 本地镜像目标路径(软链接位置,由 Core 填充)
SavePolicy policy = 4; // 保存策略
SaveType type = 5; // 文件类型
optional bytes payload = 6; // 内部 metadata 元文件内容,skip_store 设置下有效
}
2 changes: 2 additions & 0 deletions protos/swanlab/settings/core/v1/core.proto
Original file line number Diff line number Diff line change
Expand Up @@ -15,4 +15,6 @@ message CoreSettings {
int64 save_split = 7;
int64 save_part = 8;
int32 save_batch = 9;
// Skip the local Record store (run-*.swanlab). Only valid in online mode.
bool skip_store = 10;
}
3 changes: 2 additions & 1 deletion protos/swanlab/settings/probe/v1/probe.proto
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ option go_package = "github.com/swanhubx/swanlab/core/proto/swanlab/settings/pro

message ProbeSettings {
optional string run_id = 1; // probe 侧需要校验 run_id 是否存在,不存在拒绝上报
optional string run_dir = 2; // probe 侧需要校验 run_dir 是否存在,不存在拒绝上报
optional string run_dir = 2; // 运行目录
optional int64 global_system_step = 3; // probe 侧需要校验 global_system_step 是否存在,不存在拒绝上报
bool hardware = 4;
bool runtime = 5;
Expand All @@ -17,4 +17,5 @@ message ProbeSettings {
bool monitor = 10;
int32 monitor_interval = 11;
string monitor_disk_dir = 12;
bool skip_store = 13; // 跳过探针本地文件落盘,内容直接内联 SaveRecord.payload(仅 online 合法)
}
12 changes: 6 additions & 6 deletions swanlab/proto/swanlab/metric/data/v1/data_pb2.py

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

6 changes: 4 additions & 2 deletions swanlab/proto/swanlab/metric/data/v1/data_pb2.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -23,16 +23,18 @@ class MediaValue(_message.Message):
def __init__(self, items: _Optional[_Iterable[_Union[MediaItem, _Mapping]]] = ...) -> None: ...

class MediaItem(_message.Message):
__slots__ = ("filename", "sha256", "size", "caption")
__slots__ = ("filename", "sha256", "size", "caption", "payload")
FILENAME_FIELD_NUMBER: _ClassVar[int]
SHA256_FIELD_NUMBER: _ClassVar[int]
SIZE_FIELD_NUMBER: _ClassVar[int]
CAPTION_FIELD_NUMBER: _ClassVar[int]
PAYLOAD_FIELD_NUMBER: _ClassVar[int]
filename: str
sha256: str
size: int
caption: str
def __init__(self, filename: _Optional[str] = ..., sha256: _Optional[str] = ..., size: _Optional[int] = ..., caption: _Optional[str] = ...) -> None: ...
payload: bytes
def __init__(self, filename: _Optional[str] = ..., sha256: _Optional[str] = ..., size: _Optional[int] = ..., caption: _Optional[str] = ..., payload: _Optional[bytes] = ...) -> None: ...

class ScalarRecord(_message.Message):
__slots__ = ("key", "step", "type", "timestamp", "value")
Expand Down
Loading
Loading