MarisUK commited on
Commit
913be98
·
verified ·
1 Parent(s): c088728

GitHub Actions deploy 7827e5068c351f4ea0e9b17f832620d2f52141a9

Browse files
README.md CHANGED
@@ -45,3 +45,16 @@ Nepieciešams:
45
  3. dataset repo artefaktu publicēšanai;
46
  4. model repo, kur publicēt treniņa rezultātu;
47
  5. ja paredzēts īsts train darbs, Space runtime ar pietiekamiem resursiem.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  3. dataset repo artefaktu publicēšanai;
46
  4. model repo, kur publicēt treniņa rezultātu;
47
  5. ja paredzēts īsts train darbs, Space runtime ar pietiekamiem resursiem.
48
+
49
+ ## Automātisks treniņa starts
50
+
51
+ Ja šo Space izmanto kā vienkāršu persistent training runneri, vari ieslēgt:
52
+
53
+ - `MARIS_HUMAN_TRAINING_AUTO_TRAIN=true`
54
+ - pēc vajadzības `MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_PRESET` vai `MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_NAME`
55
+ - pēc vajadzības `MARIS_HUMAN_TRAINING_AUTO_TRAIN_OUTPUT_SUBDIR`
56
+
57
+ Auto-starts palaiž `train-hf.sh` ar Space-safe `huggingface/training-config.json`
58
+ konfigurāciju un pēc restarta nelaiž otru identisku skrējienu, ja output
59
+ direktorijā jau ir pabeigta treniņa artefakti. Ja vajag apzināti pārrakstīt vai
60
+ turpināt virs esoša output, iestati `MARIS_HUMAN_TRAINING_AUTO_TRAIN_FORCE=true`.
core-python/maris_core/training/space_ui.py CHANGED
@@ -30,6 +30,13 @@ MARIS_PROGRESS_EVENT_KEY = "maris_training_event"
30
  # Keep a larger rolling event window than persisted run history because live status
31
  # parsing needs several recent progress/save/eval events from the current log tail.
32
  MAX_STRUCTURED_EVENTS = 64
 
 
 
 
 
 
 
33
 
34
 
35
  def _validate_repo_id(value: str) -> str:
@@ -158,7 +165,9 @@ def resolve_optional_persistent_path(persistent_dir: str, path_value: str) -> Pa
158
  root = Path(persistent_dir).expanduser().resolve()
159
  target = (root / normalized).resolve()
160
  if os.path.commonpath([str(root), str(target)]) != str(root):
161
- raise ValueError("Continue modeļa direktorijai jāatrodas Maris persistent storage ietvaros.")
 
 
162
  return target
163
 
164
 
@@ -181,13 +190,26 @@ def build_space_training_env(
181
  ) -> dict[str, str]:
182
  """Sagatavo vidi Maris treniņa procesam."""
183
  output_dir = resolve_output_dir(persistent_dir, request.output_subdir)
184
- continue_model_dir = resolve_optional_persistent_path(persistent_dir, request.continue_model_path)
 
 
 
 
 
 
 
 
 
 
 
 
185
  env = dict(base_env)
186
  env.update(
187
  {
188
  "MARIS_PERSISTENT_DIR": persistent_dir,
189
  "MARIS_MEMORY_REPO": request.dataset_repo,
190
  "MARIS_MODEL_REPO": request.hub_model_id,
 
191
  "MARIS_TRAIN_NUM_EPOCHS": str(request.num_epochs),
192
  "MARIS_TRAIN_PUBLISH": "true" if request.push_to_hub else "false",
193
  "MARIS_TRAIN_OUTPUT_DIR": str(output_dir),
@@ -198,6 +220,7 @@ def build_space_training_env(
198
  "HF_PERSISTENT_DIR": persistent_dir,
199
  "HF_DATASET_REPO": request.dataset_repo,
200
  "HF_MODEL_REPO": request.hub_model_id,
 
201
  "HF_TRAIN_NUM_EPOCHS": str(request.num_epochs),
202
  "HF_TRAIN_PUSH_TO_HUB": "true" if request.push_to_hub else "false",
203
  "HF_TRAIN_OUTPUT_DIR": str(output_dir),
@@ -231,6 +254,13 @@ def build_space_training_env(
231
  return env
232
 
233
 
 
 
 
 
 
 
 
234
  def tail_log(log_path: str | Path, *, max_chars: int = 16000) -> str:
235
  """Atgriež loga beigas UI vajadzībām."""
236
  path = Path(log_path)
@@ -321,9 +351,7 @@ def parse_training_progress(
321
 
322
  if learning_rate is None:
323
  learning_rate_matches = list(LEARNING_RATE_RE.finditer(log_text))
324
- learning_rate = (
325
- float(learning_rate_matches[-1].group(1)) if learning_rate_matches else None
326
- )
327
 
328
  percent = 0
329
  stage = structured_stage or "queued"
@@ -360,10 +388,7 @@ def parse_training_progress(
360
  stage = structured_stage or "training"
361
  progress_ratio = min(current_step / max(total_steps, 1), 1.0)
362
  percent = min(95, max(35, int(35 + progress_ratio * 55)))
363
- label = (
364
- structured_label
365
- or f"Trenē modeli · solis {current_step}/{total_steps}"
366
- )
367
  elif current_epoch is not None:
368
  stage = structured_stage or "training"
369
  epoch_total = detected_total_epochs or total_epochs
 
30
  # Keep a larger rolling event window than persisted run history because live status
31
  # parsing needs several recent progress/save/eval events from the current log tail.
32
  MAX_STRUCTURED_EVENTS = 64
33
+ SPACE_TRAINING_CONFIG_PATH_DEFAULT = "huggingface/training-config.json"
34
+ SPACE_TRAINING_COMPLETION_MARKERS = (
35
+ "training-metrics.json",
36
+ "trainer_state.json",
37
+ "training-provenance.json",
38
+ "branch-suite.json",
39
+ )
40
 
41
 
42
  def _validate_repo_id(value: str) -> str:
 
165
  root = Path(persistent_dir).expanduser().resolve()
166
  target = (root / normalized).resolve()
167
  if os.path.commonpath([str(root), str(target)]) != str(root):
168
+ raise ValueError(
169
+ "Continue modeļa direktorijai jāatrodas Maris persistent storage ietvaros."
170
+ )
171
  return target
172
 
173
 
 
190
  ) -> dict[str, str]:
191
  """Sagatavo vidi Maris treniņa procesam."""
192
  output_dir = resolve_output_dir(persistent_dir, request.output_subdir)
193
+ continue_model_dir = resolve_optional_persistent_path(
194
+ persistent_dir, request.continue_model_path
195
+ )
196
+ config_path = (
197
+ str(
198
+ base_env.get("MARIS_SPACE_TRAIN_CONFIG_PATH")
199
+ or base_env.get("HF_SPACE_TRAINING_CONFIG_PATH")
200
+ or base_env.get("MARIS_TRAIN_CONFIG_PATH")
201
+ or base_env.get("HF_TRAINING_CONFIG_PATH")
202
+ or SPACE_TRAINING_CONFIG_PATH_DEFAULT
203
+ ).strip()
204
+ or SPACE_TRAINING_CONFIG_PATH_DEFAULT
205
+ )
206
  env = dict(base_env)
207
  env.update(
208
  {
209
  "MARIS_PERSISTENT_DIR": persistent_dir,
210
  "MARIS_MEMORY_REPO": request.dataset_repo,
211
  "MARIS_MODEL_REPO": request.hub_model_id,
212
+ "MARIS_TRAIN_CONFIG_PATH": config_path,
213
  "MARIS_TRAIN_NUM_EPOCHS": str(request.num_epochs),
214
  "MARIS_TRAIN_PUBLISH": "true" if request.push_to_hub else "false",
215
  "MARIS_TRAIN_OUTPUT_DIR": str(output_dir),
 
220
  "HF_PERSISTENT_DIR": persistent_dir,
221
  "HF_DATASET_REPO": request.dataset_repo,
222
  "HF_MODEL_REPO": request.hub_model_id,
223
+ "HF_TRAINING_CONFIG_PATH": config_path,
224
  "HF_TRAIN_NUM_EPOCHS": str(request.num_epochs),
225
  "HF_TRAIN_PUSH_TO_HUB": "true" if request.push_to_hub else "false",
226
  "HF_TRAIN_OUTPUT_DIR": str(output_dir),
 
254
  return env
255
 
256
 
257
+ def has_completed_training_artifacts(output_dir: Path) -> bool:
258
+ """Nosaka, vai Space output direktorijā jau ir pabeigta treniņa artefakti."""
259
+ return any(
260
+ output_dir.joinpath(marker).is_file() for marker in SPACE_TRAINING_COMPLETION_MARKERS
261
+ )
262
+
263
+
264
  def tail_log(log_path: str | Path, *, max_chars: int = 16000) -> str:
265
  """Atgriež loga beigas UI vajadzībām."""
266
  path = Path(log_path)
 
351
 
352
  if learning_rate is None:
353
  learning_rate_matches = list(LEARNING_RATE_RE.finditer(log_text))
354
+ learning_rate = float(learning_rate_matches[-1].group(1)) if learning_rate_matches else None
 
 
355
 
356
  percent = 0
357
  stage = structured_stage or "queued"
 
388
  stage = structured_stage or "training"
389
  progress_ratio = min(current_step / max(total_steps, 1), 1.0)
390
  percent = min(95, max(35, int(35 + progress_ratio * 55)))
391
+ label = structured_label or f"Trenē modeli · solis {current_step}/{total_steps}"
 
 
 
392
  elif current_epoch is not None:
393
  stage = structured_stage or "training"
394
  epoch_total = detected_total_epochs or total_epochs
core-python/tests/test_huggingface_human_training_space.py CHANGED
@@ -50,10 +50,42 @@ def test_runtime_endpoint_exposes_roles_and_docs() -> None:
50
  assert set(body["roles"]) == {"owner", "secretary", "trainee", "user"}
51
  assert any(item["title"] == "Onboarding guide" for item in body["documentation"])
52
  assert body["training"]["defaults"]["hub_model_id"] == "MarisUK/maris-ai-lv"
 
 
 
53
  assert body["auth_required"] is False
54
  assert body["private_session"]["user"]["role"] == "owner"
55
 
56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  def test_register_and_login_flow(monkeypatch, tmp_path: Path) -> None:
58
  client = TestClient(human_training_space_app.app)
59
  monkeypatch.setattr(
 
50
  assert set(body["roles"]) == {"owner", "secretary", "trainee", "user"}
51
  assert any(item["title"] == "Onboarding guide" for item in body["documentation"])
52
  assert body["training"]["defaults"]["hub_model_id"] == "MarisUK/maris-ai-lv"
53
+ assert isinstance(body["training"]["model_choices"], list)
54
+ assert body["training"]["model_choices"][0]["id"]
55
+ assert body["training"]["model_choices"][0]["label"]
56
  assert body["auth_required"] is False
57
  assert body["private_session"]["user"]["role"] == "owner"
58
 
59
 
60
+ def test_maybe_start_automatic_training_starts_with_human_space_defaults(
61
+ monkeypatch, tmp_path: Path
62
+ ) -> None:
63
+ calls: list[dict[str, object]] = []
64
+
65
+ monkeypatch.setenv("MARIS_HUMAN_TRAINING_AUTO_TRAIN", "true")
66
+ monkeypatch.setattr(human_training_space_app, "PERSISTENT_DIR", tmp_path)
67
+ monkeypatch.setattr(
68
+ human_training_space_app,
69
+ "has_completed_training_artifacts",
70
+ lambda output_dir: False,
71
+ )
72
+ monkeypatch.setattr(
73
+ human_training_space_app,
74
+ "_start_training_process",
75
+ lambda request: (
76
+ calls.append(request.model_dump()) or {"pid": 77, "log_path": "/tmp/human-train.log"}
77
+ ),
78
+ )
79
+
80
+ human_training_space_app._maybe_start_automatic_training()
81
+
82
+ assert len(calls) == 1
83
+ assert calls[0]["dataset_repo"] == human_training_space_app.DEFAULT_DATASET_REPO
84
+ assert calls[0]["model_repo"] == human_training_space_app.DEFAULT_HUB_MODEL_ID
85
+ assert calls[0]["model_preset"] == "balanced"
86
+ assert calls[0]["continue_from_latest_artifact"] is True
87
+
88
+
89
  def test_register_and_login_flow(monkeypatch, tmp_path: Path) -> None:
90
  client = TestClient(human_training_space_app.app)
91
  monkeypatch.setattr(
core-python/tests/test_huggingface_space_app.py CHANGED
@@ -54,6 +54,46 @@ def test_status_endpoint_includes_progress_metadata() -> None:
54
  assert "history" in body
55
 
56
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  def test_index_endpoint_defaults_to_balanced_preset() -> None:
58
  client = TestClient(space_app.app)
59
 
@@ -165,7 +205,9 @@ def test_websocket_sends_initial_snapshot(tmp_path: Path) -> None:
165
  assert "hello from log" in message["log_tail"]
166
 
167
 
168
- def test_status_endpoint_persists_training_history_and_artifacts(monkeypatch, tmp_path: Path) -> None:
 
 
169
  client = TestClient(space_app.app)
170
  output_dir = tmp_path / "runs" / "demo"
171
  output_dir.mkdir(parents=True)
@@ -895,7 +937,7 @@ def test_workspace_command_runner_stops_when_task_is_cancelled(tmp_path: Path) -
895
  cancel_event.set()
896
 
897
  try:
898
- runner({"command": "python -c \"import time; time.sleep(5)\""})
899
  except space_app.SpaceAgentCancelledError as exc:
900
  assert "req-cancel" in str(exc)
901
  assert "task-cancel" in str(exc)
 
54
  assert "history" in body
55
 
56
 
57
+ def test_maybe_start_automatic_training_starts_with_space_defaults(
58
+ monkeypatch, tmp_path: Path
59
+ ) -> None:
60
+ calls: list[dict[str, object]] = []
61
+
62
+ monkeypatch.setenv("MARIS_SPACE_AUTO_TRAIN", "true")
63
+ monkeypatch.setattr(space_app, "PERSISTENT_DIR", str(tmp_path))
64
+ monkeypatch.setattr(space_app, "has_completed_training_artifacts", lambda output_dir: False)
65
+ monkeypatch.setattr(
66
+ space_app,
67
+ "_start_training_process",
68
+ lambda request: (
69
+ calls.append(request.model_dump()) or {"pid": 99, "log_path": "/tmp/train.log"}
70
+ ),
71
+ )
72
+
73
+ space_app._maybe_start_automatic_training()
74
+
75
+ assert len(calls) == 1
76
+ assert calls[0]["dataset_repo"] == space_app.AGENT_RUNTIME.dataset_repo
77
+ assert calls[0]["model_repo"] == space_app.AGENT_RUNTIME.model_repo
78
+ assert calls[0]["model_preset"] == "balanced"
79
+ assert calls[0]["continue_from_latest_artifact"] is True
80
+
81
+
82
+ def test_maybe_start_automatic_training_skips_when_completed_artifacts_exist(
83
+ monkeypatch, tmp_path: Path
84
+ ) -> None:
85
+ monkeypatch.setenv("MARIS_SPACE_AUTO_TRAIN", "true")
86
+ monkeypatch.setattr(space_app, "PERSISTENT_DIR", str(tmp_path))
87
+ monkeypatch.setattr(space_app, "has_completed_training_artifacts", lambda output_dir: True)
88
+ monkeypatch.setattr(
89
+ space_app,
90
+ "_start_training_process",
91
+ lambda request: (_ for _ in ()).throw(AssertionError("auto training should be skipped")),
92
+ )
93
+
94
+ space_app._maybe_start_automatic_training()
95
+
96
+
97
  def test_index_endpoint_defaults_to_balanced_preset() -> None:
98
  client = TestClient(space_app.app)
99
 
 
205
  assert "hello from log" in message["log_tail"]
206
 
207
 
208
+ def test_status_endpoint_persists_training_history_and_artifacts(
209
+ monkeypatch, tmp_path: Path
210
+ ) -> None:
211
  client = TestClient(space_app.app)
212
  output_dir = tmp_path / "runs" / "demo"
213
  output_dir.mkdir(parents=True)
 
937
  cancel_event.set()
938
 
939
  try:
940
+ runner({"command": 'python -c "import time; time.sleep(5)"'})
941
  except space_app.SpaceAgentCancelledError as exc:
942
  assert "req-cancel" in str(exc)
943
  assert "task-cancel" in str(exc)
core-python/tests/test_space_ui.py CHANGED
@@ -10,6 +10,7 @@ from maris_core.training.space_ui import (
10
  SpaceTrainingRequest,
11
  build_space_training_command,
12
  build_space_training_env,
 
13
  list_space_model_choices,
14
  parse_training_progress,
15
  read_log_since,
@@ -78,6 +79,8 @@ def test_build_space_training_env_uses_preset_and_persistent_storage(tmp_path: P
78
  assert env["HF_LOCAL_MODEL_DIR"] == str(tmp_path / "runs" / "coder")
79
  assert env["HF_MODEL_REPO"] == "MarisUK/maris-ai-lv"
80
  assert env["HF_TRAIN_MODEL_PRESET"] == "coding"
 
 
81
  assert env["HF_TRAIN_PUSH_TO_HUB"] == "false"
82
  assert env["HF_TRAIN_CONTINUE_FROM_LATEST"] == "true"
83
  assert env["HF_TRAIN_CONTINUE_MODEL_PATH"] == str(tmp_path / "runs" / "checkpoints")
@@ -106,6 +109,30 @@ def test_build_space_training_env_clears_inherited_distributed_overrides(tmp_pat
106
  assert "MARIS_TRAIN_DISTRIBUTED_CONFIG_PATH" not in env
107
 
108
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
109
  def test_list_space_model_choices_exposes_presets() -> None:
110
  choices = list_space_model_choices()
111
 
@@ -192,7 +219,7 @@ def test_parse_training_progress_reports_completion() -> None:
192
 
193
  def test_parse_training_progress_prefers_structured_events() -> None:
194
  progress = parse_training_progress(
195
- '\n'.join(
196
  [
197
  '{"maris_training_event": true, "event": "log", "stage": "training", "label": "Trenē modeli · solis 12/40", "epoch": 1.5, "total_epochs": 4, "step": 12, "total_steps": 40, "loss": 0.2451, "eval_loss": 0.1987, "learning_rate": 0.0002, "eta_seconds": 180}',
198
  "Epoch 1/4",
 
10
  SpaceTrainingRequest,
11
  build_space_training_command,
12
  build_space_training_env,
13
+ has_completed_training_artifacts,
14
  list_space_model_choices,
15
  parse_training_progress,
16
  read_log_since,
 
79
  assert env["HF_LOCAL_MODEL_DIR"] == str(tmp_path / "runs" / "coder")
80
  assert env["HF_MODEL_REPO"] == "MarisUK/maris-ai-lv"
81
  assert env["HF_TRAIN_MODEL_PRESET"] == "coding"
82
+ assert env["HF_TRAINING_CONFIG_PATH"] == "huggingface/training-config.json"
83
+ assert env["MARIS_TRAIN_CONFIG_PATH"] == "huggingface/training-config.json"
84
  assert env["HF_TRAIN_PUSH_TO_HUB"] == "false"
85
  assert env["HF_TRAIN_CONTINUE_FROM_LATEST"] == "true"
86
  assert env["HF_TRAIN_CONTINUE_MODEL_PATH"] == str(tmp_path / "runs" / "checkpoints")
 
109
  assert "MARIS_TRAIN_DISTRIBUTED_CONFIG_PATH" not in env
110
 
111
 
112
+ def test_build_space_training_env_allows_explicit_space_config_override(tmp_path: Path) -> None:
113
+ request = SpaceTrainingRequest(model_preset="balanced")
114
+
115
+ env = build_space_training_env(
116
+ {"MARIS_SPACE_TRAIN_CONFIG_PATH": "huggingface/custom-space-config.json"},
117
+ request,
118
+ str(tmp_path),
119
+ )
120
+
121
+ assert env["HF_TRAINING_CONFIG_PATH"] == "huggingface/custom-space-config.json"
122
+ assert env["MARIS_TRAIN_CONFIG_PATH"] == "huggingface/custom-space-config.json"
123
+
124
+
125
+ def test_has_completed_training_artifacts_detects_finished_space_run(tmp_path: Path) -> None:
126
+ output_dir = tmp_path / "runs" / "demo"
127
+ output_dir.mkdir(parents=True)
128
+
129
+ assert has_completed_training_artifacts(output_dir) is False
130
+
131
+ (output_dir / "training-metrics.json").write_text("{}", encoding="utf-8")
132
+
133
+ assert has_completed_training_artifacts(output_dir) is True
134
+
135
+
136
  def test_list_space_model_choices_exposes_presets() -> None:
137
  choices = list_space_model_choices()
138
 
 
219
 
220
  def test_parse_training_progress_prefers_structured_events() -> None:
221
  progress = parse_training_progress(
222
+ "\n".join(
223
  [
224
  '{"maris_training_event": true, "event": "log", "stage": "training", "label": "Trenē modeli · solis 12/40", "epoch": 1.5, "total_epochs": 4, "step": 12, "total_steps": 40, "loss": 0.2451, "eval_loss": 0.1987, "learning_rate": 0.0002, "eta_seconds": 180}',
225
  "Epoch 1/4",
huggingface_human_training_space/README.md CHANGED
@@ -45,3 +45,16 @@ Nepieciešams:
45
  3. dataset repo artefaktu publicēšanai;
46
  4. model repo, kur publicēt treniņa rezultātu;
47
  5. ja paredzēts īsts train darbs, Space runtime ar pietiekamiem resursiem.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
45
  3. dataset repo artefaktu publicēšanai;
46
  4. model repo, kur publicēt treniņa rezultātu;
47
  5. ja paredzēts īsts train darbs, Space runtime ar pietiekamiem resursiem.
48
+
49
+ ## Automātisks treniņa starts
50
+
51
+ Ja šo Space izmanto kā vienkāršu persistent training runneri, vari ieslēgt:
52
+
53
+ - `MARIS_HUMAN_TRAINING_AUTO_TRAIN=true`
54
+ - pēc vajadzības `MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_PRESET` vai `MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_NAME`
55
+ - pēc vajadzības `MARIS_HUMAN_TRAINING_AUTO_TRAIN_OUTPUT_SUBDIR`
56
+
57
+ Auto-starts palaiž `train-hf.sh` ar Space-safe `huggingface/training-config.json`
58
+ konfigurāciju un pēc restarta nelaiž otru identisku skrējienu, ja output
59
+ direktorijā jau ir pabeigta treniņa artefakti. Ja vajag apzināti pārrakstīt vai
60
+ turpināt virs esoša output, iestati `MARIS_HUMAN_TRAINING_AUTO_TRAIN_FORCE=true`.
huggingface_human_training_space/app.py CHANGED
@@ -34,10 +34,13 @@ from maris_core.training.human_training import ( # noqa: E402
34
  stage_human_training_artifacts,
35
  )
36
  from maris_core.training.space_ui import ( # noqa: E402
 
37
  build_space_training_command,
38
  build_space_training_env,
 
39
  list_space_model_choices,
40
  parse_training_progress,
 
41
  tail_log,
42
  terminate_process_tree,
43
  )
@@ -278,6 +281,27 @@ def _timestamp() -> str:
278
  return datetime.now(UTC).replace(microsecond=0).isoformat()
279
 
280
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
281
  def _has_publish_token() -> bool:
282
  return bool(get_hf_token())
283
 
@@ -476,13 +500,133 @@ def _training_defaults() -> dict[str, Any]:
476
 
477
 
478
  def _training_runtime_payload() -> dict[str, Any]:
 
 
 
 
 
 
 
 
479
  return {
480
- "model_choices": list_space_model_choices(),
481
  "has_publish_token": _has_publish_token(),
482
  "defaults": _training_defaults(),
483
  }
484
 
485
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
486
  def _start_training_process(request: Any) -> dict[str, Any]:
487
  if bool(getattr(request, "push_to_hub", False)) and not _has_publish_token():
488
  raise HTTPException(
 
34
  stage_human_training_artifacts,
35
  )
36
  from maris_core.training.space_ui import ( # noqa: E402
37
+ SpaceTrainingRequest,
38
  build_space_training_command,
39
  build_space_training_env,
40
+ has_completed_training_artifacts,
41
  list_space_model_choices,
42
  parse_training_progress,
43
+ resolve_output_dir,
44
  tail_log,
45
  terminate_process_tree,
46
  )
 
281
  return datetime.now(UTC).replace(microsecond=0).isoformat()
282
 
283
 
284
+ def _env_flag(*names: str, default: bool = False) -> bool:
285
+ return get_env_any_or_default(
286
+ *names, default="true" if default else "false"
287
+ ).strip().lower() in {
288
+ "1",
289
+ "true",
290
+ "yes",
291
+ "on",
292
+ }
293
+
294
+
295
+ def _env_int(*names: str, default: int) -> int:
296
+ value = get_env_any_or_default(*names, default=str(default)).strip()
297
+ try:
298
+ return int(value)
299
+ except ValueError as exc:
300
+ raise RuntimeError(
301
+ f"Nederīga vesela skaitļa vērtība env laukam {'/'.join(names)}: {value}"
302
+ ) from exc
303
+
304
+
305
  def _has_publish_token() -> bool:
306
  return bool(get_hf_token())
307
 
 
500
 
501
 
502
  def _training_runtime_payload() -> dict[str, Any]:
503
+ model_choices = [
504
+ {
505
+ "id": preset_id,
506
+ "label": f"{preset_id} — {config['model_name']}",
507
+ **config,
508
+ }
509
+ for preset_id, config in list_space_model_choices().items()
510
+ ]
511
  return {
512
+ "model_choices": model_choices,
513
  "has_publish_token": _has_publish_token(),
514
  "defaults": _training_defaults(),
515
  }
516
 
517
 
518
+ def _auto_training_request() -> SpaceTrainingRequest:
519
+ defaults = _training_defaults()
520
+ output_subdir = get_env_any_or_default(
521
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_OUTPUT_SUBDIR",
522
+ "MARIS_SPACE_AUTO_TRAIN_OUTPUT_SUBDIR",
523
+ default=str(defaults["output_subdir"]),
524
+ ).strip()
525
+ continue_model_path = get_env_any_or_default(
526
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_CONTINUE_MODEL_PATH",
527
+ "MARIS_SPACE_AUTO_TRAIN_CONTINUE_MODEL_PATH",
528
+ "MARIS_TRAIN_CONTINUE_MODEL_PATH",
529
+ "HF_TRAIN_CONTINUE_MODEL_PATH",
530
+ default=output_subdir,
531
+ ).strip()
532
+ model_name = get_env_any_or_default(
533
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_NAME",
534
+ "MARIS_SPACE_AUTO_TRAIN_MODEL_NAME",
535
+ "MARIS_TRAIN_BASE_MODEL",
536
+ "HF_TRAIN_BASE_MODEL",
537
+ default="",
538
+ ).strip()
539
+ model_preset = get_env_any_or_default(
540
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_PRESET",
541
+ "MARIS_SPACE_AUTO_TRAIN_MODEL_PRESET",
542
+ "MARIS_TRAIN_MODEL_PRESET",
543
+ "HF_TRAIN_MODEL_PRESET",
544
+ default=str(defaults["model_preset"]),
545
+ ).strip()
546
+ return SpaceTrainingRequest(
547
+ dataset_repo=get_env_any_or_default(
548
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_DATASET_REPO",
549
+ "MARIS_MEMORY_REPO",
550
+ "MARIS_DATASET_REPO",
551
+ "HF_DATASET_REPO",
552
+ default=str(defaults["dataset_repo"]),
553
+ ),
554
+ model_repo=get_env_any_or_default(
555
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_MODEL_REPO",
556
+ "MARIS_HUMAN_TRAINING_MODEL_REPO",
557
+ "MARIS_MODEL_REPO",
558
+ "HF_MODEL_REPO",
559
+ default=str(defaults["hub_model_id"]),
560
+ ),
561
+ model_preset="" if model_name else model_preset,
562
+ model_name=model_name,
563
+ num_epochs=_env_int(
564
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_NUM_EPOCHS",
565
+ "MARIS_TRAIN_NUM_EPOCHS",
566
+ "HF_TRAIN_NUM_EPOCHS",
567
+ default=int(defaults["num_epochs"]),
568
+ ),
569
+ all_branches=_env_flag(
570
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_ALL_BRANCHES",
571
+ "MARIS_SPACE_AUTO_TRAIN_ALL_BRANCHES",
572
+ default=bool(defaults["all_branches"]),
573
+ ),
574
+ push_to_hub=_env_flag(
575
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_PUSH_TO_HUB",
576
+ "MARIS_SPACE_AUTO_TRAIN_PUSH_TO_HUB",
577
+ "MARIS_TRAIN_PUBLISH",
578
+ "HF_TRAIN_PUSH_TO_HUB",
579
+ default=bool(defaults["push_to_hub"]),
580
+ ),
581
+ output_subdir=output_subdir,
582
+ continue_from_latest_artifact=_env_flag(
583
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_CONTINUE_FROM_LATEST",
584
+ "MARIS_SPACE_AUTO_TRAIN_CONTINUE_FROM_LATEST",
585
+ "MARIS_TRAIN_CONTINUE_FROM_LATEST",
586
+ "HF_TRAIN_CONTINUE_FROM_LATEST",
587
+ default=bool(defaults["continue_from_latest_artifact"]),
588
+ ),
589
+ continue_model_path=continue_model_path,
590
+ )
591
+
592
+
593
+ def _maybe_start_automatic_training() -> None:
594
+ if not _env_flag(
595
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN",
596
+ "MARIS_SPACE_AUTO_TRAIN",
597
+ "HF_SPACE_AUTO_TRAIN",
598
+ ):
599
+ return
600
+ try:
601
+ request = _auto_training_request()
602
+ output_dir = resolve_output_dir(str(PERSISTENT_DIR), request.output_subdir)
603
+ force_start = _env_flag(
604
+ "MARIS_HUMAN_TRAINING_AUTO_TRAIN_FORCE",
605
+ "MARIS_SPACE_AUTO_TRAIN_FORCE",
606
+ "HF_SPACE_AUTO_TRAIN_FORCE",
607
+ default=False,
608
+ )
609
+ if not force_start and has_completed_training_artifacts(output_dir):
610
+ logger.info(
611
+ "Izlaižu human training Space auto-startu, jo output jau satur pabeigta skrējiena artefaktus: %s",
612
+ output_dir,
613
+ )
614
+ return
615
+ result = _start_training_process(request)
616
+ logger.info(
617
+ "Human training Space automātiskais treniņš palaists: pid=%s log=%s",
618
+ result["pid"],
619
+ result["log_path"],
620
+ )
621
+ except Exception: # noqa: BLE001
622
+ logger.exception("Neizdevās automātiski palaist human training Space treniņu starta laikā.")
623
+
624
+
625
+ @APP.on_event("startup")
626
+ def _startup_auto_training() -> None:
627
+ _maybe_start_automatic_training()
628
+
629
+
630
  def _start_training_process(request: Any) -> dict[str, Any]:
631
  if bool(getattr(request, "push_to_hub", False)) and not _has_publish_token():
632
  raise HTTPException(