feat: add HF Jobs-based data refresh flow

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.py[cod]

README.md

@@ -36,4 +36,8 @@ short_description: 米国市場の終値から翌日の日本市場業種ETFの
 
 ## 参考文献
 
-中川ら (2026) 「部分空間正則化付き主成分分析を用いた日米業種リードラグ投資戦略」, SIG-FIN-036
+中川ら (2026) 「部分空間正則化付き主成分分析を用いた日米業種リードラグ投資戦略」, SIG-FIN-036
+
+## Maintenance
+
+Scheduled refresh setup is documented in `docs/hf-jobs-setup.md`.

app.py

@@ -278,26 +278,12 @@ st.sidebar.caption(
 # ---------------------------------------------------------------------------
 
 if not _data_files_exist():
-    st.info("初回起動: データをダウンロード中...（1〜2分かかります）")
-    with st.spinner("データ取得中..."):
-        try:
-            from src.data.fetch_jp_etf import fetch_jp_etf
-            from src.data.fetch_us_etf import fetch_us_etf
-            from src.data.fetch_ff_factors import fetch_ff_factors
-            from src.data.preprocess import preprocess_returns
-            from src.data.build_calendar import build_calendar
-
-            fetch_jp_etf()
-            fetch_us_etf()
-            fetch_ff_factors()
-            us_ret, jp_ret = preprocess_returns()
-            jp_cc = jp_ret.xs("cc", axis=1, level="ReturnType")
-            build_calendar(us_ret.index, jp_cc.index)
-            st.success("データ取得完了! ページを再読み込みします...")
-            st.rerun()
-        except Exception as e:
-            st.error(f"データ取得に失敗しました: {e}")
-            st.stop()
+    st.error(
+        "Required data files are missing. Run the scheduled HF update job "
+        "to regenerate data for this Space."
+    )
+    st.caption("Setup instructions: docs/hf-jobs-setup.md")
+    st.stop()
 
 # Load base data (cheap, cached)
 us_ret, jp_ret_full, date_map = load_data()

docs/hf-jobs-setup.md

@@ -0,0 +1,61 @@
+# HF Jobs Setup
+
+This Space stays read-only at runtime. Data refresh should happen through scheduled Hugging Face Jobs that update the Space repository and trigger a rebuild only when the data changed.
+
+## Prerequisites
+
+- Hugging Face account with Jobs access
+- `hf` CLI installed and logged in
+- a token with permission to push to `spaces/Kabutsurayuki/sector-leadlag`
+
+Official references:
+
+- Jobs overview: https://huggingface.co/docs/huggingface_hub/guides/jobs
+- Jobs CLI: https://huggingface.co/docs/huggingface_hub/guides/cli
+
+## Required secret
+
+Create the scheduled jobs with:
+
+- `--secrets HF_TOKEN`
+
+The CLI docs describe this flag as the way to inject local secrets into a Job environment.
+
+## Schedule
+
+The jobs are defined in UTC:
+
+- JP close refresh: `45 6 * * *` -> `15:45 JST`
+- US close refresh: `0 22 * * *` -> `07:00 JST`
+
+## Scheduled Job Commands
+
+These commands clone the Space repo with token-backed auth, install dependencies, run the update script, and push only when the data changed.
+
+### JP close job
+
+```bash
+hf jobs scheduled run "45 6 * * *" --secrets HF_TOKEN python:3.10 bash -lc "apt-get update && apt-get install -y git && git clone https://Kabutsurayuki:${HF_TOKEN}@huggingface.co/spaces/Kabutsurayuki/sector-leadlag repo && cd repo && pip install -r requirements.txt && python scripts/update_data.py --mode jp --commit --push"
+```
+
+### US close job
+
+```bash
+hf jobs scheduled run "0 22 * * *" --secrets HF_TOKEN python:3.10 bash -lc "apt-get update && apt-get install -y git && git clone https://Kabutsurayuki:${HF_TOKEN}@huggingface.co/spaces/Kabutsurayuki/sector-leadlag repo && cd repo && pip install -r requirements.txt && python scripts/update_data.py --mode us --commit --push"
+```
+
+## Expected behavior
+
+- If data refresh fails, the job exits non-zero and the current Space stays on the last good commit.
+- If the raw or processed CSV files did not change, the script exits successfully without committing or rebuilding the Space.
+- If data changed, the script commits only `data/raw` and `data/processed`, then pushes to `origin`.
+
+## Useful follow-up commands
+
+```bash
+hf jobs scheduled ps
+hf jobs scheduled inspect <scheduled_job_id>
+hf jobs scheduled suspend <scheduled_job_id>
+hf jobs scheduled resume <scheduled_job_id>
+hf jobs scheduled delete <scheduled_job_id>
+```

scripts/update_data.py

@@ -0,0 +1,244 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import subprocess
+import sys
+from pathlib import Path
+
+import pandas as pd
+
+
+LOGGER = logging.getLogger("update_data")
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description="Refresh market data and derived CSVs for the HF Space.",
+    )
+    parser.add_argument(
+        "--mode",
+        choices=("jp", "us", "all"),
+        required=True,
+        help="Which market data source to refresh before rebuilding derived files.",
+    )
+    parser.add_argument(
+        "--repo-dir",
+        type=Path,
+        default=Path(__file__).resolve().parents[1],
+        help="Path to the cloned Space repository.",
+    )
+    parser.add_argument(
+        "--commit",
+        action="store_true",
+        help="Create a git commit when data files changed.",
+    )
+    parser.add_argument(
+        "--push",
+        action="store_true",
+        help="Push the commit to origin. Requires --commit.",
+    )
+    parser.add_argument(
+        "--message",
+        help="Override the default git commit message.",
+    )
+    return parser.parse_args()
+
+
+def ensure_repo_root(repo_dir: Path) -> Path:
+    repo_root = repo_dir.resolve()
+    expected = (
+        repo_root / "app.py",
+        repo_root / "src",
+        repo_root / "data",
+    )
+    missing = [str(path) for path in expected if not path.exists()]
+    if missing:
+        raise FileNotFoundError(
+            f"Repository root looks incomplete: missing {', '.join(missing)}"
+        )
+    return repo_root
+
+
+def configure_import_path(repo_root: Path) -> None:
+    root_str = str(repo_root)
+    if root_str not in sys.path:
+        sys.path.insert(0, root_str)
+
+
+def refresh_market_data(mode: str) -> None:
+    from src.data.build_calendar import build_calendar
+    from src.data.fetch_jp_etf import fetch_jp_etf
+    from src.data.fetch_us_etf import fetch_us_etf
+    from src.data.preprocess import preprocess_returns
+
+    if mode in {"jp", "all"}:
+        LOGGER.info("Refreshing JP ETF data")
+        fetch_jp_etf()
+
+    if mode in {"us", "all"}:
+        LOGGER.info("Refreshing US ETF data")
+        fetch_us_etf()
+
+    LOGGER.info("Rebuilding processed returns")
+    us_returns, jp_returns = preprocess_returns()
+    jp_cc = jp_returns.xs("cc", axis=1, level="ReturnType")
+
+    LOGGER.info("Rebuilding US-JP trading calendar map")
+    build_calendar(us_returns.index, jp_cc.index)
+
+
+def validate_outputs(repo_root: Path) -> None:
+    # The Streamlit app expects these files and schemas to exist before startup.
+    raw_dir = repo_root / "data" / "raw"
+    processed_dir = repo_root / "data" / "processed"
+
+    required_raw = [
+        raw_dir / "jp_etf_ohlc.csv",
+        raw_dir / "us_etf_ohlc.csv",
+    ]
+    required_processed = [
+        processed_dir / "jp_returns.csv",
+        processed_dir / "us_returns.csv",
+        processed_dir / "us_jp_date_map.csv",
+    ]
+
+    for path in required_raw + required_processed:
+        if not path.exists():
+            raise FileNotFoundError(f"Required output file is missing: {path}")
+
+    us_returns = pd.read_csv(
+        processed_dir / "us_returns.csv",
+        index_col=0,
+        parse_dates=True,
+    )
+    jp_returns = pd.read_csv(
+        processed_dir / "jp_returns.csv",
+        header=[0, 1],
+        index_col=0,
+        parse_dates=True,
+    )
+    date_map = pd.read_csv(
+        processed_dir / "us_jp_date_map.csv",
+        parse_dates=["us_date", "jp_next_date"],
+    )
+
+    if us_returns.empty:
+        raise ValueError("us_returns.csv is empty")
+    if jp_returns.empty:
+        raise ValueError("jp_returns.csv is empty")
+    if date_map.empty:
+        raise ValueError("us_jp_date_map.csv is empty")
+
+    return_types = set(jp_returns.columns.get_level_values(1))
+    if {"cc", "oc"} - return_types:
+        raise ValueError("jp_returns.csv is missing expected return types")
+
+    if us_returns.index.isna().any():
+        raise ValueError("us_returns.csv contains null dates")
+    if jp_returns.index.isna().any():
+        raise ValueError("jp_returns.csv contains null dates")
+    if date_map[["us_date", "jp_next_date"]].isna().any().any():
+        raise ValueError("us_jp_date_map.csv contains null dates")
+
+    LOGGER.info(
+        "Validated outputs: latest US=%s latest JP=%s latest map=(%s -> %s)",
+        us_returns.index.max().date(),
+        jp_returns.index.max().date(),
+        date_map["us_date"].max().date(),
+        date_map["jp_next_date"].max().date(),
+    )
+
+
+def run_git(repo_root: Path, *args: str) -> subprocess.CompletedProcess[str]:
+    return subprocess.run(
+        ["git", *args],
+        cwd=repo_root,
+        check=True,
+        text=True,
+        capture_output=True,
+    )
+
+
+def read_git_config(repo_root: Path, key: str) -> str:
+    try:
+        result = run_git(repo_root, "config", "--get", key)
+    except subprocess.CalledProcessError:
+        return ""
+    return result.stdout.strip()
+
+
+def ensure_git_identity(repo_root: Path) -> None:
+    if not read_git_config(repo_root, "user.name"):
+        run_git(repo_root, "config", "user.name", "hf-jobs-bot")
+    if not read_git_config(repo_root, "user.email"):
+        run_git(
+            repo_root,
+            "config",
+            "user.email",
+            "hf-jobs-bot@users.noreply.huggingface.co",
+        )
+
+
+def data_diff_status(repo_root: Path) -> str:
+    result = run_git(repo_root, "status", "--short", "--", "data/raw", "data/processed")
+    return result.stdout.strip()
+
+
+def default_commit_message(mode: str) -> str:
+    if mode == "jp":
+        return "chore: refresh JP market data"
+    if mode == "us":
+        return "chore: refresh US market data"
+    return "chore: refresh market data"
+
+
+def commit_and_push(repo_root: Path, mode: str, message: str | None, push: bool) -> None:
+    ensure_git_identity(repo_root)
+    run_git(repo_root, "add", "data/raw", "data/processed")
+    run_git(repo_root, "commit", "-m", message or default_commit_message(mode))
+
+    if push:
+        LOGGER.info("Pushing updated data to origin")
+        run_git(repo_root, "push")
+
+
+def main() -> int:
+    args = parse_args()
+    if args.push and not args.commit:
+        raise SystemExit("--push requires --commit")
+
+    logging.basicConfig(
+        level=logging.INFO,
+        format="%(asctime)s %(levelname)s %(message)s",
+    )
+
+    repo_root = ensure_repo_root(args.repo_dir)
+    configure_import_path(repo_root)
+
+    LOGGER.info("Repository root: %s", repo_root)
+    LOGGER.info("Refresh mode: %s", args.mode)
+
+    refresh_market_data(args.mode)
+    validate_outputs(repo_root)
+
+    diff = data_diff_status(repo_root)
+    if not diff:
+        LOGGER.info("No changes detected in data/raw or data/processed")
+        return 0
+
+    LOGGER.info("Changed files:\n%s", diff)
+    if args.commit:
+        commit_and_push(repo_root, args.mode, args.message, args.push)
+    else:
+        LOGGER.info("Changes detected, but --commit was not requested")
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except Exception as exc:
+        LOGGER.exception("Data refresh failed: %s", exc)
+        raise SystemExit(1) from exc