{ "title": "Farm Scheduled Jobs & Watchdogs", "as_of": "2026-10-06T15:31:32+10:00", "timezone": "Australia/Brisbane", "readme": "Edit this file to add or update a job. Each entry: id (unique slug), name, owner_bot, schedule_brisbane, schedule_et (optional), what_it_does, last_run (ISO 8601 with offset, or null), last_status (succeeded | failed | never run | planned | status not confirmed), active (yes | no | not confirmed), notes, optional catch_up, source. Optional live_status_url points to a JSON with last_run + success. Optional live_status_key reads jobs[] from a multi-job status file instead (status, started, finished, done, size, verify_status, notes), e.g. /dev/farm-schedule/data-downloads.json. Never invent a last_run: use null and 'status not confirmed'. Bump as_of when you edit.", "jobs": [ { "id": "nitro-crt-aplus-weekly-backtest", "name": "Weekly Nitro CRT A+ backtest", "owner_bot": "Video_Scrap Yt-dlp", "schedule_brisbane": "Mondays 8:00 PM", "schedule_et": "Mondays 6:00 AM ET", "what_it_does": "Runs the weekly Nitro CRT A+ backtest.", "last_run": "2026-09-28T20:00:00+10:00", "last_status": "succeeded", "active": "yes", "notes": "WARNING: still uses the old stock data flagged bad on 2026-09-29. Its results are not valid until it moves to the clean data. If the trigger is pinned to ET, the Brisbane time becomes 9:00 PM after US daylight saving ends (1 Nov 2026).", "source": "David's brief (authoritative), 2026-10-02" }, { "id": "nitro-trades-upload-check", "name": "Nitro Trades new-upload check", "owner_bot": "Video_Scrap Yt-dlp", "schedule_brisbane": "Weekdays 7:04 AM", "schedule_et": null, "what_it_does": "Checks the Nitro Trades YouTube channel for new uploads and queues them for a strategy pass.", "last_run": null, "last_status": "never run", "active": "yes", "notes": "First weekday trigger is Fri 2 Oct 2026 7:04 AM Brisbane. Whether that run happened is not confirmed.", "source": "David's brief (authoritative), 2026-10-02" }, { "id": "farm-code-dashboard-daily-backup", "name": "Daily farm code + dashboard backup", "owner_bot": "Video_Scrap Yt-dlp", "schedule_brisbane": "Every day 11:04 PM", "schedule_et": null, "what_it_does": "Zips smt-python strategy code, the DSG dashboard and every strategy (NinjaTrader .cs + farm presets) to C:\\GROK_BOT_TradingFarm\\backups\\smt-python\\YYYY-MM-DD on PC1–PC4 and to Google Drive 'Strategy Code Backups'. 30-day rotation.", "last_run": null, "last_status": "never run", "active": "yes", "notes": "Created 2026-10-02. First run due 2 Oct 2026 11:04 PM Brisbane.", "catch_up": "If a PC is offline at run time it is skipped; on the next nightly run after it reconnects it gets the latest full backup set. David is alerted if a PC misses 2+ runs in a row. 30-day rotation unchanged.", "source": "David's brief (authoritative), 2026-10-02" }, { "id": "dsg-dashboard-nightly-drive-backup", "name": "Nightly DSG dashboard backup to Drive", "owner_bot": "Dashboard Dev_HQ", "schedule_brisbane": "Every day 11:39 PM", "schedule_et": null, "what_it_does": "Backs up the live VPS dashboard plus the working copy (including /dev) in split parts to Google Drive GROK_BOT_TradingFarm/dashboard_backup. Alerts on failure.", "last_run": "2026-10-01T23:46:00+10:00", "last_status": "succeeded", "active": "yes", "notes": "", "source": "David's brief (authoritative), 2026-10-02" }, { "id": "vps-site-snapshot", "name": "Nightly VPS site snapshot", "owner_bot": "Dashboard Dev_HQ (server cron)", "schedule_brisbane": "Every day 2:30 AM", "schedule_et": null, "what_it_does": "/opt/farm-backups/backup-site.sh takes rsync --link-dest snapshots of /opt/farm-dashboard/html plus config. Keeps 28 days.", "last_run": "2026-10-02T02:30:06+10:00", "last_status": "succeeded", "active": "yes", "notes": "The page reads /dev/api/backup-status.json live, so this row's last run updates by itself. Values here are what that file said when this JSON was written.", "source": "Root crontab '30 16 * * * /opt/farm-backups/backup-site.sh' (16:30 UTC) + /dev/api/backup-status.json + /opt/farm-backups/backup.log ('backup OK: 2026-10-02'), checked over SSH 2026-10-02", "live_status_url": "/dev/api/backup-status.json" }, { "id": "vps-openclaw-backup", "name": "OpenClaw data backup (root crontab)", "owner_bot": "status not confirmed", "schedule_brisbane": "Every day 11:00 AM", "schedule_et": null, "what_it_does": "/root/backup_openclaw.sh tars /docker/openclaw-xwmv/data/.openclaw/ (the openclaw-xwmv OpenClaw bot's data) to /root/backups/openclaw/openclaw_YYYY-MM-DD.tar.gz, keeps only the newest copy, and logs to /root/backups/backup.log.", "last_run": "2026-10-01T11:01:48+10:00", "last_status": "succeeded", "active": "yes", "notes": "Proof: /root/backups/backup.log line 'Thu Oct 1 01:01:48 UTC 2026: Backup done: openclaw_2026-10-01.tar.gz (625M)', with daily lines back to 27 Sep. Only one copy is kept, on the same VPS (no off-box copy from this script). Script read, not run.", "source": "Root crontab '0 1 * * * /root/backup_openclaw.sh' (01:00 UTC) + script + /root/backups/backup.log, checked over SSH 2026-10-02" }, { "id": "hermie-delayed-10", "name": "Hermie 'Delayed 10' deferred learner", "owner_bot": "Hermie (grok-farm-hub container)", "schedule_brisbane": "About every 10 minutes (checks each minute; 600 s tick)", "schedule_et": null, "what_it_does": "Background thread 'grok-deferred-learn' in grok-farm-hub (learn.py). Gathers PC chat logs, builds a shared Hermie memory/skills snapshot and swaps it in atomically ('Hermes first, then copy to Grok'). Not a trading job.", "last_run": "2026-10-02T07:04:34+10:00", "last_status": "succeeded", "active": "yes", "notes": "Proof: /opt/grok-farm-hub/data/learn/published/memory/LAST_RUN.md 'utc: 2026-10-01T21:04:34Z' and cursor.json last_tick. Runs in the container, not cron.", "source": "VPS /opt/grok-farm-hub/learn.py (LEARN_TICK_SEC=600) + data/learn/published/memory/LAST_RUN.md; box /workspace/2026-09-13T071900Z-boot.json ('Delayed 10 = VPS deferred learner every 10 min')" }, { "id": "openclaw-bot-watchdog", "name": "OpenClaw Hermie + Kermit bot watchdog", "owner_bot": "status not confirmed (VPS systemd)", "schedule_brisbane": "Continuous, 60 s ticks (systemd daemon)", "schedule_et": null, "what_it_does": "openclaw-bot-watchdog.service runs /srv/openclaw-bot-watchdog.sh --daemon. Each tick: keeps the Hermie and Kermit containers up, restarts on a Telegram poll stall, gateway probe failure or RPC freeze, and sends David a Telegram alert on every auto-restart.", "last_run": "2026-10-01T21:45:58+10:00", "last_status": "status not confirmed", "active": "yes", "notes": "systemctl: active (running) since 12 Sep 2026. The log only records notes and restarts. The last line (11:45:58Z on 1 Oct) is a skipped 'Telegram API unreachable' tick, so outcome is not confirmed. Cron fallback: /etc/cron.d/openclaw-telegram-watchdog runs the same script each minute only if the service is not active.", "source": "VPS /etc/systemd/system/openclaw-bot-watchdog.service + /etc/cron.d/openclaw-telegram-watchdog + /var/log/openclaw-telegram-watchdog.log, checked over SSH 2026-10-02" }, { "id": "openclaw-model-watchdog", "name": "OpenClaw model watchdog", "owner_bot": "status not confirmed (VPS cron)", "schedule_brisbane": "Every 5 minutes", "schedule_et": null, "what_it_does": "/srv/openclaw-model-watchdog.sh: if the agent edits its primary model in openclaw.json to an id not in the fallback-proxy registry, revert it to the canonical model and log it. Silent otherwise.", "last_run": null, "last_status": "status not confirmed", "active": "yes", "notes": "Silent by design, so there is no proof of recent runs. The last log entry is an alert on 17 May 2026.", "source": "VPS /etc/cron.d/openclaw-model-watchdog + /var/log/openclaw-model-watchdog.log" }, { "id": "openclaw-model-sync", "name": "OpenClaw model registry sync", "owner_bot": "status not confirmed (VPS cron)", "schedule_brisbane": "Every day 2:00 PM", "schedule_et": null, "what_it_does": "/srv/fallback-proxy/sync-models.sh refreshes the OpenClaw fallback proxy's model list (04:00 UTC).", "last_run": "2026-10-01T14:00:01+10:00", "last_status": "succeeded", "active": "yes", "notes": "Proof: /var/log/openclaw-model-sync.log '[2026-10-01T04:00:01Z] updated 7 model(s)'.", "source": "VPS /etc/cron.d/openclaw-model-sync + /var/log/openclaw-model-sync.log" }, { "id": "hermie-ping-david", "name": "Hermie one-shot morning ping", "owner_bot": "status not confirmed (unit named hermie-ping-david; VPS systemd timer)", "schedule_brisbane": "One-shot: 19 May 2026 6:00 PM (done)", "schedule_et": null, "what_it_does": "hermie-ping-david.timer ran /usr/local/bin/hermie-ping-david.sh once ('Ping David tomorrow morning').", "last_run": "2026-05-19T18:00:01+10:00", "last_status": "status not confirmed", "active": "no", "notes": "Timer is disabled with no next run. systemd shows it fired at 08:00:01 UTC on 19 May. No journal entries remain, so the outcome is not confirmed.", "source": "VPS systemctl list-timers + /etc/systemd/system/hermie-ping-david.timer" }, { "id": "vps-docker-prune", "name": "VPS Docker cleanup", "owner_bot": "status not confirmed (VPS cron)", "schedule_brisbane": "Image prune every day 1:42 PM · builder prune Saturdays 12:40 PM", "schedule_et": null, "what_it_does": "docker image prune -af --filter until=24h (daily 03:42 UTC) and docker builder prune -f (Sat 02:40 UTC). Shared by every container on the VPS, including farm-dashboard, grok-farm-hub and nuggets-api.", "last_run": null, "last_status": "status not confirmed", "active": "yes", "notes": "Output goes to /dev/null, so there is no run record.", "source": "VPS /etc/cron.d/docker-image-prune + /etc/cron.d/docker-builder-prune" }, { "id": "vps-tvl-backup", "name": "TVL weekly backup (non-farm, same VPS)", "owner_bot": "status not confirmed (VPS cron)", "schedule_brisbane": "Sundays 10:00 PM", "schedule_et": null, "what_it_does": "/srv/tvl-backup.sh backs up the TVL Postgres DB, OpenClaw data dir, app .env files and YouTube cookies to /srv/backups and the MinIO bucket tvl-backups. Keeps one copy.", "last_run": "2026-09-27T22:02:28+10:00", "last_status": "succeeded", "active": "yes", "notes": "Proof: last line of /var/log/tvl-backup.log is 'TVL backup DONE total=644M'. The file was last written at 12:02:28 UTC on 27 Sep. This is the TVL app, not the farm. It is listed because it shares the VPS and also copies OpenClaw data.", "source": "VPS /etc/cron.d/tvl-backup + /srv/tvl-backup.sh header + /var/log/tvl-backup.log" }, { "id": "grokfarmnode-pc1", "name": "GrokFarmNode heartbeat (PC1)", "owner_bot": "GrokFarmNode (PC1, Windows Task Scheduler)", "schedule_brisbane": "Every 1 minute (task GrokFarmNode-PC1; or Continuous mode at logon, 20 s)", "schedule_et": null, "what_it_does": "GrokFarmNode.ps1 posts the PC's state (idle/now/last/next) to the farm hub at https://datasystemsglobal.com/farm-api. It does not drive NinjaTrader.", "last_run": "2026-10-01T12:48:03+10:00", "last_status": "succeeded", "active": "not confirmed", "notes": "Last heartbeat the hub received from PC1 was 02:48:03Z on 1 Oct. Nothing newer when checked. The heartbeat proves the node ran, but not which mode (scheduled task or continuous) it uses. The task itself was not checked on the PC.", "source": "Box /workspace/grok-farm-hub/node/Install-GrokFarmNode.ps1 + README; VPS /opt/grok-farm-hub/data/status.json" }, { "id": "grokfarmnode-pc3", "name": "GrokFarmNode heartbeat (PC3)", "owner_bot": "GrokFarmNode (PC3, Windows Task Scheduler)", "schedule_brisbane": "Every 1 minute (task GrokFarmNode-PC3; or Continuous mode at logon, 20 s)", "schedule_et": null, "what_it_does": "GrokFarmNode.ps1 posts the PC's state (idle/now/last/next) to the farm hub at https://datasystemsglobal.com/farm-api. It does not drive NinjaTrader.", "last_run": "2026-10-02T07:06:03+10:00", "last_status": "succeeded", "active": "yes", "notes": "Hub received a PC3 heartbeat at 21:06:03Z on 1 Oct, minutes before this JSON was written. The heartbeat proves the node ran, but not which mode (scheduled task or continuous) it uses. The task itself was not checked on the PC.", "source": "Box /workspace/grok-farm-hub/node/Install-GrokFarmNode.ps1 + README; VPS /opt/grok-farm-hub/data/status.json" }, { "id": "grokfarmnode-pc4", "name": "GrokFarmNode heartbeat (PC4)", "owner_bot": "GrokFarmNode (PC4, Windows Task Scheduler)", "schedule_brisbane": "Every 1 minute (task GrokFarmNode-PC4; or Continuous mode at logon, 20 s)", "schedule_et": null, "what_it_does": "GrokFarmNode.ps1 posts the PC's state (idle/now/last/next) to the farm hub at https://datasystemsglobal.com/farm-api. It does not drive NinjaTrader.", "last_run": "2026-09-13T20:13:28+10:00", "last_status": "succeeded", "active": "not confirmed", "notes": "Last heartbeat the hub received from PC4 was 10:13:28Z on 13 Sep. Stale since then. The heartbeat proves the node ran, but not which mode (scheduled task or continuous) it uses. The task itself was not checked on the PC.", "source": "Box /workspace/grok-farm-hub/node/Install-GrokFarmNode.ps1 + README; VPS /opt/grok-farm-hub/data/status.json" }, { "id": "nt-historical-download", "name": "NinjaTrader daily historical download", "owner_bot": "status not confirmed (farm PCs)", "schedule_brisbane": "Every day 8:00 AM (PC local time)", "schedule_et": null, "what_it_does": "Windows task 'GrokFarm-NtHistoricalDownload' runs _RunDataDownloadScheduled.ps1 / Install-NtDataDownloadTask.ps1 to download minute history (Forex, FXCM crypto, Tier-1 futures, from 2026-01-01). PC3 is the hub.", "last_run": null, "last_status": "status not confirmed", "active": "not confirmed", "notes": "Install scripts were staged on the box on 15 Sep 2026. Their STATUS.json says BLOCKED, with no machine access and no proof, so whether the task is installed on any PC is not confirmed.", "source": "Box /workspace/farm_nt_daily_deploy/bridges/Install-GrokFarmNtDailyTasks.ps1 + STATUS.json + PARENT_HANDOFF.md" }, { "id": "nt-minute-sync-from-pc3", "name": "NinjaTrader minute sync from PC3 (spokes)", "owner_bot": "status not confirmed (farm PCs)", "schedule_brisbane": "Every day 8:30 AM (PC local time)", "schedule_et": null, "what_it_does": "Windows task 'GrokFarm-NtMinuteSyncFromPC3' on spoke PCs runs Sync-NtMinuteFromPC3.ps1 to copy the hub's minute data after the 8:00 AM download.", "last_run": null, "last_status": "status not confirmed", "active": "not confirmed", "notes": "Same staged package as the daily download. Install on PC1/PC2/PC4 is not confirmed.", "source": "Box /workspace/farm_nt_daily_deploy/bridges/Install-GrokFarmNtDailyTasks.ps1 + parent_cmds/03_SPOKE_INSTALL.ps1" }, { "id": "pc4-farmdatadownload-legacy", "name": "FarmDataDownload (legacy task, PC4)", "owner_bot": "status not confirmed (PC4)", "schedule_brisbane": "status not confirmed", "schedule_et": null, "what_it_does": "Older Windows data-download task on PC4. The spoke install step disables it (schtasks /Change /TN FarmDataDownload /DISABLE).", "last_run": null, "last_status": "status not confirmed", "active": "not confirmed", "notes": "Whether it still exists or was disabled is not confirmed.", "source": "Box /workspace/farm_nt_daily_deploy/parent_cmds/03_SPOKE_INSTALL.ps1 + PARENT_HANDOFF.md" }, { "id": "pc2-nt-weekly-data-watchdog", "name": "NT weekly data watchdog (PC2)", "owner_bot": "status not confirmed (PC2)", "schedule_brisbane": "Weekly (intended). No scheduled-task registration found", "schedule_et": null, "what_it_does": "nt_weekly_data_watchdog.ps1 scans the last 8 days of download receipts and re-runs failed or missing symbols with a Backfill from 2026-01-01.", "last_run": null, "last_status": "status not confirmed", "active": "not confirmed", "notes": "Staged for PC2 but blocked (no machine access). No task registration exists in the scripts, so it may only be run by hand.", "source": "Box /workspace/pc2_nt_data_download/scripts/nt_weekly_data_watchdog.ps1 + PARENT_HANDOFF_PC2_NT_DOWNLOAD.md" }, { "id": "daily-market-data-download-watchdog", "name": "Daily market data download (all symbols) — watchdog", "owner_bot": "Video_Scrap Yt-dlp (farm data)", "schedule_brisbane": "Daily after US close on PC1, PC2, PC3, PC4 (exact time set when built)", "schedule_et": null, "what_it_does": "Downloads each day's new 1-minute bars for every symbol the farm trades (NQ plus every other farm symbol), appends them to the existing series (e.g. /workspace/setuphunter_py/data/nq_front_month_m1.csv and equivalents), alerts if a day is missing, and handles contract roll weeks correctly going forward (one contract per bar, no mixed-contract bars).", "last_run": null, "last_status": "PLANNED", "active": "no", "notes": "NQ file currently runs 1 Jul 2025 to 14 Sep 2026 (~426k bars, UTC, bar-end). Past roll weeks (2025-12-10..12, 2026-03-11..13 + 03-15, 2026-06-11..12) still mix two contracts and need fixing; gap 15 Sep to today needs backfilling.", "catch_up": "If a PC misses its daily run because it was offline, when it comes back online it checks the last bar in each symbol's file, finds every missed trading day and backfills all of them in order (roll weeks handled, no duplicate bars), not just the current day.", "source": "Video_Scrap Yt-dlp brief, 2 Oct 2026 (David request)" }, { "id": "data-download-futures-ticks-1s", "name": "Futures tick download → 1-second bars", "owner_bot": "CEO MASTER BOT", "schedule_brisbane": "One-off run, started 6 Oct 2026 (start time read live from data-downloads.json)", "schedule_et": null, "what_it_does": "Downloads ticks for 13 full-size contracts (NQ ES YM RTY GC SI CL NG ZN 6E 6B 6A FDAX), 1 Jan 2026 → 31 Jul 2026, from NinjaTrader on DESKTOP-09E03F9 and turns them into 1-second bars in /workspace/data_pull/ticks_secbars/.", "last_run": null, "last_status": "running", "active": "yes", "notes": "Live from /dev/farm-schedule/data-downloads.json (job tick). Per-contract progress and the verify step are on /dev/data.html.", "source": "CEO MASTER BOT brief, 6 Oct 2026 3:08 PM (David voice request)", "live_status_url": "/dev/farm-schedule/data-downloads.json", "live_status_key": "tick" }, { "id": "data-download-fresh-1m", "name": "Fresh 1-minute download", "owner_bot": "CEO MASTER BOT", "schedule_brisbane": "One-off run, started 6 Oct 2026 (start time read live from data-downloads.json)", "schedule_et": null, "what_it_does": "Fresh 1-minute download for the same 13 full-size contracts, 1 Jan 2025 → 6 Oct 2026, with cleanliness checks and a comparison against held data before anything is replaced. Output /workspace/data_pull/m1_fullsize_2025-01_2026-07/.", "last_run": null, "last_status": "running", "active": "yes", "notes": "Live from /dev/farm-schedule/data-downloads.json (job m1). Per-contract progress and the verify step are on /dev/data.html.", "source": "CEO MASTER BOT brief, 6 Oct 2026 3:08 PM (David voice request)", "live_status_url": "/dev/farm-schedule/data-downloads.json", "live_status_key": "m1" } ] }