-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathscriptgen.ts
More file actions
2680 lines (2606 loc) · 207 KB
/
Copy pathscriptgen.ts
File metadata and controls
2680 lines (2606 loc) · 207 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
/**
* Generates a tailored worker setup, wrapping the official
* `lightchain-worker-toolkit` (idempotent 9-phase scripts). The browser can't
* install anything itself - this produces the exact, personalized commands the
* operator runs locally, with the production gotchas already handled.
*/
import { NETWORKS, DEFAULT_MODEL, type NetworkId, type NetworkConfig } from "./network";
// Sizes are MEASURED in the shared catalog (never inferred from a tag), so the
// generated script can gate a pull on real disk/VRAM numbers instead of guessing.
import { MODEL_CATALOG, lookupModel, residentVramGb } from "./model-catalog";
export type OS = "macos" | "linux" | "windows";
const TOOLKIT = "https://github.com/lightchain-protocol/lightchain-worker-toolkit";
// Bump on every install-script change so the log shows which version actually ran.
export const INSTALLER_REV = "2026-07-29.2";
export interface ScriptBundle {
os: OS;
network: NetworkId;
model: string;
prereqs: { label: string; cmd: string }[];
oneLiner: string; // single paste-and-run bootstrap (clone → all phases → run)
setup: string; // the explicit step-by-step (advanced)
verify: string;
watchdog: string;
ops: { label: string; cmd: string }[];
}
const PHASES =
"00-generate-key 01-resolve-addresses 02-prepare-ollama 03-pull-image 04-import-key 05-generate-ecdh 06-fund-worker 07-register 08-run-worker";
// Desktop one-click provides the worker key itself and funds it directly from the
// user's wallet, so it skips 00 (generate-key) and 06 (funder→worker transfer).
const DESKTOP_PHASES =
"01-resolve-addresses 02-prepare-ollama 03-pull-image 04-import-key 05-generate-ecdh 07-register 08-run-worker";
/**
* The toolkit's phase 08 starts a GATEWAY-mode container (it sets
* WORKER_GATEWAY_URL and nothing else). On networks that moved worker selection
* on-chain (testnet, since the 2026-07-14 sortition upgrade), the gateway no
* longer dispatches jobs, so a gateway-mode worker never wins a session and
* serves nothing. For those networks we drop phase 08 and run the worker
* ourselves in SORTITION mode via {@link sortitionRunUnix} / {@link sortitionRunWin}.
*/
const withoutRunPhase = (phases: string) => phases.replace(" 08-run-worker", "");
/**
* Worker RUN for on-chain-sortition networks, replacing the toolkit's phase 08.
* Runs from the toolkit's scripts/bash dir (env.sh + resolved.env resolved
* there). Keeps the SAME container name (lightchain-worker) so the watchdog,
* Stop, and Deregister keep working unchanged.
*
* Redis: the sortition build's heartbeat targets 127.0.0.1:6379 and is FATAL at
* init if unreachable. We run Redis as a sidecar and put the worker in its
* network namespace (--network container:), so the binary's default resolves
* with no REDIS_* override on every OS - host networking would break
* host.docker.internal on Docker Desktop, so it can't be used here.
* host.docker.internal is mapped on the Redis container so the shared namespace
* still reaches Ollama on the host.
*/
function sortitionRunUnix(net: NetworkConfig): string {
if (!net.sortition || !net.sessionManager) return "";
return [
'echo "▶ starting the worker in SORTITION mode (on-chain claimSession - gateway dispatch was removed on this network)"',
// env.sh resolves RPC/CHAIN/IMAGE/BEACON/addresses; source it without leaking its `set -e`.
". ./env.sh >/dev/null 2>&1 || true; set +eu 2>/dev/null || true",
'KD="${KEYS_DIR:-$HOME/lightchain-worker/keys}"',
"KSF=\"$(ls \"$KD/eth-keystore\" 2>/dev/null | grep -iE '^UTC--' | head -1)\"",
"docker rm -f lightchain-worker lightchain-redis >/dev/null 2>&1 || true",
// Heartbeat store; ephemeral (no persistence). Shares its netns with the worker.
'docker run -d --restart always --name lightchain-redis --add-host=host.docker.internal:host-gateway redis:7-alpine redis-server --save "" --appendonly no >/dev/null',
"for _ in $(seq 1 15); do docker exec lightchain-redis redis-cli ping >/dev/null 2>&1 && break; sleep 1; done",
// Operator-managed extras (web-search keys, tuning), kept in their own file.
//
// Deliberately NOT collected in the UI: a credential typed into the app would
// travel through this generated script, which is streamed to the install log
// and shown on screen. A file the operator writes once never passes through
// either. It also survives reinstalls, which -e flags baked in here would not:
// before this, re-running install silently dropped any added settings.
//
// Placed BEFORE the -e flags so the values this script manages always win. A
// stale local file must never be able to repoint RPC_URL or the registry.
'LOCAL_ENV="$(dirname "$KD")/worker.local.env"',
'EXTRA_ENV=""',
'[ -f "$LOCAL_ENV" ] && { EXTRA_ENV="--env-file $LOCAL_ENV"; echo "▶ applying operator settings from $LOCAL_ENV"; }',
"docker run -d --restart always --user root --name lightchain-worker \\",
" --network container:lightchain-redis \\",
" $EXTRA_ENV \\",
' -v "$KD:/data" \\',
' -e "WORKER_KEYSTORE_PATH=/data/eth-keystore/$KSF" \\',
' -e "WORKER_KEYSTORE_PASSWORD=${WORKER_PASSWORD:-}" \\',
' -e "ENCRYPTION_KEYSTORE_PATH=/data/worker-encryption.key" \\',
' -e "RPC_URL=$RPC_URL" -e "CHAIN_ID=$CHAIN_ID" \\',
' -e "WORKER_REGISTRY_ADDRESS=$WORKER_REGISTRY_ADDRESS" \\',
' -e "AI_CONFIG_ADDRESS=$AI_CONFIG_ADDRESS" \\',
' -e "JOB_REGISTRY_ADDRESS=$JOB_REGISTRY_ADDRESS" \\',
' -e "SUPPORTED_MODELS=$SUPPORTED_MODELS" \\',
' -e "OLLAMA_URL=${OLLAMA_URL:-http://host.docker.internal:11434}" \\',
' -e "BEACON_API_URL=$BEACON_API_URL" -e "BLOB_MODE=beacon" \\',
' -e "SESSION_KEY_FILE=/data/session-keys.enc" \\',
' -e "SORTITION_ENABLED=true" \\',
' -e "SESSION_MANAGER_ADDRESS=' + net.sessionManager + '" \\',
' -e "SORTITION_STATE_DIR=/data/sortition" \\',
' "$IMAGE"',
'echo "✓ worker started in SORTITION mode (expect log: worker sidecar running (sortition mode))"',
"docker logs --tail 25 lightchain-worker 2>&1 || true",
].join("\n");
}
/** PowerShell equivalent of {@link sortitionRunUnix}. */
function sortitionRunWin(net: NetworkConfig): string {
if (!net.sortition || !net.sessionManager) return "";
return [
'Write-Host "▶ starting the worker in SORTITION mode (on-chain claimSession - gateway dispatch was removed on this network)"',
". .\\env.ps1 2>$null",
'$ksf = (Get-ChildItem "$($env:KEYS_DIR)\\eth-keystore" -Filter "UTC--*" -ErrorAction SilentlyContinue | Select-Object -First 1).Name',
"docker rm -f lightchain-worker lightchain-redis 2>$null | Out-Null",
'docker run -d --restart always --name lightchain-redis --add-host=host.docker.internal:host-gateway redis:7-alpine redis-server --save "" --appendonly no | Out-Null',
"for ($i=0; $i -lt 15; $i++){ docker exec lightchain-redis redis-cli ping 2>$null | Out-Null; if ($LASTEXITCODE -eq 0) { break }; Start-Sleep 1 }",
// Same operator-managed extras file as the unix path - see the comment there.
// Built as an array rather than a string: splatting an empty string into a
// docker argument list leaves a stray empty arg, which docker rejects.
'$localEnv = Join-Path (Split-Path $env:KEYS_DIR -Parent) "worker.local.env"',
"$extraEnv = @()",
'if (Test-Path $localEnv) { $extraEnv = @("--env-file", $localEnv); Write-Host "> applying operator settings from $localEnv" }',
"docker run -d --restart always --user root --name lightchain-worker `",
" --network container:lightchain-redis `",
" @extraEnv `",
' -v "$($env:KEYS_DIR):/data" `',
' -e "WORKER_KEYSTORE_PATH=/data/eth-keystore/$ksf" `',
' -e "WORKER_KEYSTORE_PASSWORD=$($env:WORKER_PASSWORD)" `',
' -e "ENCRYPTION_KEYSTORE_PATH=/data/worker-encryption.key" `',
' -e "RPC_URL=$($env:RPC_URL)" -e "CHAIN_ID=$($env:CHAIN_ID)" `',
' -e "WORKER_REGISTRY_ADDRESS=$($env:WORKER_REGISTRY_ADDRESS)" `',
' -e "AI_CONFIG_ADDRESS=$($env:AI_CONFIG_ADDRESS)" `',
' -e "JOB_REGISTRY_ADDRESS=$($env:JOB_REGISTRY_ADDRESS)" `',
' -e "SUPPORTED_MODELS=$($env:SUPPORTED_MODELS)" `',
' -e "OLLAMA_URL=$($env:OLLAMA_URL)" `',
' -e "BEACON_API_URL=$($env:BEACON_API_URL)" -e "BLOB_MODE=beacon" `',
' -e "SESSION_KEY_FILE=/data/session-keys.enc" `',
' -e "SORTITION_ENABLED=true" `',
' -e "SESSION_MANAGER_ADDRESS=' + net.sessionManager + '" `',
' -e "SORTITION_STATE_DIR=/data/sortition" `',
" $($env:IMAGE)",
'Write-Host "✓ worker started in SORTITION mode"',
"docker logs --tail 25 lightchain-worker 2>&1 | ForEach-Object { Write-Host $_ }",
].join("\n");
}
/**
* Find OUR sleep-inhibitor holder by PID, without ever using a `-f` (full command
* line) pattern.
*
* pgrep/pkill -f match the whole command line, and the command line of the shell
* running any of these generated scripts IS the script text (the app runs them as
* `bash -lc "<script>"`). Every script that touches the holder also contains the
* literal `systemd-inhibit … --who=lightnode-awake` invocation it starts, so a -f
* pattern unavoidably matches that shell: the "already running?" probe returns a
* false positive (the holder is then never started) and the teardown pkill SIGTERMs
* the very shell running it (rc 143 - nothing after the pkill runs). No cleverer
* pattern spelling dodges that, because the text being matched IS the real command.
*
* So: match the exact process NAME (-x can never match a bash), then confirm each
* candidate is ours by reading its own /proc/<pid>/cmdline. systemd-inhibit is
* Linux-only, so /proc is always there on the only path that runs this.
*/
const AWAKE_PIDS_UNIX =
'ln_awake_pids(){ for LP in $(pgrep -x systemd-inhibit 2>/dev/null); do grep -qa lightnode-awake "/proc/$LP/cmdline" 2>/dev/null && printf "%s " "$LP"; done; return 0; }';
/** Stop the holder {@link AWAKE_PIDS_UNIX} finds (a no-op when it isn't running). */
const AWAKE_KILL_UNIX = `${AWAKE_PIDS_UNIX}; LNAP="$(ln_awake_pids)"; [ -n "$LNAP" ] && kill $LNAP 2>/dev/null; true`;
/**
* Keep-online watchdog (macOS + Linux), installed automatically by the worker
* setup. A worker only earns while its Docker container runs, and the container
* (--restart always) only runs while the Docker engine is up - but Docker
* Desktop is an app, so a reboot, logout, or long sleep stops it and the worker
* goes offline (lost earnings; a crash mid-job risks a slash). This watchdog
* runs every ~10 min via launchd (macOS) / cron (Linux) and:
* 1. starts the Docker engine if it is down (so it also auto-starts on login),
* 2. starts the worker container if it is stopped.
* It writes the script + registers the scheduler idempotently, and never aborts
* the install (wrapped in set +e by the caller).
*/
const KEEP_ONLINE_UNIX = `echo "▶ installing keep-online watchdog (auto-start Docker + worker)"
cat > "$HOME/.lightnode/keep-online.sh" <<'KEEPEOF'
#!/usr/bin/env bash
# LightNode keep-online watchdog - ensure Docker + the worker are running.
export PATH="/opt/homebrew/bin:/usr/local/bin:$HOME/.docker/bin:/Applications/Docker.app/Contents/Resources/bin:/usr/bin:/bin:/usr/sbin:/sbin:$PATH"
log(){ echo "$(date -u +%FT%TZ) $*"; }
# Optional alerting: if ~/.lightnode/alerts.webhook holds a URL, post a message to
# it on STATE CHANGES only (no spam) - worker down / Docker down / recovered.
# Discord-compatible JSON body; a plain webhook receives the same {"content":...}.
alert_state(){ local W; W="$(cat "$HOME/.lightnode/alerts.webhook" 2>/dev/null)"; [ -z "$W" ] && return 0; local C="$1"; local L; L="$(cat "$HOME/.lightnode/alerts.last" 2>/dev/null)"; [ "$C" = "$L" ] && return 0; printf '%s' "$C" > "$HOME/.lightnode/alerts.last"; local HN; HN="$(hostname 2>/dev/null || echo worker)"; local M=""; case "$C" in down) M="LightChain worker is DOWN on $HN and could not be restarted.";; docker_down) M="LightChain worker host $HN: Docker is not running.";; stale) M="LightChain worker on $HN is running but not connected to the gateway (stale) - it is not taking jobs.";; ok) case "$L" in down|docker_down|stale) M="LightChain worker is back online on $HN.";; esac;; esac; [ -n "$M" ] && curl -s -m 8 -H "content-type: application/json" -d "{\\"content\\":\\"$M\\"}" "$W" >/dev/null 2>&1; return 0; }
# Economic alerts (stuck jobs / settle-now / out-of-gas) - the on-chain conditions
# the operator must NOT miss, posted even when the desktop app is closed. Each
# category dedups via its own marker (alert_key) so it pings once per change. The
# worker address + deployed base URL come from ~/.lightnode/alerts.conf (written by
# the app's Downtime alerts card); the heavy lifting runs server-side in the public
# /api/worker-alert (the same checks the dashboard shows), so the watchdog only
# curls + greps - no cast / subgraph parsing in bash.
alert_key(){ local W; W="$(cat "$HOME/.lightnode/alerts.webhook" 2>/dev/null)"; [ -z "$W" ] && return 0; local LF="$HOME/.lightnode/alerts.$1"; local P; P="$(cat "$LF" 2>/dev/null)"; [ "$2" = "$P" ] && return 0; printf '%s' "$2" > "$LF"; [ -n "$2" ] && curl -s -m 8 -H "content-type: application/json" -d "{\\"content\\":\\"$2\\"}" "$W" >/dev/null 2>&1; return 0; }
econ_alerts(){
[ -s "$HOME/.lightnode/alerts.webhook" ] || return 0
[ -s "$HOME/.lightnode/alerts.conf" ] || return 0
local A N B; A="$(sed -nE 's/^WORKER_ADDR=//p' "$HOME/.lightnode/alerts.conf" | head -1)"; N="$(sed -nE 's/^NET=//p' "$HOME/.lightnode/alerts.conf" | head -1)"; B="$(sed -nE 's/^BASE=//p' "$HOME/.lightnode/alerts.conf" | head -1)"
[ -z "$A" ] && return 0; [ -z "$B" ] && return 0; [ -z "$N" ] && N="mainnet"
local J; J="$(curl -s -m 12 "$B/api/worker-alert?net=$N&address=$A" 2>/dev/null)"
printf '%s' "$J" | grep -q '"ok":true' || return 0
local HN; HN="$(hostname 2>/dev/null || echo worker)"
if printf '%s' "$J" | grep -q '"outOfGas":true'; then alert_key gas "LightChain worker on $HN is OUT OF GAS - its wallet ($A) cannot pay to acknowledge jobs, settle, or claim. Send it a little LCAI."; else alert_key gas ""; fi
local S; S="$(printf '%s' "$J" | sed -nE 's/.*"stuck":[[:space:]]*([0-9]+).*/\\1/p')"
if [ -n "$S" ] && [ "$S" -gt 0 ] 2>/dev/null; then alert_key stuck "LightChain worker on $HN has $S job(s) past their deadline (stuck) - clear them in the app to avoid a timeout slash."; else alert_key stuck ""; fi
local R; R="$(printf '%s' "$J" | sed -nE 's/.*"settleNow":[[:space:]]*([0-9]+).*/\\1/p')"
if [ -n "$R" ] && [ "$R" -gt 0 ] 2>/dev/null; then alert_key settle "LightChain worker on $HN has $R completed job(s) ready to settle - open the app and Settle to collect your earnings."; else alert_key settle ""; fi
local C; C="$(printf '%s' "$J" | sed -nE 's/.*"claimableLcai":[[:space:]]*([0-9.]+).*/\\1/p')"
if [ -n "$C" ] && awk -v c="$C" 'BEGIN{exit !(c+0 >= 0.01)}' 2>/dev/null; then alert_key claimable "LightChain worker on $HN has ~$C LCAI of earnings claimable - open the app and Withdraw to collect them."; else alert_key claimable ""; fi
}
# Respect an intentional Stop/Deregister: while this marker exists, leave the
# worker alone (Install or Restart clears it to re-arm).
AWAKE="$HOME/Library/LaunchAgents/ai.lightchain.worker-awake.plist"
if [ -f "$HOME/.lightnode/keep-online.paused" ]; then
log "paused by user - leaving worker as-is, allowing the machine to sleep"
[ "$(uname -s)" = "Darwin" ] && launchctl unload "$AWAKE" 2>/dev/null || true
alert_state paused
exit 0
fi
# Keep the machine awake while the worker should be online - a sleep mid-job
# drops it (acked-then-asleep = timeout = slash). macOS: a KeepAlive caffeinate
# launchd agent; Linux: a systemd-inhibit holder.
if [ "$(uname -s)" = "Darwin" ]; then
launchctl list ai.lightchain.worker-awake >/dev/null 2>&1 || launchctl load -w "$AWAKE" 2>/dev/null || true
elif command -v systemd-inhibit >/dev/null 2>&1; then
# Never a -f probe here: the INSTALLER's command line is the whole install
# script, which contains this very invocation - so a -f match reports "already
# running" during an install and the holder is never started (see AWAKE_PIDS_UNIX).
${AWAKE_PIDS_UNIX}
[ -n "$(ln_awake_pids)" ] || ( nohup systemd-inhibit --what=idle:sleep --who=lightnode-awake --why="worker running" sleep infinity >/dev/null 2>&1 & )
fi
if ! docker info >/dev/null 2>&1; then
log "docker down - starting"
if [ "$(uname -s)" = "Darwin" ]; then open -a Docker 2>/dev/null || true; else systemctl --user start docker-desktop 2>/dev/null || sudo systemctl start docker 2>/dev/null || true; fi
for _ in $(seq 1 45); do docker info >/dev/null 2>&1 && break; sleep 2; done
fi
docker info >/dev/null 2>&1 || { log "docker still down - retry next tick"; alert_state docker_down; exit 0; }
if docker ps -a --format '{{.Names}}' | grep -q '^lightchain-worker$'; then
docker ps --format '{{.Names}}' | grep -q '^lightchain-worker$' || { log "worker stopped - starting"; docker start lightchain-worker >/dev/null 2>&1 && log "worker started"; }
fi
# Alert on the final run-state (after any restart attempt), once per transition.
# Running-but-not-connected counts as stale: the worker re-auths with the gateway
# about hourly, so no auth/connect log in 70 min while up means it has dropped off.
if docker ps --format '{{.Names}}' | grep -q '^lightchain-worker$'; then
if docker logs --since 70m lightchain-worker 2>&1 | grep -qiE "authenticated with worker-gateway|websocket connected"; then alert_state ok; else alert_state stale; fi
else
alert_state down
fi
# On-chain economic alerts (best-effort), regardless of the local run-state.
econ_alerts
# Keep every served model warm in Ollama so none cold-loads mid-job. Reads the
# set from a file (one per line) so a model change is picked up, and the
# residency policy from ~/.lightnode/keep-alive so a "swap on demand" operator
# isn't silently re-pinned. Missing/empty file = -1 (pin for ever), which is what
# every install before the knob existed did.
KA="$(cat "$HOME/.lightnode/keep-alive" 2>/dev/null)"; [ -n "$KA" ] || KA=-1
while IFS= read -r M; do [ -n "$M" ] && curl -s -m 5 http://127.0.0.1:11434/api/generate -d "{\\"model\\":\\"$M\\",\\"prompt\\":\\"ok\\",\\"keep_alive\\":$KA,\\"stream\\":false}" >/dev/null 2>&1 & done < "$HOME/.lightnode/model" 2>/dev/null || true
KEEPEOF
chmod +x "$HOME/.lightnode/keep-online.sh"
if [ "$(uname -s)" = "Darwin" ]; then
PLIST="$HOME/Library/LaunchAgents/ai.lightchain.worker-watchdog.plist"
mkdir -p "$HOME/Library/LaunchAgents"
# Sleep-prevention agent: a KeepAlive caffeinate holds the system awake while
# loaded (unloaded by Stop/Free up/Deregister, and by the watchdog when paused).
AWAKE="$HOME/Library/LaunchAgents/ai.lightchain.worker-awake.plist"
cat > "$AWAKE" <<AWAKEEOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>Label</key><string>ai.lightchain.worker-awake</string>
<key>ProgramArguments</key><array><string>/usr/bin/caffeinate</string><string>-i</string><string>-s</string></array>
<key>KeepAlive</key><true/>
<key>RunAtLoad</key><true/>
</dict></plist>
AWAKEEOF
launchctl unload "$AWAKE" 2>/dev/null || true
launchctl load -w "$AWAKE" 2>/dev/null && echo "✓ sleep prevention active (machine stays awake while the worker runs)" || true
cat > "$PLIST" <<PLISTEOF
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0"><dict>
<key>Label</key><string>ai.lightchain.worker-watchdog</string>
<key>ProgramArguments</key><array><string>/bin/bash</string><string>$HOME/.lightnode/keep-online.sh</string></array>
<key>StartInterval</key><integer>600</integer>
<key>RunAtLoad</key><true/>
<key>StandardOutPath</key><string>$HOME/.lightnode/keep-online.log</string>
<key>StandardErrorPath</key><string>$HOME/.lightnode/keep-online.log</string>
</dict></plist>
PLISTEOF
launchctl unload "$PLIST" 2>/dev/null || true
launchctl load -w "$PLIST" 2>/dev/null && echo "✓ keep-online watchdog active (launchd, every 10 min)" || true
else
( crontab -l 2>/dev/null | grep -v 'lightnode/keep-online.sh'; echo "*/10 * * * * /bin/bash $HOME/.lightnode/keep-online.sh >> $HOME/.lightnode/keep-online.log 2>&1" ) | crontab - 2>/dev/null && echo "✓ keep-online watchdog active (cron, every 10 min)" || true
command -v systemctl >/dev/null 2>&1 && sudo systemctl enable docker >/dev/null 2>&1 || true
fi`;
// Sleep-prevention toggles (unix). ON = load the caffeinate agent (macOS) /
// start a systemd-inhibit holder (Linux). OFF = the reverse, so the machine can
// sleep again once the worker is intentionally down.
const AWAKE_ON_UNIX =
`if [ "$(uname -s)" = "Darwin" ]; then launchctl load -w "$HOME/Library/LaunchAgents/ai.lightchain.worker-awake.plist" 2>/dev/null || true; elif command -v systemd-inhibit >/dev/null 2>&1; then ${AWAKE_PIDS_UNIX}; [ -n "$(ln_awake_pids)" ] || ( nohup systemd-inhibit --what=idle:sleep --who=lightnode-awake --why="worker running" sleep infinity >/dev/null 2>&1 & ); fi`;
const AWAKE_OFF_UNIX =
`if [ "$(uname -s)" = "Darwin" ]; then launchctl unload "$HOME/Library/LaunchAgents/ai.lightchain.worker-awake.plist" 2>/dev/null || true; fi; ${AWAKE_KILL_UNIX}; echo "✓ sleep prevention off - the machine can sleep again"`;
// Robustly start Docker Desktop on Windows. The install knows the exact path, but
// the watchdog/ops historically used `Start-Process "Docker Desktop"` by NAME,
// which often fails to resolve - so after a reboot the engine never came up and
// the worker (and the keep-online restart) stayed down: jobs sat "Submitted".
// Prefer the real exe under %ProgramFiles%, fall back to the name.
const WIN_START_DOCKER =
'$dd = Join-Path $env:ProgramFiles "Docker\\Docker\\Docker Desktop.exe"; if (Test-Path $dd) { Start-Process $dd } else { Start-Process "Docker Desktop" -ErrorAction SilentlyContinue }';
// AppImage library-pollution guard (unix). An AppImage exports LD_LIBRARY_PATH
// (and friends) pointing at its OWN bundled libs; the system curl/git/docker we
// shell out to then load those mismatched libs and crash - e.g. the system curl
// picks up the bundle's newer libcurl against the host's older libnghttp2:
// "undefined symbol: nghttp2_option_set_no_rfc9113_...". Strip the bundle-prefixed
// entries (keeping any the user set) so host tools use host libraries. No-op
// unless launched from an AppImage (APPDIR set) - so .deb/.dmg/.exe are untouched.
// Ships web-side, so it fixes even users still on an older AppImage binary.
const APPIMAGE_ENV_GUARD_UNIX =
`if [ -n "\${APPDIR:-}" ]; then for V in LD_LIBRARY_PATH LD_PRELOAD GIO_MODULE_DIR GTK_PATH GST_PLUGIN_SYSTEM_PATH_1_0 PYTHONPATH PYTHONHOME PERLLIB; do APPV="\${!V}"; [ -z "$APPV" ] && continue; APPNEW="$(printf '%s' "$APPV" | tr ':' '\\n' | grep -vF "$APPDIR" | paste -sd: -)"; if [ -z "$APPNEW" ]; then unset "$V"; else export "$V=$APPNEW"; fi; done; fi`;
// Fallback when the guard above didn't (or couldn't) repair a broken system curl:
// say plainly it's the AppImage build and point at the .deb, instead of the
// misleading "RPC unreachable / check your connection" the curl failure produces.
const APPIMAGE_CURL_HINT_UNIX =
'if command -v curl >/dev/null 2>&1 && ! curl --version >/dev/null 2>&1; then echo "⛔ your system curl is crashing on startup (a broken library link - common with the AppImage build). That is why the network checks fail, NOT your connection. Install the .deb instead: download it from lightnode.app, or run: sudo apt install ./LightNode_*.deb - it has no bundled libraries. (Updating to the latest AppImage also fixes it.)"; OK=0; fi';
/** One command: clone, set the password, run all 9 phases (06 prompts for the funder key). */
function bootstrap(os: OS, network: NetworkId, model: string): string {
const net = NETWORKS[network];
// On sortition networks, drop the toolkit's gateway-mode phase 08 and run the
// worker ourselves in sortition mode instead (see withoutRunPhase).
const phases = net.sortition ? withoutRunPhase(PHASES) : PHASES;
if (os === "windows") {
const runWin = net.sortition ? `\n${sortitionRunWin(net)}` : "";
return `git clone ${TOOLKIT}.git; cd lightchain-worker-toolkit\\scripts\\powershell; Copy-Item -ErrorAction Ignore secrets.example.ps1 secrets.ps1; ` +
`$p=Read-Host -AsSecureString "Set a worker keystore password"; ` +
`$env:WORKER_PASSWORD=[Runtime.InteropServices.Marshal]::PtrToStringAuto([Runtime.InteropServices.Marshal]::SecureStringToBSTR($p)); ` +
`$env:NETWORK="${network}"; $env:SUPPORTED_MODELS="${model}"; ` +
`'${phases}'.Split(' ') | ForEach-Object { & ".\\$_.ps1"; if ($LASTEXITCODE -ne 0){ Write-Host "stopped at $_"; break } }${runWin}`;
}
const runUnix = net.sortition ? ` && \\\n${sortitionRunUnix(net)}` : "";
return (
`git clone ${TOOLKIT}.git && cd lightchain-worker-toolkit/scripts/bash && cp -n secrets.example.sh secrets.env && \\\n` +
`read -rs -p "Set a worker keystore password: " WP; echo && \\\n` +
`sed -i.bak "s|WORKER_PASSWORD=.*|WORKER_PASSWORD=\\"$WP\\"|" secrets.env && rm -f secrets.env.bak && \\\n` +
`export NETWORK=${network} SUPPORTED_MODELS=${model} && \\\n` +
`for p in ${phases}; do bash "$p.sh" || { echo "⛔ stopped at $p"; break; }; done${runUnix}`
);
}
/** Idempotent prerequisite checks: install a tool only when it's missing. */
const SMART_PREREQS = `have(){ command -v "$1" >/dev/null 2>&1; }
# Put tool dirs on PATH up front so an already-installed Foundry / Docker / Ollama
# is detected and we SKIP re-running their installers (foundryup is a network call
# that otherwise ran on every install even when cast was already present).
export PATH="$HOME/.foundry/bin:/opt/homebrew/bin:/usr/local/bin:$HOME/.docker/bin:/Applications/Docker.app/Contents/Resources/bin:$PATH"
OS="$(uname -s)"
if [ "$OS" = "Darwin" ] && ! have brew; then echo "⛔ Install Homebrew first: https://brew.sh"; exit 1; fi
# ── Root escalation that CANNOT hang ─────────────────────────────────────────
# The desktop app runs this installer through \`bash -lc\` with stdin INHERITED
# and no tty. A plain \`sudo\` there either dies ("no tty present") - which under
# the set -e above tears the WHOLE install down - or, when a terminal happens to
# be attached, blocks forever on a password prompt the user never sees. Both are
# why a Linux install could not finish. So the ladder is strictly:
# already root -> sudo -n (never prompts; exits non-zero instead) -> pkexec
# pkexec raises a GRAPHICAL polkit dialog, the only prompt that can reach a user
# who launched us from a desktop icon; with no authentication agent registered it
# fails immediately rather than falling back to a tty prompt. Bare \`sudo\` is
# deliberately NOT in the ladder. Arguments pass through untouched so callers
# never nest quotes, and stdin is closed so a vendor script cannot swallow the
# app's pipe (or block waiting on it).
as_root() {
if [ "$(id -u)" = "0" ]; then "$@" </dev/null; return $?; fi
if sudo -n true 2>/dev/null; then sudo -n "$@" </dev/null; return $?; fi
if have pkexec; then pkexec "$@" </dev/null; return $?; fi
return 127
}
# Whether as_root has ANY chance of working. Checked BEFORE downloading a vendor
# installer so we fail with an actionable message instead of half-way through.
can_root(){ [ "$(id -u)" = "0" ] || sudo -n true 2>/dev/null || have pkexec; }
# ── Model residency knobs (defaults deliberately UNCHANGED) ──────────────────
# OLLAMA_KEEP_ALIVE=-1 pins every served model in memory for ever. That default
# is a financial decision, not a performance one: a job that misses its deadline
# is SLASHED, and cold-loading a 20-60 GB model costs tens of seconds against a
# ~120s budget. The price is that the memory is never handed back, and since
# every selected model is pinned, serving N models needs the SUM of their
# resident footprints.
# A future "swap on demand" mode wants the opposite trade (fit more models than
# VRAM, pay one cold load per switch), so both knobs are read from the
# environment instead of being hardcoded here. Nothing in the app sets them, so
# the generated script behaves exactly as before unless an operator opts in with
# e.g. LIGHTNODE_KEEP_ALIVE=5m LIGHTNODE_MAX_LOADED_MODELS=1 - and that operator
# is trading slash risk for memory, knowingly.
LN_KEEP_ALIVE="\${LIGHTNODE_KEEP_ALIVE:--1}"
LN_MAX_LOADED="\${LIGHTNODE_MAX_LOADED_MODELS:-}"
# JSON form for /api/generate: a bare number must stay bare, a duration string
# ("5m") must be quoted or the request body is invalid JSON.
case "$LN_KEEP_ALIVE" in ""|*[!0-9-]*) LN_KEEP_ALIVE_JSON="\\"$LN_KEEP_ALIVE\\"";; *) LN_KEEP_ALIVE_JSON="$LN_KEEP_ALIVE";; esac
# The systemd drop-in that makes Ollama reachable from the worker CONTAINER (see
# the Linux note in section 3) plus the residency knobs. Staged into a file WE
# own, so the only privileged step is copying it into place - no quoting games
# inside a root shell - and the same file serves both the fresh-install path and
# the "Ollama was already here" path.
write_ollama_dropin() {
mkdir -p "$HOME/.lightnode"
{ echo "[Service]"
echo 'Environment="OLLAMA_HOST=0.0.0.0:11434"'
echo 'Environment="OLLAMA_KEEP_ALIVE='"$LN_KEEP_ALIVE"'"'
[ -n "$LN_MAX_LOADED" ] && echo 'Environment="OLLAMA_MAX_LOADED_MODELS='"$LN_MAX_LOADED"'"' || true
} > "$HOME/.lightnode/ollama-lightnode.conf"
}
# 0) Disk guard. A near-full startup disk makes Docker Desktop's backend crash while
# writing its lock files, into an unrecoverable state ("no space left on device").
# Fail fast with a clear message BEFORE we ever start Docker. (df -k is portable;
# col 4 = available KB; /1048576 = GiB.)
FREE_G="$(df -k / 2>/dev/null | awk 'NR==2 {print int($4/1048576)}')"
if [ -n "$FREE_G" ] && [ "$FREE_G" -lt 5 ]; then
echo "⛔ Only ~$FREE_G GB free on your startup disk. Docker needs headroom to start safely (a near-full disk crashes its backend into an unrecoverable state), and the AI model needs several GB more. Free up space, then run install again."
exit 1
fi
[ -n "$FREE_G" ] && [ "$FREE_G" -lt 15 ] && echo "⚠ Only ~$FREE_G GB free - the model download alone needs several GB; you may run low."
# 1) Install only what's missing (idempotent; each is a no-op when present).
# Linux: BOTH vendor installers need root, and BOTH shell out to sudo themselves
# (get.docker.com sets sh_c='sudo -E sh -c'; ollama's install.sh sets SUDO=sudo).
# Piped straight into \`sh\`, that sudo fires from inside a pipe with no tty to
# prompt on - so the install either died or hung on a prompt nobody could see.
# Instead we download the script as the user and hand the whole job to as_root
# ONCE: one visible prompt, no hidden one, and a clean actionable failure.
if have docker; then echo "✓ Docker already installed"; else
echo "▶ installing Docker"
if [ "$OS" = "Darwin" ]; then brew install --cask docker; else
can_root || { echo "⛔ Docker is not installed, and installing it needs administrator rights this app cannot obtain here (you are not root, passwordless sudo is not configured, and pkexec - the graphical admin prompt - is unavailable)."; echo " Run this ONCE in a terminal, then click Install again:"; echo " curl -fsSL https://get.docker.com | sudo sh && sudo usermod -aG docker $(id -un) && newgrp docker"; exit 1; }
mkdir -p "$HOME/.lightnode"
curl -fsSL https://get.docker.com -o "$HOME/.lightnode/get-docker.sh" || { echo "⛔ could not download the Docker installer from get.docker.com - check your connection, then run install again."; exit 1; }
# One escalated invocation covering every root step: install, add you to the
# docker group, enable the service. Split up, this would ask three times.
cat > "$HOME/.lightnode/.root-docker.sh" <<ROOTDOCKEREOF
#!/bin/sh
sh "$HOME/.lightnode/get-docker.sh" || exit 1
usermod -aG docker "$(id -un)" 2>/dev/null || true
systemctl enable --now docker 2>/dev/null || true
exit 0
ROOTDOCKEREOF
echo "… approve the administrator prompt to install Docker"
as_root sh "$HOME/.lightnode/.root-docker.sh" || { echo "⛔ the Docker install did not complete - the administrator prompt was declined, or no polkit agent is running in this session."; echo " Run this ONCE in a terminal, then click Install again:"; echo " curl -fsSL https://get.docker.com | sudo sh && sudo usermod -aG docker $(id -un) && newgrp docker"; exit 1; }
hash -r 2>/dev/null || true
# A new group only applies to a NEW login session, so THIS shell still can't
# reach the socket. Say that now, plainly, instead of failing four minutes
# later in the engine wait with a generic "Docker didn't come up".
for _ in $(seq 1 15); do docker info >/dev/null 2>&1 && break; sleep 1; done
if ! docker info >/dev/null 2>&1 && docker info 2>&1 | grep -qi "permission denied"; then
echo "⛔ Docker is installed and running, but your user was only just added to the 'docker' group and Linux applies group changes at LOGIN. Log out and back in (or reboot), then click Install again. Nothing has been staked."
exit 1
fi
fi
fi
if have ollama; then echo "✓ Ollama already installed"; else
echo "▶ installing Ollama"
if [ "$OS" = "Darwin" ]; then brew install ollama; else
can_root || { echo "⛔ Ollama is not installed, and installing it needs administrator rights this app cannot obtain here (you are not root, passwordless sudo is not configured, and pkexec - the graphical admin prompt - is unavailable)."; echo " Run this ONCE in a terminal, then click Install again:"; echo " curl -fsSL https://ollama.com/install.sh | sh"; exit 1; }
mkdir -p "$HOME/.lightnode"
curl -fsSL https://ollama.com/install.sh -o "$HOME/.lightnode/get-ollama.sh" || { echo "⛔ could not download the Ollama installer from ollama.com - check your connection, then run install again."; exit 1; }
# Stage the drop-in first so ONE escalation does both root jobs: the vendor
# install AND the 0.0.0.0 bind the worker container needs (section 3). Asking
# twice is the difference between one polkit dialog and two.
write_ollama_dropin
cat > "$HOME/.lightnode/.root-ollama.sh" <<ROOTOLLAMAEOF
#!/bin/sh
sh "$HOME/.lightnode/get-ollama.sh" || exit 1
mkdir -p /etc/systemd/system/ollama.service.d
cp "$HOME/.lightnode/ollama-lightnode.conf" /etc/systemd/system/ollama.service.d/lightnode.conf || exit 1
chmod 0644 /etc/systemd/system/ollama.service.d/lightnode.conf
systemctl daemon-reload 2>/dev/null || true
systemctl enable ollama 2>/dev/null || true
systemctl restart ollama 2>/dev/null || true
exit 0
ROOTOLLAMAEOF
echo "… approve the administrator prompt to install Ollama (the same prompt also binds it to 0.0.0.0, which the worker container needs)"
as_root sh "$HOME/.lightnode/.root-ollama.sh" || { echo "⛔ the Ollama install (or the 0.0.0.0 bind that follows it) did not complete - the administrator prompt was declined, or no polkit agent is running in this session."; echo " Run this ONCE in a terminal, then click Install again:"; echo " curl -fsSL https://ollama.com/install.sh | sh"; exit 1; }
hash -r 2>/dev/null || true
have ollama || { echo "⛔ the Ollama installer ran but 'ollama' is still not on PATH. Open a new terminal and run 'ollama --version'; if that fails, reinstall from https://ollama.com/download/linux, then click Install again."; exit 1; }
fi
fi
if have cast; then echo "✓ Foundry already installed"; else
echo "▶ installing Foundry"
# foundryup installs the binaries fine but can return non-zero (e.g. libusb
# warning); tolerate its exit code and verify 'cast' afterward instead.
curl -L https://foundry.paradigm.xyz | bash || true
export PATH="$HOME/.foundry/bin:$PATH"
foundryup || true
fi
hash -r 2>/dev/null || true
# 2) Start Docker AND Ollama TOGETHER so Ollama boots during Docker's (much slower)
# cold start instead of after it. Keep the model resident (no idle eviction) so it
# never cold-loads mid-job - set before starting the server so it's picked up.
export OLLAMA_KEEP_ALIVE="$LN_KEEP_ALIVE"
[ -n "$LN_MAX_LOADED" ] && export OLLAMA_MAX_LOADED_MODELS="$LN_MAX_LOADED" || true
[ "$OS" = "Darwin" ] && { launchctl setenv OLLAMA_KEEP_ALIVE "$LN_KEEP_ALIVE" 2>/dev/null || true; [ -n "$LN_MAX_LOADED" ] && launchctl setenv OLLAMA_MAX_LOADED_MODELS "$LN_MAX_LOADED" 2>/dev/null || true; }
if ! curl -s http://127.0.0.1:11434/api/tags >/dev/null 2>&1; then
echo "▶ starting the Ollama server"
# sudo -n only, never bare sudo: a password prompt here has no tty to appear on
# (see as_root). The nohup fallback needs no privileges at all, so this always
# has a way through.
if [ "$OS" = "Darwin" ]; then open -a Ollama 2>/dev/null || brew services start ollama 2>/dev/null || (nohup ollama serve >/dev/null 2>&1 &)
else sudo -n systemctl start ollama 2>/dev/null || systemctl --user start ollama 2>/dev/null || (nohup ollama serve >/dev/null 2>&1 &); fi
fi
# Engine not on the default socket? Try the common alternates (Docker Desktop /
# Colima / Rancher) and pin DOCKER_HOST to whichever answers, before starting it.
if ! docker info >/dev/null 2>&1; then
for s in "$HOME/.docker/run/docker.sock" "/var/run/docker.sock" "$HOME/.colima/default/docker.sock" "$HOME/.rd/docker.sock"; do
if [ -S "$s" ] && DOCKER_HOST="unix://$s" docker info >/dev/null 2>&1; then export DOCKER_HOST="unix://$s"; break; fi
done
fi
# pgrep/pkill -f test the FULL command line, and this entire script IS the command
# line of the shell running it (the app runs it as: bash -lc "<script>"). Any
# process name spelled out literally here therefore ALSO matches the installer
# itself: the "is Docker wedged?" probe below would always say yes, and the pkill
# that follows would SIGTERM the installer (rc 143 - nothing after it ever runs).
# Assembling the name at runtime keeps the literal out of the script text, so the
# probes below can only ever match Docker's real backend processes. (-x, exact
# process NAME, is the other safe form - but this name is 18 chars and macOS
# truncates the name it matches on to 16, so -x would silently never match.)
DBK="com.docker"; DBK="$DBK.backend"
DOCKER_BACKEND_LOG="$HOME/Library/Containers/com.docker.docker/Data/log/host/$DBK.log"
if ! docker info >/dev/null 2>&1; then
if [ "$OS" = "Darwin" ]; then
# A crashed session can leave Docker's backend processes running while the
# daemon is dead - a graceful quit won't clear them, and a new launch collides
# with the zombies. If any are alive while docker is down, clear them first
# (TERM, then KILL - the parent backend often survives SIGTERM).
if pgrep -f "$DBK" >/dev/null 2>&1; then
echo "▶ Docker looks wedged (leftover backend from a crashed session) - clearing it first"
osascript -e 'quit app "Docker Desktop"' >/dev/null 2>&1 || true
pkill -f "MacOS/$DBK" 2>/dev/null || true
# The GUI app is matched by exact process NAME: a -f "Docker Desktop.app"
# pattern appears in this script's own text and would kill the installer.
pkill -x "Docker Desktop" 2>/dev/null || true
sleep 2
BPIDS="$(pgrep -f "$DBK" 2>/dev/null)"; [ -n "$BPIDS" ] && kill -9 $BPIDS 2>/dev/null; true
sleep 2
fi
echo "▶ starting the Docker engine"
open -a Docker 2>/dev/null || open -a "Docker Desktop" 2>/dev/null || true
else
echo "▶ starting the Docker engine"
# Silent paths first (passwordless sudo, then the Docker Desktop user unit);
# a graphical admin prompt only as a last resort. Never bare sudo - it would
# hang on a password prompt with no tty to show it.
sudo -n systemctl start docker 2>/dev/null || systemctl --user start docker-desktop 2>/dev/null || as_root systemctl start docker 2>/dev/null || true
fi
fi
# 3) Wait for Docker (the slow one), then Ollama (which booted in parallel, so this
# is usually instant). One clean recovery at ~90s covers both the macOS half-state
# (GUI open, engine never started) and a wedged/zombie backend.
echo "… waiting for the Docker engine (a cold start - e.g. right after 'Free up memory' - can take 1-2 min; approve any Docker permission dialog if it appears)"
for i in $(seq 1 120); do
docker info >/dev/null 2>&1 && break
if [ "$OS" = "Darwin" ] && [ "$i" = "45" ] && ! docker info >/dev/null 2>&1; then
echo "▶ Docker still not up - restarting it cleanly (clearing any stuck backend)..."
osascript -e 'quit app "Docker Desktop"' >/dev/null 2>&1 || true; sleep 2
BPIDS="$(pgrep -f "$DBK" 2>/dev/null)"; [ -n "$BPIDS" ] && kill -9 $BPIDS 2>/dev/null; true
pkill -x "Docker Desktop" 2>/dev/null || true; sleep 3
open -a Docker 2>/dev/null || open -a "Docker Desktop" 2>/dev/null || true
fi
[ $((i % 15)) -eq 0 ] && echo "… still waiting for Docker ($((i * 2))s elapsed)"
sleep 2
done
if ! docker info >/dev/null 2>&1; then
# Surface the REAL cause from Docker's own backend log, not a generic message.
if [ "$OS" = "Darwin" ] && [ -f "$DOCKER_BACKEND_LOG" ] && tail -n 60 "$DOCKER_BACKEND_LOG" 2>/dev/null | grep -qiE "no space left|writing locks"; then
echo "⛔ Docker can't start: your startup disk is full - its backend crashed writing lock files. Free up several GB, then run install again."
elif [ "$OS" = "Darwin" ] && [ -f "$DOCKER_BACKEND_LOG" ] && tail -n 60 "$DOCKER_BACKEND_LOG" 2>/dev/null | grep -qi "backend crashed"; then
echo "⛔ Docker's backend crashed on startup. Open Docker Desktop manually; if it offers 'Reset to factory defaults', use it, then run install again."
echo " last backend error:"; tail -n 2 "$DOCKER_BACKEND_LOG" 2>/dev/null | sed 's/^/ /'
else
echo "⛔ Docker engine didn't come up. Open Docker Desktop manually (approve any permission prompt), wait for the whale icon in the menu bar to settle, then run install again."
fi
exit 1
fi
echo "✓ Docker engine ready"
echo "… waiting for Ollama on 127.0.0.1:11434"
for _ in $(seq 1 30); do curl -s http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && break; sleep 1; done
curl -s http://127.0.0.1:11434/api/tags >/dev/null 2>&1 || { echo "⛔ Ollama isn't responding on 127.0.0.1:11434 - open the Ollama app (or run 'ollama serve'), then re-run."; exit 1; }
echo "✓ Ollama server running"
# Linux ONLY: the worker runs in Docker and reaches Ollama on the host via
# host.docker.internal -> the docker-bridge gateway (172.17.0.1). Ollama defaults
# to listening on 127.0.0.1 ONLY, so that connection is REFUSED ("dial tcp
# 172.17.0.1:11434: connect: connection refused") and EVERY job fails at the
# inference stage. (macOS/Windows are fine: Docker Desktop's VM proxies
# host.docker.internal to the host loopback - bare-metal Linux has no such proxy.)
# Bind Ollama to 0.0.0.0 so the bridge can reach it. Idempotent.
if [ "$OS" = "Linux" ]; then
# Judge the LISTEN address, not the unit file: the unit file lies whenever
# Ollama was started some other way (a user session, a nohup, a snap). Returns
# 0 = all interfaces, 1 = loopback only, 2 = can't tell (no ss/netstat), so an
# unknown never becomes a block on a guess.
ollama_bind_state() {
if have ss; then OBS="$(ss -ltn 2>/dev/null)"; elif have netstat; then OBS="$(netstat -ltn 2>/dev/null)"; else return 2; fi
printf '%s' "$OBS" | grep -q ':11434' || return 2
if printf '%s' "$OBS" | grep -Fq '0.0.0.0:11434' || printf '%s' "$OBS" | grep -Fq '[::]:11434' || printf '%s' "$OBS" | grep -Fq '*:11434'; then return 0; fi
return 1
}
OB=0; ollama_bind_state || OB=$?
# 1 = definitely loopback-only, 2 = we could not tell (no ss/netstat, or :11434
# missing from their output). BOTH get the fix. Skipping the unknown case is the
# worse trade by far: an unknown that really IS loopback-only sails through this
# gate and stakes a worker whose every job fails at inference. The drop-in is
# idempotent and a no-op when the bind was already right, so "attempt it anyway"
# costs an Ollama restart and buys back the only outcome that costs money.
if [ "$OB" != "0" ]; then
if [ "$OB" = "2" ]; then echo "▶ could not read Ollama's listen address here (no ss/netstat) - applying the 0.0.0.0 bind anyway; it is idempotent, and a silent loopback-only bind would fail every job"; else echo "▶ allowing the worker container to reach Ollama (binding it to 0.0.0.0)"; fi
write_ollama_dropin
if systemctl list-unit-files 2>/dev/null | grep -q '^ollama.service'; then
# Editing the system unit needs root - same no-hang ladder as everything
# else (as_root); a declined prompt is reported, never waited on.
cat > "$HOME/.lightnode/.root-ollama-bind.sh" <<ROOTBINDEOF
#!/bin/sh
mkdir -p /etc/systemd/system/ollama.service.d
cp "$HOME/.lightnode/ollama-lightnode.conf" /etc/systemd/system/ollama.service.d/lightnode.conf || exit 1
chmod 0644 /etc/systemd/system/ollama.service.d/lightnode.conf
systemctl daemon-reload || exit 1
systemctl restart ollama || exit 1
exit 0
ROOTBINDEOF
echo "… approve the administrator prompt so the worker container can reach Ollama"
as_root sh "$HOME/.lightnode/.root-ollama-bind.sh" >/dev/null 2>&1 || echo "⚠ could not rebind Ollama automatically (the admin prompt was declined or unavailable)"
else
# No systemd unit: Ollama is a plain process we can restart ourselves, so
# this path needs no privileges at all.
#
# -x (exact process NAME) is the ONLY safe form here. Matching on the full
# command line instead would match the command line of the bash running this
# installer, which IS this whole script - and the script names the Ollama
# server several times. pkill skips its own PID but not its parent, so that
# form SIGTERMs the installer itself (rc 143) and nothing below ever runs.
pkill -x ollama 2>/dev/null || true; sleep 1
OLLAMA_HOST=0.0.0.0:11434 OLLAMA_KEEP_ALIVE="$LN_KEEP_ALIVE" nohup ollama serve >/dev/null 2>&1 &
fi
for _ in $(seq 1 30); do curl -s http://127.0.0.1:11434/api/tags >/dev/null 2>&1 && break; sleep 1; done
OB=0; ollama_bind_state || OB=$?
fi
case "$OB" in
0) echo "✓ Ollama listens on all interfaces - the worker container can reach it" ;;
1) echo "⛔ Ollama still only listens on 127.0.0.1, so the Dockerized worker cannot reach it and EVERY job would fail at inference - a staked worker that earns nothing and can be slashed. Install stops here; nothing has been staked."
echo " Fix it once in a terminal, then click Install again:"
echo " sudo mkdir -p /etc/systemd/system/ollama.service.d"
echo " printf '[Service]\\nEnvironment=\\"OLLAMA_HOST=0.0.0.0:11434\\"\\n' | sudo tee /etc/systemd/system/ollama.service.d/lightnode.conf"
echo " sudo systemctl daemon-reload && sudo systemctl restart ollama"
exit 1 ;;
*) echo "⚠ could not confirm which address Ollama listens on (no ss/netstat here). If jobs fail at inference with 'connection refused' to 172.17.0.1:11434, bind Ollama to 0.0.0.0." ;;
esac
# Belt and braces: the container talks to the bridge GATEWAY, so a host firewall
# can still block a correctly-bound Ollama. WARN only - on Docker Desktop for
# Linux the daemon lives in a VM where this address legitimately differs from
# the one the host sees, and a false block there is worse than a warning.
if [ "$OB" = "0" ]; then
BRIDGE_GW="$(docker network inspect bridge --format '{{range .IPAM.Config}}{{.Gateway}}{{end}}' 2>/dev/null | head -1)"; [ -n "$BRIDGE_GW" ] || BRIDGE_GW=172.17.0.1
curl -s -m 5 "http://$BRIDGE_GW:11434/api/tags" >/dev/null 2>&1 || echo "⚠ Ollama is bound to all interfaces but did not answer on the docker bridge gateway ($BRIDGE_GW:11434). If jobs fail at inference a host firewall is blocking it - e.g. sudo ufw allow in on docker0 to any port 11434"
fi
fi
# 4) Foundry must be on PATH for the cast calls in the toolkit phases.
export PATH="$HOME/.foundry/bin:$PATH"
hash -r 2>/dev/null || true
have cast || { echo "⛔ Foundry installed but 'cast' isn't on PATH yet - fully quit and reopen LightNode, then run again."; exit 1; }
echo "✓ Foundry (cast) ready"`;
/** Smart, idempotent install for macOS + Linux (bash). The app passes the
* WORKER key + password via env; we fund the worker directly from the user's
* wallet, so there's no separate funder and no phase 00/06. */
function unixInstall(network: NetworkId, models: string[]): string {
const net = NETWORKS[network];
const chainId = NETWORKS[network].chainId;
const minStake = NETWORKS[network].minStakeLcai; // build-time fallback only; real value read live from AIConfig
const rpc = NETWORKS[network].rpc;
const explorer = NETWORKS[network].explorer;
const workerRegistry = NETWORKS[network].workerRegistry;
// Sortition networks run their own worker; drop the toolkit's gateway phase 08.
const desktopPhases = net.sortition ? withoutRunPhase(DESKTOP_PHASES) : DESKTOP_PHASES;
// Threshold for the funding gate: min stake + 0.5 LCAI gas cushion, in wei.
// BigInt because the value overflows JS Number for mainnet (50_000.5 * 1e18).
const thrWei = (BigInt(minStake) * 10n ** 18n + 5n * 10n ** 17n).toString();
const list = models.length ? models : [DEFAULT_MODEL];
const supported = list.join(","); // SUPPORTED_MODELS the worker advertises
const shellList = list.map((m) => `"${m}"`).join(" "); // for `for M in ...` loops
// Download sizes, straight from the shared catalog so they can never drift from
// what the UI showed. A tag we don't know gets no size and simply skips the
// disk gate - `known: false` stays first-class, we never guess a footprint.
const sizeCases = MODEL_CATALOG.map((e) => ` "${e.tag}") echo ${e.downloadGb};;`).join("\n");
// Resident VRAM the whole selected set needs. Every served model is pinned
// (keep_alive), so it is the SUM that has to fit, not the largest. Only emitted
// when every selected tag is in the catalog - a partial sum would understate it.
const entries = list.map((m) => lookupModel(m));
const vramNeedGb = entries.every((e) => e !== undefined)
? Math.round(entries.reduce((s, e) => s + residentVramGb(e!), 0) * 10) / 10
: 0;
return [
"set -e",
"exec 2>&1", // surface stderr (git clone, cast, etc.) in the streamed log
APPIMAGE_ENV_GUARD_UNIX, // host libs for shelled-out curl/git/docker on AppImage
`echo "▶ LightNode installer rev ${INSTALLER_REV} (${network})"`,
SMART_PREREQS,
// The app's working dir may be "/" (non-writable). Work in a real home dir.
'mkdir -p "$HOME/.lightnode" && cd "$HOME/.lightnode" && echo "✓ workdir: $HOME/.lightnode"',
// Persist the resolved residency policy so the watchdog re-warms with the SAME
// one the install used (it defaults to -1 when the file is absent, so every
// pre-existing install keeps pinning exactly as before).
`printf '%s\\n' "$LN_KEEP_ALIVE_JSON" > "$HOME/.lightnode/keep-alive"`,
// Changing the served set? Unload any previously-served model that is NOT in
// the new set (each is pinned with keep_alive:-1 and never evicts on its own),
// so its memory is freed instead of sitting resident.
`NEWSET="${list.join(" ")}"`,
'for OM in $(cat "$HOME/.lightnode/model" 2>/dev/null); do case " $NEWSET " in *" $OM "*) : ;; *) curl -s -m 10 http://127.0.0.1:11434/api/generate -d "{\\"model\\":\\"$OM\\",\\"keep_alive\\":0}" >/dev/null 2>&1; echo "✓ unloaded $OM (no longer served)";; esac; done',
// Record the served set (one model per line) so the watchdog warms each.
`printf '%s\\n' ${shellList} > "$HOME/.lightnode/model"`,
// Installing means the user wants the worker running - clear any pause set by
// a previous Stop/Deregister so the watchdog resumes guarding it.
'rm -f "$HOME/.lightnode/keep-online.paused" 2>/dev/null || true',
// Arm the keep-online watchdog on every run (best-effort, never aborts the
// install) so it's refreshed even when the worker is already running.
"set +e",
KEEP_ONLINE_UNIX,
'cd "$HOME/.lightnode"',
"set -e",
// Ensure EACH selected model is in Ollama under its exact on-chain name. The
// toolkit's phase 02 only handles llama3-8b, so pull + alias any others here.
// The pull tag is the on-chain name with the size turned back into a tag
// (llama3-70b -> llama3:70b); already-present models are skipped.
//
// Pull with THROTTLED progress: ollama assumes a TTY and emits thousands of
// escape-coded progress frames per download. Streamed verbatim that floods the
// app's log channel (and on a multi-GB model can choke the install). Piping the
// pull to a file makes ollama emit terse non-TTY output; we sample the percent
// every couple of seconds, so the app sees a handful of clean lines.
"pull_model() {",
' PM_NAME="$1"; PM_TAG="$2"; PM_LOG="$HOME/.lightnode/.pull.log"; : > "$PM_LOG"',
' echo "▶ downloading $PM_NAME - a multi-GB model can take several minutes"',
' ( ollama pull "$PM_TAG" > "$PM_LOG" 2>&1; echo "__PULLRC__:$?" >> "$PM_LOG" ) &',
' PM_PID=$!; PM_LAST=""',
' while kill -0 "$PM_PID" 2>/dev/null; do',
" PM_PCT=\"$(tr '\\r' '\\n' < \"$PM_LOG\" 2>/dev/null | grep -oE '[0-9]+%' | tail -1)\"",
' [ -n "$PM_PCT" ] && [ "$PM_PCT" != "$PM_LAST" ] && { echo " downloading $PM_NAME $PM_PCT"; PM_LAST="$PM_PCT"; } || true',
" sleep 2",
" done",
' wait "$PM_PID" 2>/dev/null || true',
// The exit code is reported, but it is NOT the gate. `ollama pull` can exit 0
// on a partially-written model, and a scraped marker can be missed entirely -
// so the only thing we trust is asking Ollama what it actually has, below.
" PM_RC=\"$(grep -oE '__PULLRC__:[0-9]+' \"$PM_LOG\" | tail -1 | cut -d: -f2)\"; PM_TAIL=\"$(tr '\\r' '\\n' < \"$PM_LOG\" 2>/dev/null | grep -v '^ *$' | tail -2 | tr '\\n' ' ')\"; rm -f \"$PM_LOG\"",
' if [ "${PM_RC:-1}" = "0" ]; then echo "✓ downloaded $PM_NAME"; else echo "⚠ $PM_NAME download exited ${PM_RC:-?}: $PM_TAIL"; fi',
"}",
// Is the model REALLY in Ollama under its exact on-chain name? `ollama list`
// prints NAME as tag[:latest], so an implicit :latest counts. Whole-line FIXED
// compare against the NAME column only: a substring match would accept
// "llama3-8b-instruct" for "llama3-8b", and a regex would treat the "." in
// "glm-4.7-flash" as a wildcard.
"model_present() {",
" ollama list 2>/dev/null | awk 'NR>1 {print $1}' | sed 's/:latest$//' | grep -qxF \"$(printf '%s' \"$1\" | sed 's/:latest$//')\"",
"}",
// Measured download size per known tag (generated from lib/model-catalog.ts).
// Empty for a tag we don't know, which just skips the disk gate.
"model_download_gb() {",
' case "$1" in',
sizeCases,
' *) echo "";;',
" esac",
"}",
// WHERE Ollama actually writes models - which is very often NOT $HOME. The
// stock Linux install runs Ollama as a SYSTEM SERVICE under its own user, so
// the blobs land in /usr/share/ollama/.ollama/models; measuring $HOME there
// gates on a filesystem the download never touches (it passes with a full /usr
// and blocks on a full /home - both answers wrong). Resolution order mirrors
// Ollama's own: OLLAMA_MODELS in this env, then whatever the unit exports,
// then the service user's home, then ~/.ollama (correct for macOS and for a
// user-local Linux install).
"ollama_models_dir() {",
' [ -n "${OLLAMA_MODELS:-}" ] && { printf \'%s\' "$OLLAMA_MODELS"; return 0; }',
" OMD=\"$(systemctl show ollama -p Environment 2>/dev/null | tr ' ' '\\n' | sed -n 's/^OLLAMA_MODELS=//p' | tail -1)\"",
' [ -n "$OMD" ] && { printf \'%s\' "$OMD"; return 0; }',
' OMU="$(systemctl show ollama -p User --value 2>/dev/null)"',
// Deliberately NOT gated on the dir existing: the service user's home is
// typically mode 0700 (/usr/share/ollama on Ubuntu), so a [ -d ] test on
// anything INSIDE it fails for us with EACCES and would silently fall back to
// $HOME - the exact wrong answer this is here to prevent. The walk-up below
// reduces it to the nearest ancestor we CAN stat, which is on the same
// filesystem, so df still measures the right one.
' if [ -n "$OMU" ]; then OMH="$(getent passwd "$OMU" 2>/dev/null | cut -d: -f6)"; case "${OMH:-}" in ""|/) : ;; *) printf \'%s\' "$OMH/.ollama/models"; return 0;; esac; fi',
` printf '%s' "$HOME/.ollama/models"`,
"}",
// df needs a path that EXISTS, and the models dir may not be created until the
// first pull - walk up to the nearest existing ancestor (they are on the same
// filesystem unless a mount appears underneath, which df would then report for
// the ancestor anyway). $HOME is the last resort, never the default.
'LN_DF_DIR="$(ollama_models_dir)"',
'while [ -n "$LN_DF_DIR" ] && [ ! -d "$LN_DF_DIR" ]; do LN_DF_DIR="$(dirname "$LN_DF_DIR")"; if [ "$LN_DF_DIR" = "/" ] || [ "$LN_DF_DIR" = "." ]; then break; fi; done',
'[ -d "$LN_DF_DIR" ] || LN_DF_DIR="$HOME"',
...(vramNeedGb > 0
? [
// Fit check - WARN, never a block: Ollama still runs an oversized model,
// it just spills to the CPU at a fraction of the speed, which is the most
// common cause of a missed deadline (and a slash). The operator may also
// have swapped GPUs since we measured.
`LN_VRAM_NEED=${vramNeedGb}`,
`GPU_MB="$(nvidia-smi --query-gpu=memory.total --format=csv,noheader,nounits 2>/dev/null | head -1)"`,
`if [ -n "$GPU_MB" ] && awk -v g="$GPU_MB" -v n="$LN_VRAM_NEED" 'BEGIN{exit !(g/1024 < n)}'; then GPU_GB="$(awk -v g="$GPU_MB" 'BEGIN{printf "%.1f", g/1024}')"; echo "⚠ the model(s) you picked need about $LN_VRAM_NEED GB resident (all served models stay loaded at once) and this GPU reports $GPU_GB GB. Whatever does not fit runs on the CPU - far slower, and slow jobs miss their deadline, which is slashable on mainnet. Consider serving fewer or smaller models."; fi`,
]
: []),
// Pull each missing model, then GATE ON PRESENCE. The old code only warned
// ("continuing") on a failed pull and walked straight into staking + register,
// producing a staked worker advertising a model it cannot run - every job it
// wins then fails. Real tags like qwen3-coder-next (51.7 GB) and gpt-oss:120b
// (65.4 GB) make that easy to hit, so the failure has to be loud and terminal.
'MISSING_MODELS=""',
`for M in ${shellList}; do`,
' if model_present "$M"; then echo "✓ model $M present"; continue; fi',
` TAG="$(printf '%s' "$M" | sed -E 's/-([0-9.]+[bB])$/:\\1/')"`,
' NEED_GB="$(model_download_gb "$M")"',
' if [ -n "$NEED_GB" ]; then',
// -P forces POSIX single-line output: a long device name otherwise wraps and
// NR==2 reads the header's continuation instead of the numbers.
` FREE_NOW="$(df -Pk "$LN_DF_DIR" 2>/dev/null | awk 'NR==2 {print int($4/1048576)}')"`,
` if [ -n "$FREE_NOW" ] && awk -v f="$FREE_NOW" -v n="$NEED_GB" 'BEGIN{exit !(f < n + 2)}'; then echo "⛔ $M is a ~$NEED_GB GB download and only $FREE_NOW GB is free on the filesystem Ollama stores models on ($LN_DF_DIR). Free up space there or pick a smaller model, then run install again - nothing has been staked."; exit 1; fi`,
' echo "▶ $M is a ~$NEED_GB GB download"',
" fi",
' pull_model "$M" "$TAG"',
' [ "$TAG" != "$M" ] && ollama cp "$TAG" "$M" >/dev/null 2>&1 && echo "✓ aliased $TAG -> $M" || true',
' if model_present "$M"; then echo "✓ model $M ready"; else MISSING_MODELS="$MISSING_MODELS $M"; fi',
"done",
'if [ -n "$MISSING_MODELS" ]; then',
' echo "⛔ these selected model(s) are NOT on this machine after the download:$MISSING_MODELS"',
' echo " A worker advertises what it serves, so registering now would stake your LCAI on a model that fails every job it wins (slashable on mainnet). Install stops here."',
' echo " Nothing was staked or registered - your funds are untouched."',
' echo " The download error is above; the usual causes are out of disk, out of memory, or a tag that does not exist in the Ollama registry."',
' echo " Check by hand with: ollama pull <tag> then ollama list"',
" exit 1",
"fi",
`if [ -d lightchain-worker-toolkit ]; then echo "✓ toolkit present - updating"; (cd lightchain-worker-toolkit && git pull --ff-only || true); else git clone ${TOOLKIT}.git; fi`,
"cd lightchain-worker-toolkit/scripts/bash",
"[ -f secrets.env ] || cp secrets.example.sh secrets.env",
// Pass secrets via the environment (the app already exported WORKER_PASSWORD +
// WORKER_PRIVKEY) - strip any file-set copies so they can't override, and add
// the derived address. Avoids sed-escaping pitfalls with special chars.
"grep -vE '^[[:space:]]*export (WORKER_PASSWORD|WORKER_ADDR|WORKER_PRIVKEY|FUNDER_PRIVKEY)=' secrets.env > secrets.env.tmp || true; mv secrets.env.tmp secrets.env",
// Prefer the address the app passed (public, always known). Only derive it
// from the key when absent - a switch-back to an already-registered worker may
// run without the raw key in the app (the on-disk keystore holds it).
'export WORKER_ADDR="${WORKER_ADDR:-$(cast wallet address --private-key "$WORKER_PRIVKEY" 2>/dev/null)}"',
'[ -n "$WORKER_ADDR" ] || { echo "⛔ no worker address or key available to install - generate/select a worker first."; exit 1; }',
`export NETWORK=${network} SUPPORTED_MODELS=${supported}`,
// Per-network keystore dir so installing one network never touches another's
// keys (a mainnet operator can set up testnet without risking their mainnet
// key). The legacy ~/lightchain-worker/keys is still read by key derivation.
`export KEYS_DIR="$HOME/lightchain-worker/keys-${network}"`,
// ── Derive the REAL minimum stake LIVE from chain. Never hardcode it. ──────
// The WorkerRegistry predeploy points at AIConfig, which holds the canonical
// getMinWorkerStake(). cast prints "<wei> [sci-notation]" so take field 1.
// Falls back to the build-time NETWORKS value only if the read fails (network
// hiccup) so a transient RPC blip can't brick the install.
`MIN_FALLBACK_WEI="$(python3 -c 'print(${minStake} * 10**18)')"`,
`AICFG_ADDR="$(cast call "${workerRegistry}" 'aiConfig()(address)' --rpc-url "${rpc}" 2>/dev/null | awk '{print $1}')"`,
`MIN_STAKE_WEI="$(cast call "$AICFG_ADDR" 'getMinWorkerStake()(uint256)' --rpc-url "${rpc}" 2>/dev/null | awk '{print $1}')"`,
'case "${MIN_STAKE_WEI:-}" in ""|*[!0-9]*) MIN_STAKE_WEI="$MIN_FALLBACK_WEI"; echo "⚠ could not read min stake from AIConfig; using fallback";; esac',
// Whole-LCAI stake, the +1 guard threshold, and the funding threshold (stake +
// 0.5 LCAI gas cushion), all computed from the LIVE wei value.
`MIN_STAKE_LCAI="$(python3 -c 'import sys; print(int(sys.argv[1])//10**18)' "$MIN_STAKE_WEI")"`,
`GUARD_LCAI="$(python3 -c 'import sys; print(int(sys.argv[1])//10**18 + 1)' "$MIN_STAKE_WEI")"`,
`THR_WEI="$(python3 -c 'import sys; print(int(sys.argv[1]) + 5*10**17)' "$MIN_STAKE_WEI")"`,
`echo "✓ min stake (live from AIConfig): $MIN_STAKE_LCAI LCAI"`,
// The toolkit prints a hardcoded "STAKE 50,000 LCAI" line; say "the network
// minimum" so it's honest on every network.
`sed -i.bak "s/STAKE 50,000 LCAI/STAKE the network minimum/g" 07-register.sh && rm -f 07-register.sh.bak`,
// The toolkit's 07-register pre-flight balance guard is hardcoded to the
// MAINNET stake ("Worker has less than 50,001 LCAI", and a `b < 50001` /
// `-lt 50001` test), so a correctly funded testnet worker wrongly fails it and
// never reaches the real register tx. Rewrite the threshold to the LIVE
// minimum + 1. Patch both literal forms (the "50,001" display string and the
// bare 50001 used in the test); 50001 only ever appears as this threshold, so a
// global replace is safe. No \\b word boundary (BSD sed lacks it).
'sed -i.bak "s/50,001/$GUARD_LCAI/g; s/50001/$GUARD_LCAI/g" 07-register.sh && rm -f 07-register.sh.bak',
'echo "✓ register pre-flight threshold set to the live minimum ($MIN_STAKE_LCAI LCAI + gas)"',
`echo "▶ funding worker: send to $WORKER_ADDR"`,
// Eligibility on the registry is per (worker, modelId) - one id can NEVER
// speak for the set. The old code hashed only `cut -d, -f1` of SUPPORTED_MODELS
// and gated everything on that single id, so a 3-model worker printed
// "✅ worker online" while models 2 and 3 were unknown to the registry: they
// won no jobs, and re-installing could not heal them either, because every
// short-circuit above saw model #1 eligible and skipped straight past the add.
// So: check EVERY selected model, and leave the failures in ME_MISSING so the
// caller can name them. Returns non-zero unless the whole set verifies.
[
"models_eligible() {",
' ME_MISSING=""',
` for ME_M in ${shellList}; do`,
// keccak of the EXACT on-chain name - the same preimage the worker advertises.
` ME_ID="$(cast keccak "$ME_M" 2>/dev/null | tr -d '\\r\\n')"`,
` if [ -z "$ME_ID" ] || ! cast call "${workerRegistry}" "isEligible(address,bytes32)(bool)" "$WORKER_ADDR" "$ME_ID" --rpc-url "${rpc}" 2>/dev/null | grep -qi true; then ME_MISSING="$ME_MISSING $ME_M"; fi`,
" done",
' [ -z "$ME_MISSING" ]',
"}",
].join("\n"),
// This machine runs ONE worker container at a time. If a container for THIS
// network is already running AND the worker is genuinely live on-chain
// (registered + eligible for EVERY selected model), there's nothing to do. If
// it's for a DIFFERENT network, stop it and carry on (phase 08 recreates it).
// CRITICAL: a running container does NOT mean a working worker - a prior
// install can leave the container Up while the on-chain register/add-model
// failed (e.g. the daemon's add-model OutOfGas bug), so the worker is staked
// but serving nothing. We must verify on-chain before declaring "online",
// otherwise we'd falsely report success and skip the re-register that fixes it.
`if docker ps --format '{{.Names}} {{.Status}}' 2>/dev/null | grep -qE '^lightchain-worker Up'; then RUNCHAIN="$(docker inspect lightchain-worker --format '{{range .Config.Env}}{{println .}}{{end}}' 2>/dev/null | grep '^CHAIN_ID=' | head -1 | cut -d= -f2)"; if [ -n "$RUNCHAIN" ] && [ "$RUNCHAIN" != "${chainId}" ]; then echo "▶ a worker for the other network (chain $RUNCHAIN) is running; this machine runs one at a time. Stopping it to install ${network} (chain ${chainId}). Its stake + keys stay intact - reinstall that network to bring it back."; docker stop lightchain-worker >/dev/null 2>&1 || true; else REG_OK="$(cast call "${workerRegistry}" 'isWorkerRegistered(address)(bool)' "$WORKER_ADDR" --rpc-url "${rpc}" 2>/dev/null | awk '{print $1}')"; ELIG_ALL=0; models_eligible || ELIG_ALL=1; if [ "$REG_OK" = "true" ] && [ "$ELIG_ALL" = "0" ]; then echo "✓ worker already running on ${network} and live on-chain (serving every selected model) - nothing to reinstall"; echo "✅ worker online"; exit 0; else echo "▶ a worker container is running but it is not fully live on-chain (registered=$REG_OK, models the registry does not list it as serving:\${ME_MISSING:- none}) - a prior setup left it staked-but-not-serving. Recreating it."; docker stop lightchain-worker >/dev/null 2>&1 || true; fi; fi; fi`,
// A keystore may already exist (our key on a re-run, or a stale key from a
// prior worker). Skip the import if it's already ours; otherwise back up the
// old one (never delete) so our key can be imported.
'KS="${KEYS_DIR:-$HOME/lightchain-worker/keys}/eth-keystore"',
'WADDR="$(printf "%s" "$WORKER_ADDR" | sed "s/^0x//" | tr "A-Z" "a-z")"',
'SKIP_IMPORT=0',
'if [ -d "$KS" ] && [ -n "$(ls -A "$KS" 2>/dev/null)" ]; then if ls "$KS" | grep -qi "$WADDR"; then echo "✓ worker key already imported - skipping import"; SKIP_IMPORT=1; else echo "▶ backing up a previous worker keystore (not deleting)"; mv "$KS" "${KS}.bak-$(date +%s)"; fi; fi',
// The ECDH key (worker-encryption.key) is encrypted with the worker password.
// A leftover from a different worker can't be decrypted with this password, so
// back it up (via a marker recording which worker owns this keys dir) and let
// phase 05 regenerate it for the current worker.
'ENCKEY="$(dirname "$KS")/worker-encryption.key"; SESS="$(dirname "$KS")/session-keys.enc"; MARKER="$(dirname "$KS")/.lightnode-worker"',
// Different worker → back up ALL its password-encrypted state (ECDH + session store).
'if [ "$(cat "$MARKER" 2>/dev/null)" != "$WADDR" ]; then for f in "$ENCKEY" "$SESS"; do [ -f "$f" ] && { echo "▶ backing up old worker state: $(basename "$f")"; mv "$f" "${f}.bak-$(date +%s)"; }; done; fi',
// Even if the marker matches, a session store older than the ECDH key is stale
// (it predates this setup) and was encrypted with a different password.
'if [ -f "$SESS" ] && [ -f "$ENCKEY" ] && [ "$SESS" -ot "$ENCKEY" ]; then echo "▶ stale session store (older than ECDH key) - backing it up"; mv "$SESS" "${SESS}.bak-$(date +%s)"; fi',
'mkdir -p "$(dirname "$MARKER")"; echo "$WADDR" > "$MARKER"',
// The toolkit uses bash 4+ syntax (e.g. ${var,,}); macOS ships bash 3.2. Run
// the phases with a modern bash (install via brew if the system one is old).
'if bash -c "declare -A _t" 2>/dev/null; then RUNBASH=bash; else echo "▶ system bash is too old for the toolkit - installing bash 4+ via brew"; brew install bash >/dev/null 2>&1 || true; RUNBASH="$(brew --prefix 2>/dev/null)/bin/bash"; fi',
'"$RUNBASH" -c "declare -A _t" 2>/dev/null || { echo "⛔ The toolkit needs bash 4+. Run: brew install bash, then retry."; exit 1; }',
'echo "✓ phase shell: $("$RUNBASH" --version | head -1)"',
// ──────────────────────────────────────────────────────────────────────────
// Pre-flight for phase 07-register, in two steps the toolkit can't do for us:
//
// 1. Multi-password keystore resolve. When a previous attempt left a key on
// disk (SKIP_IMPORT=1), the password the user types this session may not
// match the one used originally - in that case the toolkit signs with the
// wrong key and register silently fails. Mirror the settle/deregister/
// withdraw fix: try each saved slot against the keystore, lock onto the
// one that decrypts, and fail clearly (with a pointer to "Recover a
// replaced key") only when no slot works.
//
// 2. Funding gate. The toolkit's 07-register transfers the stake in LCAI
// and pays gas in LCAI. If the wallet is short (or a funding tx is still
// pending) it would otherwise fail with a generic on-chain revert. Wait
// up to ~90s so a just-funded retry just proceeds; fail clearly only when
// the wallet is genuinely empty after the wait.
// ──────────────────────────────────────────────────────────────────────────
[
"resolve_password() {",
' KSF="$(ls -1 "$KS" 2>/dev/null | head -1)"; [ -z "$KSF" ] && return 0',
' for PW in "${WORKER_PASSWORD:-}" "${WORKER_PASSWORD_ALT1:-}" "${WORKER_PASSWORD_ALT2:-}" "${WORKER_PASSWORD_ALT3:-}"; do',
' [ -z "$PW" ] && continue',
' if cast wallet decrypt-keystore "$KSF" --keystore-dir "$KS" --unsafe-password "$PW" >/dev/null 2>&1; then',
' export WORKER_PASSWORD="$PW"; echo "✓ existing worker keystore unlocked"; return 0',
" fi",
" done",
" return 1",
"}",
].join("\n"),
[
"gate_funding() {",
" GATE_LCAI=0",
" for w in $(seq 1 18); do",
` BAL_HEX="$(curl -s -m 5 -X POST -H 'Content-Type: application/json' --data '{"jsonrpc":"2.0","method":"eth_getBalance","params":["'"$WORKER_ADDR"'","latest"],"id":1}' '${rpc}' | sed -nE 's/.*"result":"(0x[0-9a-fA-F]+)".*/\\1/p')"`,
` BAL_WEI="$(python3 -c 'import sys; print(int(sys.argv[1] or "0x0", 16))' "\${BAL_HEX:-0x0}" 2>/dev/null || echo 0)"`,
` GATE_LCAI="$(python3 -c 'import sys; print(round(int(sys.argv[1])/10**18,3))' "$BAL_WEI" 2>/dev/null || echo 0)"`,
` if python3 -c 'import sys; sys.exit(0 if int(sys.argv[1])>=int(sys.argv[2]) else 1)' "$BAL_WEI" "$THR_WEI"; then echo "✓ worker wallet funded ($GATE_LCAI LCAI)"; return 0; fi`,
` if [ "$w" = "1" ] || [ "$(($w % 6))" = "0" ]; then echo "▶ waiting for funding: worker wallet at $WORKER_ADDR has $GATE_LCAI LCAI, needs at least $MIN_STAKE_LCAI.5 LCAI (stake + a small gas cushion)"; fi`,
" sleep 5",
" done",
` echo "⛔ funding-gate timeout: worker wallet at $WORKER_ADDR still has only $GATE_LCAI LCAI. Send at least $MIN_STAKE_LCAI.5 LCAI to that address (see ${explorer}/address/$WORKER_ADDR) and run install again - your existing setup is reused."`,
" return 1",
"}",
].join("\n"),
'if [ "$SKIP_IMPORT" = "1" ]; then',
" if ! resolve_password; then",
// The on-disk keystore was encrypted with a password none of the saved slots
// match. If the app still holds this worker's raw key (WORKER_PRIVKEY), we