From c9b1d81dad90626af5e2a739937f9c6e571a54fd Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:36:35 +0200 Subject: [PATCH 1/9] distribute: internal-only TestFlight groups and ToInternalGroup SubmitTestFlight gains Internal, which refuses an external group before anything changes and creates missing names internal; ToInternalGroup checks the group, uploads with wait and adds the build. release.Options passes Internal through as InternalGroups. --- internal/distribute/internalgroup.go | 59 +++++++++++++++++ internal/distribute/internalgroup_test.go | 78 +++++++++++++++++++++++ internal/distribute/testflight.go | 36 +++++++++-- internal/release/release.go | 5 +- 4 files changed, 172 insertions(+), 6 deletions(-) create mode 100644 internal/distribute/internalgroup.go create mode 100644 internal/distribute/internalgroup_test.go diff --git a/internal/distribute/internalgroup.go b/internal/distribute/internalgroup.go new file mode 100644 index 0000000..fd73954 --- /dev/null +++ b/internal/distribute/internalgroup.go @@ -0,0 +1,59 @@ +package distribute + +import ( + "context" + "io" + "time" + + "github.com/MobAI-App/ios-builder/internal/asc" + "github.com/MobAI-App/ios-builder/internal/ipa" +) + +// InternalGroupOptions configures ToInternalGroup. +type InternalGroupOptions struct { + IPAPath string + // Group is the internal TestFlight group; a missing one is created. + Group string + Notes string + NoEncryption bool + PollInterval time.Duration + Log io.Writer +} + +// InternalGroupResult is what ToInternalGroup reports. +type InternalGroupResult struct { + Upload *UploadResult `json:"upload"` + TestFlight *TestFlightResult `json:"testflight,omitempty"` +} + +// ToInternalGroup uploads an App Store signed IPA, waits until App Store +// Connect has processed it and adds it to an internal TestFlight group, whose +// testers install it from the TestFlight app without beta review. The group +// is checked before the upload, so an external one costs nothing. +func ToInternalGroup(ctx context.Context, client *asc.Client, opts *InternalGroupOptions) (*InternalGroupResult, error) { + info, err := ipa.ReadInfo(opts.IPAPath) + if err != nil { + return nil, err + } + app, err := client.AppByBundleID(ctx, info.BundleID) + if err != nil { + return nil, err + } + groups, err := client.ListBetaGroups(ctx, app.ID) + if err != nil { + return nil, err + } + if err := refuseExternal(groups, opts.Group); err != nil { + return nil, err + } + res := &InternalGroupResult{} + res.Upload, err = Upload(ctx, client, &UploadOptions{IPAPath: opts.IPAPath, Wait: true, NoEncryption: opts.NoEncryption, PollInterval: opts.PollInterval, Log: opts.Log}) + if err != nil { + return res, err + } + res.TestFlight, err = SubmitTestFlight(ctx, client, &TestFlightOptions{ + BundleID: info.BundleID, Version: info.Version, BuildNumber: info.BuildNumber, Groups: []string{opts.Group}, Internal: true, + Notes: opts.Notes, NoEncryption: opts.NoEncryption, PollInterval: opts.PollInterval, Log: opts.Log, + }) + return res, err +} diff --git a/internal/distribute/internalgroup_test.go b/internal/distribute/internalgroup_test.go new file mode 100644 index 0000000..a925089 --- /dev/null +++ b/internal/distribute/internalgroup_test.go @@ -0,0 +1,78 @@ +package distribute + +import ( + "bytes" + "context" + "strings" + "testing" + "time" +) + +func TestToInternalGroupUploadsWaitsAndAdds(t *testing.T) { + f := newFake(t) + var log bytes.Buffer + res, err := ToInternalGroup(context.Background(), f.client(t), &InternalGroupOptions{IPAPath: writeIPA(t, plistExempt), Group: "team", Notes: "try it", PollInterval: time.Millisecond, Log: &log}) + if err != nil { + t.Fatalf("%v\n%s", err, log.String()) + } + if res.Upload == nil || res.Upload.Build == nil || res.Upload.Build.ID != "build-9" { + t.Errorf("upload = %+v", res.Upload) + } + tf := res.TestFlight + if tf == nil || len(tf.Groups) != 1 || tf.Groups[0].ID != "g-int" || !tf.Groups[0].Internal || tf.BetaReview != nil || tf.Notes != "try it" { + t.Fatalf("testflight = %+v", tf) + } + links := arr(t, f.body("POST /v1/builds/build-9/relationships/betaGroups"), "data") + if len(links) != 1 || obj(t, links[0])["id"] != "g-int" { + t.Errorf("linkage = %v", links) + } + if f.called("POST /v1/betaAppReviewSubmissions") { + t.Error("an internal group needs no beta review") + } +} + +func TestToInternalGroupCreatesMissingGroupInternal(t *testing.T) { + f := newFake(t) + res, err := ToInternalGroup(context.Background(), f.client(t), &InternalGroupOptions{IPAPath: writeIPA(t, plistExempt), Group: "Phones", PollInterval: time.Millisecond}) + if err != nil { + t.Fatal(err) + } + if attrs := obj(t, f.body("POST /v1/betaGroups"), "data", "attributes"); attrs["isInternalGroup"] != true || !res.TestFlight.Groups[0].Created { + t.Errorf("attrs = %v, groups = %+v", attrs, res.TestFlight.Groups) + } +} + +// An external group is refused before the IPA goes anywhere. +func TestToInternalGroupRefusesExternalBeforeUpload(t *testing.T) { + f := newFake(t) + _, err := ToInternalGroup(context.Background(), f.client(t), &InternalGroupOptions{IPAPath: writeIPA(t, plistExempt), Group: "Beta Testers", PollInterval: time.Millisecond}) + if err == nil || !strings.Contains(err.Error(), "is external") { + t.Fatalf("err = %v", err) + } + if f.called("POST /v1/buildUploads") { + t.Errorf("uploaded anyway: %v", f.calls) + } +} + +// SubmitTestFlight with Internal refuses an external group before compliance, +// notes or group changes, and creates missing groups internal even with External. +func TestSubmitTestFlightInternalOnly(t *testing.T) { + f := newFake(t) + _, err := SubmitTestFlight(context.Background(), f.client(t), &TestFlightOptions{BundleID: "com.example.app", Groups: []string{"Team", "beta testers"}, Internal: true, Notes: "n", NoEncryption: true}) + if err == nil || !strings.Contains(err.Error(), "Beta Testers is external") { + t.Fatalf("err = %v", err) + } + for _, c := range f.calls { + if !strings.HasPrefix(c, "GET ") { + t.Errorf("changed something before refusing: %s", c) + } + } + f = newFake(t) + res, err := SubmitTestFlight(context.Background(), f.client(t), &TestFlightOptions{BundleID: "com.example.app", Groups: []string{"New"}, Internal: true, External: true, NoEncryption: true}) + if err != nil { + t.Fatal(err) + } + if !res.Groups[0].Internal || res.BetaReview != nil { + t.Errorf("groups = %+v", res.Groups) + } +} diff --git a/internal/distribute/testflight.go b/internal/distribute/testflight.go index cd51e42..73cfd47 100644 --- a/internal/distribute/testflight.go +++ b/internal/distribute/testflight.go @@ -21,6 +21,10 @@ type TestFlightOptions struct { // nowhere and reports the available groups instead. Groups []string External bool + // Internal restricts Groups to internal ones: an existing external group + // is refused before anything changes, and missing names are created + // internal whatever External says. + Internal bool // Notes is the "What to Test" text; Locale defaults to the app's primary locale. Notes string Locale string @@ -69,6 +73,17 @@ func SubmitTestFlight(ctx context.Context, client *asc.Client, opts *TestFlightO if err != nil { return nil, err } + groups, err := client.ListBetaGroups(ctx, app.ID) + if err != nil { + return nil, err + } + if opts.Internal { + for _, name := range opts.Groups { + if err := refuseExternal(groups, name); err != nil { + return nil, err + } + } + } build, err := pickBuild(ctx, client, app.ID, opts.Version, opts.BuildNumber) if err != nil { return nil, err @@ -97,10 +112,6 @@ func SubmitTestFlight(ctx context.Context, client *asc.Client, opts *TestFlightO logf(opts.Log, "Set What to Test (%s)", locale) } - groups, err := client.ListBetaGroups(ctx, app.ID) - if err != nil { - return res, err - } if len(opts.Groups) == 0 { for _, g := range groups { res.AvailableGroups = append(res.AvailableGroups, GroupRef{ID: g.ID, Name: g.Name, Internal: g.Internal}) @@ -123,7 +134,7 @@ func SubmitTestFlight(ctx context.Context, client *asc.Client, opts *TestFlightO var ids, names []string var external bool for _, name := range opts.Groups { - g, err := findOrCreateGroup(ctx, client, opts.Log, app.ID, groups, name, !opts.External) + g, err := findOrCreateGroup(ctx, client, opts.Log, app.ID, groups, name, opts.Internal || !opts.External) if err != nil { return res, err } @@ -184,6 +195,21 @@ func SubmitTestFlight(ctx context.Context, client *asc.Client, opts *TestFlightO return res, nil } +// refuseExternal fails when name matches an external group: those need beta +// review and reach testers outside the team, which an install on one's own +// device must not trigger. A name that matches nothing is fine (it is created +// internal). +func refuseExternal(groups []asc.BetaGroup, name string) error { + g, err := asc.MatchBetaGroup(groups, name) + if err != nil { + return err + } + if g != nil && !g.Internal { + return fmt.Errorf("TestFlight group %s is external, which needs beta review; pick an internal group (team members only) or a new name, which is created internal", g.Name) + } + return nil +} + func groupKind(internal bool) string { if internal { return "internal" diff --git a/internal/release/release.go b/internal/release/release.go index 680fcd3..12fc25f 100644 --- a/internal/release/release.go +++ b/internal/release/release.go @@ -41,6 +41,9 @@ type Options struct { ReleaseType string Groups []string Notes string + // InternalGroups refuses external Groups (ios build --distribute + // --backend testflight); missing ones are created internal. + InternalGroups bool // NoEncryption answers export compliance with "no" when still unanswered. NoEncryption bool PollInterval time.Duration @@ -205,7 +208,7 @@ func Run(ctx context.Context, cfg *config.Config, builder Builder, client *asc.C return res, err } tf, err := distribute.SubmitTestFlight(ctx, client, &distribute.TestFlightOptions{ - BundleID: bundleID, Version: info.Version, BuildNumber: res.BuildNumber, Groups: opts.Groups, Notes: opts.Notes, + BundleID: bundleID, Version: info.Version, BuildNumber: res.BuildNumber, Groups: opts.Groups, Notes: opts.Notes, Internal: opts.InternalGroups, NoEncryption: opts.NoEncryption, PollInterval: opts.PollInterval, Log: opts.Log, }) if tf != nil { From 3ad76b18adbf4ba1343bc62ee7417a1b2ea57559 Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:43:42 +0200 Subject: [PATCH 2/9] otainstall: backend names and backend-aware IPA and profile checks distribute.backend in builder.json (with the bucket, account and group settings) and BackendName pick github, s3, azure or testflight. Inspect and CheckDistribution take the backend: the over-the-air ones keep refusing App Store builds, testflight refuses everything else. --- cmd/builder/distribute.go | 2 +- cmd/builder/root.go | 2 +- internal/config/types.go | 24 +++++++++ internal/otainstall/app.go | 48 ++++++++++++----- internal/otainstall/app_test.go | 84 +++++++++++++++++++++++++----- internal/otainstall/backend.go | 32 ++++++++++++ internal/otainstall/github_test.go | 2 +- 7 files changed, 166 insertions(+), 28 deletions(-) diff --git a/cmd/builder/distribute.go b/cmd/builder/distribute.go index dd06ddc..ce924e7 100644 --- a/cmd/builder/distribute.go +++ b/cmd/builder/distribute.go @@ -106,7 +106,7 @@ func distributeBackend(cfg *config.Config) (otainstall.Backend, error) { // runDistribute is the flow behind `ios distribute` and `ios build --distribute`. func runDistribute(cmd *cobra.Command, cfg *config.Config, ipaPath string) error { - app, err := otainstall.Inspect(ipaPath) + app, err := otainstall.Inspect(ipaPath, otainstall.BackendGitHub) if err != nil { return err } diff --git a/cmd/builder/root.go b/cmd/builder/root.go index 0dcd66e..74d2d49 100644 --- a/cmd/builder/root.go +++ b/cmd/builder/root.go @@ -692,7 +692,7 @@ func runIOSBuild(cmd *cobra.Command, args []string) error { if err != nil { return err } - if err := otainstall.CheckDistribution(&s); err != nil { + if err := otainstall.CheckDistribution(&s, otainstall.BackendGitHub); err != nil { return err } } diff --git a/internal/config/types.go b/internal/config/types.go index 2d38ec0..82a89bc 100644 --- a/internal/config/types.go +++ b/internal/config/types.go @@ -25,6 +25,30 @@ type Config struct { // runs have no flags, so it is also the only way they can select a profile. DefaultProfile string `json:"defaultProfile,omitempty"` Profiles map[string]Profile `json:"profiles,omitempty"` + // Distribute configures where `ios distribute` puts the IPA; nil is the + // GitHub backend. + Distribute *DistributeConfig `json:"distribute,omitempty"` +} + +// DistributeConfig selects and configures the `ios distribute` backend. +// Credentials never live here: they come from the environment (AWS_*, +// AZURE_STORAGE_*) or the App Store Connect key. +type DistributeConfig struct { + // Backend is github (default), s3, azure or testflight; --backend wins. + Backend string `json:"backend,omitempty"` + // Bucket, Region, Endpoint: the s3 backend. Endpoint is for S3-compatible + // stores (Cloudflare R2, MinIO, Google Cloud Storage), addressed path-style. + Bucket string `json:"bucket,omitempty"` + Region string `json:"region,omitempty"` + Endpoint string `json:"endpoint,omitempty"` + // Prefix is put before every object (s3) or blob (azure) name. + Prefix string `json:"prefix,omitempty"` + // Account and Container: the azure backend (Endpoint overrides the + // account's blob endpoint, e.g. for Azurite). + Account string `json:"account,omitempty"` + Container string `json:"container,omitempty"` + // Group is the internal TestFlight group of the testflight backend; --group wins. + Group string `json:"group,omitempty"` } // SigningConfig is where the signing material lives on this machine. diff --git a/internal/otainstall/app.go b/internal/otainstall/app.go index 843b72b..86f1a8d 100644 --- a/internal/otainstall/app.go +++ b/internal/otainstall/app.go @@ -38,14 +38,20 @@ func (p Profile) String() string { return p.Type } -// Inspect reads the IPA and refuses one an iPhone cannot install this way. -func Inspect(path string) (*App, error) { +// Inspect reads the IPA and refuses one the backend cannot deliver: the +// over-the-air backends need a development, ad-hoc or enterprise signature; +// testflight needs the opposite, an App Store one. +func Inspect(path, backend string) (*App, error) { info, err := ipa.ReadInfo(path) if err != nil { return nil, err } + testflight := backend == BackendTestFlight profile, err := ipa.ReadProfile(path) if errors.Is(err, ipa.ErrUnsigned) { + if testflight { + return nil, fmt.Errorf("%s is unsigned; TestFlight needs an App Store signed IPA (builder ios build --profile )", path) + } return nil, fmt.Errorf("%s is unsigned; build with a profile whose distribution is development, ad-hoc or enterprise (builder ios build --profile )", path) } if err != nil { @@ -55,27 +61,43 @@ func Inspect(path string) (*App, error) { if err != nil { return nil, fmt.Errorf("%s: %w", path, err) } - if typ == signing.TypeStore { - return nil, fmt.Errorf("%s is signed for the App Store, which cannot be installed over the air; use TestFlight (builder ios release) or build with a development or ad-hoc profile", path) + app := &App{Path: path, BundleID: info.BundleID, Version: info.Version, Build: info.BuildNumber, Title: info.Name(), Profile: Profile{Type: string(typ)}} + switch { + case testflight && typ != signing.TypeStore: + return nil, fmt.Errorf("%s is signed for %s, but TestFlight takes only App Store signed builds; build with a store profile (builder signing setup --distribution store writes one) or drop --backend testflight to install it over the air", path, typ) + case testflight: + return app, nil + case typ == signing.TypeStore: + return nil, fmt.Errorf("%s is signed for the App Store, which cannot be installed over the air; use --backend testflight --group , or build with a development or ad-hoc profile", path) } - devices, err := signing.ProfileDevices(profile) - if err != nil { + if app.Profile.Devices, err = signing.ProfileDevices(profile); err != nil { return nil, fmt.Errorf("%s: %w", path, err) } - return &App{ - Path: path, BundleID: info.BundleID, Version: info.Version, Build: info.BuildNumber, Title: info.Name(), - Profile: Profile{Type: string(typ), Devices: devices}, - }, nil + return app, nil } // CheckDistribution is the preflight of `ios build --distribute`: the profile -// must sign for devices, and that is known before anything is pushed. -func CheckDistribution(s *config.BuildSettings) error { +// must sign the way the backend needs, and that is known before anything is +// pushed. For testflight an empty profile passes, since the release preflight +// then picks the only store profile. +func CheckDistribution(s *config.BuildSettings, backend string) error { + if backend == BackendTestFlight { + switch s.Distribution { + case config.DistributionStore: + return nil + case "": + if s.Profile == "" { + return nil + } + return fmt.Errorf("profile %q has no distribution, so the build is unsigned; TestFlight needs \"distribution\": \"store\" (builder signing setup --distribution store)", s.Profile) + } + return fmt.Errorf("profile %q has distribution %s; TestFlight takes only App Store builds, so pass a store profile (builder signing setup --distribution store writes one) or another --backend", s.Profile, s.Distribution) + } switch s.Distribution { case config.DistributionDevelopment, config.DistributionAdHoc, config.DistributionEnterprise: return nil case config.DistributionStore: - return fmt.Errorf("profile %q has distribution store; an App Store build cannot be installed over the air. Use TestFlight (builder ios release) or a development or ad-hoc profile (builder signing setup --devices-from-mobai writes one)", s.Profile) + return fmt.Errorf("profile %q has distribution store; an App Store build cannot be installed over the air. Use --backend testflight --group , or a development or ad-hoc profile (builder signing setup --devices-from-mobai writes one)", s.Profile) } if s.Profile == "" { return errors.New("--distribute needs a signed build: pass --profile with a development, ad-hoc or enterprise profile (builder signing setup --devices-from-mobai writes one)") diff --git a/internal/otainstall/app_test.go b/internal/otainstall/app_test.go index cdb0f21..77303d2 100644 --- a/internal/otainstall/app_test.go +++ b/internal/otainstall/app_test.go @@ -65,7 +65,7 @@ func TestInspect(t *testing.T) { {"development", twoDevices + taskAllow, Profile{Type: config.DistributionDevelopment, Devices: 2}, ""}, {"ad-hoc", twoDevices + noTaskAllow, Profile{Type: config.DistributionAdHoc, Devices: 2}, ""}, {"enterprise", `ProvisionsAllDevices` + noTaskAllow, Profile{Type: config.DistributionEnterprise}, ""}, - {"store", noTaskAllow, Profile{}, "signed for the App Store"}, + {"store", noTaskAllow, Profile{}, "use --backend testflight"}, {"unsigned", "", Profile{}, "is unsigned"}, } { t.Run(tc.name, func(t *testing.T) { @@ -75,7 +75,7 @@ func TestInspect(t *testing.T) { // An embedded framework's profile must not be the one read. entries["Payload/App.app/Frameworks/X.framework/embedded.mobileprovision"] = mobileprovision(noTaskAllow) } - app, err := Inspect(writeIPA(t, entries)) + app, err := Inspect(writeIPA(t, entries), BackendS3) if tc.wantErr != "" { if err == nil || !strings.Contains(err.Error(), tc.wantErr) { t.Fatalf("err = %v, want %q", err, tc.wantErr) @@ -92,6 +92,61 @@ func TestInspect(t *testing.T) { } } +// The testflight backend wants the opposite signature of the OTA ones. +func TestInspectForTestFlight(t *testing.T) { + for _, tc := range []struct { + name, profile, wantErr string + }{ + {"store", noTaskAllow, ""}, + {"development", twoDevices + taskAllow, "signed for development, but TestFlight takes only App Store"}, + {"ad-hoc", twoDevices + noTaskAllow, "signed for ad-hoc"}, + {"enterprise", `ProvisionsAllDevices` + noTaskAllow, "signed for enterprise"}, + {"unsigned", "", "TestFlight needs an App Store signed IPA"}, + } { + t.Run(tc.name, func(t *testing.T) { + entries := map[string]string{"Payload/App.app/Info.plist": appPlist} + if tc.profile != "" { + entries["Payload/App.app/embedded.mobileprovision"] = mobileprovision(tc.profile) + } + app, err := Inspect(writeIPA(t, entries), BackendTestFlight) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatal(err) + } + if app.Profile.Type != config.DistributionStore || app.BundleID != "run.mobai.tapdash" { + t.Errorf("app = %+v", app) + } + }) + } +} + +func TestBackendName(t *testing.T) { + cfg := &config.Config{Distribute: &config.DistributeConfig{Backend: "s3"}} + for _, tc := range []struct { + flag string + cfg *config.Config + want string + }{ + {"", nil, BackendGitHub}, + {"", &config.Config{}, BackendGitHub}, + {"", cfg, BackendS3}, + {"azure", cfg, BackendAzure}, + {"testflight", nil, BackendTestFlight}, + } { + if got, err := BackendName(tc.flag, tc.cfg); err != nil || got != tc.want { + t.Errorf("BackendName(%q) = %q, %v; want %q", tc.flag, got, err, tc.want) + } + } + if _, err := BackendName("mobai", nil); err == nil || !strings.Contains(err.Error(), "unknown distribute backend") { + t.Errorf("err = %v", err) + } +} + func TestProfileString(t *testing.T) { if got := (Profile{Type: "development", Devices: 2}).String(); got != "development, 2 device(s)" { t.Errorf("got %q", got) @@ -103,21 +158,26 @@ func TestProfileString(t *testing.T) { func TestCheckDistribution(t *testing.T) { for _, tc := range []struct { - profile, distribution, wantErr string + backend, profile, distribution, wantErr string }{ - {"dev", config.DistributionDevelopment, ""}, - {"internal", config.DistributionAdHoc, ""}, - {"inhouse", config.DistributionEnterprise, ""}, - {"store", config.DistributionStore, `profile "store" has distribution store`}, - {"plain", "", `profile "plain" has no distribution`}, - {"", "", "pass --profile"}, + {BackendGitHub, "dev", config.DistributionDevelopment, ""}, + {BackendS3, "internal", config.DistributionAdHoc, ""}, + {BackendAzure, "inhouse", config.DistributionEnterprise, ""}, + {BackendGitHub, "store", config.DistributionStore, `profile "store" has distribution store`}, + {BackendGitHub, "plain", "", `profile "plain" has no distribution`}, + {BackendS3, "", "", "pass --profile"}, + {BackendTestFlight, "store", config.DistributionStore, ""}, + {BackendTestFlight, "", "", ""}, + {BackendTestFlight, "dev", config.DistributionDevelopment, "TestFlight takes only App Store builds"}, + {BackendTestFlight, "inhouse", config.DistributionEnterprise, "TestFlight takes only App Store builds"}, + {BackendTestFlight, "plain", "", `profile "plain" has no distribution`}, } { - err := CheckDistribution(&config.BuildSettings{Profile: tc.profile, Distribution: tc.distribution}) + err := CheckDistribution(&config.BuildSettings{Profile: tc.profile, Distribution: tc.distribution}, tc.backend) if tc.wantErr == "" && err != nil { - t.Errorf("%s/%s: %v", tc.profile, tc.distribution, err) + t.Errorf("%s %s/%s: %v", tc.backend, tc.profile, tc.distribution, err) } if tc.wantErr != "" && (err == nil || !strings.Contains(err.Error(), tc.wantErr)) { - t.Errorf("%s/%s: err = %v, want %q", tc.profile, tc.distribution, err, tc.wantErr) + t.Errorf("%s %s/%s: err = %v, want %q", tc.backend, tc.profile, tc.distribution, err, tc.wantErr) } } } diff --git a/internal/otainstall/backend.go b/internal/otainstall/backend.go index 0512605..6424efe 100644 --- a/internal/otainstall/backend.go +++ b/internal/otainstall/backend.go @@ -2,9 +2,39 @@ package otainstall import ( "context" + "fmt" "time" + + "github.com/MobAI-App/ios-builder/internal/config" +) + +// Backend names, for --backend and distribute.backend in builder.json. +// github, s3 and azure implement Backend; testflight is not an over-the-air +// install and goes through App Store Connect instead (see Inspect and +// CheckDistribution). A hosted short-link service would be one more Backend: +// Upload stores the IPA, Mint returns its short manifest URL. +const ( + BackendGitHub = "github" + BackendS3 = "s3" + BackendAzure = "azure" + BackendTestFlight = "testflight" ) +// BackendName is flag, else distribute.backend in builder.json, else github. +func BackendName(flag string, cfg *config.Config) (string, error) { + name := flag + if name == "" && cfg != nil && cfg.Distribute != nil { + name = cfg.Distribute.Backend + } + switch name { + case "": + return BackendGitHub, nil + case BackendGitHub, BackendS3, BackendAzure, BackendTestFlight: + return name, nil + } + return "", fmt.Errorf("unknown distribute backend %q (choose github, s3, azure or testflight)", name) +} + // Backend stores the IPA and the manifest where an iPhone can fetch them. type Backend interface { // Upload stores the IPA once for the session. @@ -32,4 +62,6 @@ type Links struct { ExpiresAt time.Time `json:"expires_at"` ReleaseID int64 `json:"release_id,omitempty"` GistID string `json:"gist_id,omitempty"` + // Objects names the IPA and the manifest in a bucket backend (s3://…, azure://…). + Objects []string `json:"objects,omitempty"` } diff --git a/internal/otainstall/github_test.go b/internal/otainstall/github_test.go index e4276cf..e9c5607 100644 --- a/internal/otainstall/github_test.go +++ b/internal/otainstall/github_test.go @@ -179,7 +179,7 @@ func testApp(t *testing.T) *App { "Payload/App.app/Info.plist": appPlist, "Payload/App.app/embedded.mobileprovision": mobileprovision(twoDevices + taskAllow), }) - app, err := Inspect(path) + app, err := Inspect(path, BackendGitHub) if err != nil { t.Fatal(err) } From 046f0bf22566da62bfeb277a2a690cff83504bff Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:44:05 +0200 Subject: [PATCH 3/9] otainstall: s3 backend for S3 and S3-compatible buckets The IPA and a short m.plist go under ios-builder// with marker metadata and are linked by SigV4 presigned GET URLs (query presign written here, checked against AWS's published vectors), valid for --ttl up to seven days. Each mint presigns the IPA again and rewrites the manifest in place. Cleanup lists the upload folder and deletes only objects whose HEAD shows the marker. Credentials come from AWS_* or the shared credentials file; R2, MinIO and GCS go through distribute.endpoint, path-style. A presigned manifest URL makes a version 13 code (69 modules) against the gist's version 6; temporary credentials push it to ~105, and the session says so when a code is wider than 80 columns. --- internal/otainstall/bucket.go | 193 +++++++++++++ internal/otainstall/s3.go | 321 +++++++++++++++++++++ internal/otainstall/s3_test.go | 455 ++++++++++++++++++++++++++++++ internal/otainstall/session.go | 8 + internal/otainstall/sigv4.go | 177 ++++++++++++ internal/otainstall/sigv4_test.go | 98 +++++++ 6 files changed, 1252 insertions(+) create mode 100644 internal/otainstall/bucket.go create mode 100644 internal/otainstall/s3.go create mode 100644 internal/otainstall/s3_test.go create mode 100644 internal/otainstall/sigv4.go create mode 100644 internal/otainstall/sigv4_test.go diff --git a/internal/otainstall/bucket.go b/internal/otainstall/bucket.go new file mode 100644 index 0000000..854fac7 --- /dev/null +++ b/internal/otainstall/bucket.go @@ -0,0 +1,193 @@ +package otainstall + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/google/uuid" +) + +const ( + // BucketDir is the folder, under the configured prefix, that holds every + // upload; cleanup only looks there and only deletes objects that also + // carry the marker metadata. + BucketDir = "ios-builder/" + // manifestObject is short on purpose: the manifest URL is the QR code. + manifestObject = "m.plist" + // DefaultBucketTTL is how long a bucket link lives without --ttl. + DefaultBucketTTL = time.Hour + // MinTTL keeps the refresh (a minute before expiry) from spinning. + MinTTL = 2 * time.Minute + // MaxTTL is the longest SigV4 presigned URL; azure keeps the same cap. + MaxTTL = 7 * 24 * time.Hour +) + +// objectStore is the little a bucket backend needs from S3 or Azure Blob. +type objectStore interface { + // put stores body under key with Builder's marker metadata. + put(ctx context.Context, key, contentType string, body io.Reader, size int64) error + // delete removes key; a missing key is not an error. + delete(ctx context.Context, key string) error + // listMarked lists the keys under prefix that carry Builder's marker. + listMarked(ctx context.Context, prefix string) ([]string, error) + // presign is a GET URL for key that needs no credentials until now+ttl. + presign(key string, ttl time.Duration, now time.Time) (string, error) + // name is how a key is shown to the user (s3://bucket/key). + name(key string) string +} + +// Bucket keeps the IPA and the manifest as objects under one upload folder +// and links them with presigned URLs, which live up to seven days. +type Bucket struct { + store objectStore + prefix string + ttl time.Duration + now func() time.Time +} + +// CheckTTL refuses a link lifetime a bucket backend cannot give. +func CheckTTL(ttl time.Duration) error { + if ttl < MinTTL || ttl > MaxTTL { + return fmt.Errorf("--ttl must be between %s and 168h (seven days, the longest an S3 presigned URL lives), got %s", MinTTL, ttl) + } + return nil +} + +func newBucket(store objectStore, prefix string, ttl time.Duration) (*Bucket, error) { + if ttl == 0 { + ttl = DefaultBucketTTL + } + if err := CheckTTL(ttl); err != nil { + return nil, err + } + prefix = strings.TrimLeft(prefix, "/") + if prefix != "" && !strings.HasSuffix(prefix, "/") { + prefix += "/" + } + return &Bucket{store: store, prefix: prefix, ttl: ttl, now: time.Now}, nil +} + +type bucketUpload struct { + b *Bucket + // remaining are the keys Close still has to delete, IPA first. + remaining []string + ipaKey string + manifestKey string +} + +func (b *Bucket) Upload(ctx context.Context, app *App, progress func(done, total int64)) (Upload, error) { + f, err := os.Open(app.Path) + if err != nil { + return nil, err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return nil, err + } + dir := b.prefix + BucketDir + uuid.NewString()[:8] + "/" + up := &bucketUpload{b: b, ipaKey: dir + assetName(app.Title), manifestKey: dir + manifestObject} + var body io.Reader = f + if progress != nil { + body = &progressReader{r: f, total: st.Size(), progress: progress} + } + if err := b.store.put(ctx, up.ipaKey, "application/octet-stream", body, st.Size()); err != nil { + // A failed PUT stores nothing, but one cut short by Ctrl-C may have. + dctx, cancel := context.WithTimeout(context.Background(), cleanupTimeout) + defer cancel() + _ = b.store.delete(dctx, up.ipaKey) + return nil, err + } + up.remaining = []string{up.ipaKey} + return up, nil +} + +// Cleanup deletes every marked object under the upload folder. +func (b *Bucket) Cleanup(ctx context.Context) (int, error) { + keys, err := b.store.listMarked(ctx, b.prefix+BucketDir) + if err != nil { + return 0, err + } + n := 0 + for _, k := range keys { + if err := b.store.delete(ctx, k); err != nil { + return n, err + } + n++ + } + return n, nil +} + +// Mint presigns the IPA, writes the manifest naming that URL over the +// previous one (so an older link that is still valid serves the newest +// manifest) and presigns the manifest. +func (u *bucketUpload) Mint(ctx context.Context, build func(ipaURL string) ([]byte, error)) (*Links, error) { + now := u.b.now() + ipaURL, err := u.b.store.presign(u.ipaKey, u.b.ttl, now) + if err != nil { + return nil, err + } + body, err := build(ipaURL) + if err != nil { + return nil, err + } + if err := u.b.store.put(ctx, u.manifestKey, "application/xml", strings.NewReader(string(body)), int64(len(body))); err != nil { + return nil, err + } + if len(u.remaining) == 1 && u.remaining[0] == u.ipaKey { + u.remaining = append(u.remaining, u.manifestKey) + } + manifestURL, err := u.b.store.presign(u.manifestKey, u.b.ttl, now) + if err != nil { + return nil, err + } + return &Links{ + Link: Link(manifestURL), ManifestURL: manifestURL, IPAURL: ipaURL, ExpiresAt: now.Add(u.b.ttl), + Objects: []string{u.b.store.name(u.ipaKey), u.b.store.name(u.manifestKey)}, + }, nil +} + +// Close deletes the manifest and the IPA; what it fails to delete stays in +// Leftovers. +func (u *bucketUpload) Close(ctx context.Context) error { + var errs []error + var left []string + for _, k := range u.remaining { + if err := u.b.store.delete(ctx, k); err != nil { + errs = append(errs, err) + left = append(left, k) + } + } + u.remaining = left + return errors.Join(errs...) +} + +func (u *bucketUpload) Leftovers() []string { + out := make([]string, 0, len(u.remaining)) + for _, k := range u.remaining { + out = append(out, u.b.store.name(k)) + } + return out +} + +// progressReader reports how much of total has been read. +type progressReader struct { + r io.Reader + done int64 + total int64 + progress func(done, total int64) +} + +func (p *progressReader) Read(b []byte) (int, error) { + n, err := p.r.Read(b) + p.done += int64(n) + if n > 0 || errors.Is(err, io.EOF) { + p.progress(p.done, p.total) + } + return n, err +} diff --git a/internal/otainstall/s3.go b/internal/otainstall/s3.go new file mode 100644 index 0000000..0b90e1a --- /dev/null +++ b/internal/otainstall/s3.go @@ -0,0 +1,321 @@ +package otainstall + +import ( + "bufio" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/MobAI-App/ios-builder/internal/config" +) + +const ( + // s3MarkerHeader is the user metadata every Builder object carries; + // cleanup deletes nothing without it. + s3MarkerHeader = "X-Amz-Meta-Ios-Builder" + markerValue = "distribute" +) + +// S3Options configures NewS3. +type S3Options struct { + Bucket string + // Region defaults to us-east-1 (Cloudflare R2 takes auto or us-east-1). + Region string + // Endpoint is an S3-compatible store's URL (R2, MinIO, GCS), addressed + // path-style; empty is AWS, virtual-hosted. + Endpoint string + Prefix string + Credentials AWSCredentials + TTL time.Duration + HTTPClient *http.Client +} + +// NewS3 returns the bucket backend for S3 and S3-compatible stores. +func NewS3(opts *S3Options) (*Bucket, error) { + if opts.Bucket == "" { + return nil, errors.New(`the s3 backend needs a bucket: set "distribute": {"bucket": "..."} in builder.json`) + } + if opts.Credentials.AccessKeyID == "" || opts.Credentials.SecretAccessKey == "" { + return nil, errors.New("the s3 backend needs credentials") + } + region := opts.Region + if region == "" { + region = "us-east-1" + } + base, err := s3BaseURL(opts.Bucket, region, opts.Endpoint) + if err != nil { + return nil, err + } + client := opts.HTTPClient + if client == nil { + client = &http.Client{} + } + store := &s3Store{bucket: opts.Bucket, base: base, signer: &sigv4{creds: opts.Credentials, region: region, service: "s3"}, http: client, now: time.Now} + return newBucket(store, opts.Prefix, opts.TTL) +} + +// s3BaseURL is where keys are appended: https://bucket.s3.region.amazonaws.com/ +// on AWS (path-style when the bucket name has a dot, which TLS would reject +// as a host), endpoint/bucket/ elsewhere. +func s3BaseURL(bucket, region, endpoint string) (string, error) { + if endpoint == "" { + if strings.Contains(bucket, ".") { + return "https://s3." + region + ".amazonaws.com/" + uriEncode(bucket, true) + "/", nil + } + return "https://" + bucket + ".s3." + region + ".amazonaws.com/", nil + } + u, err := url.Parse(endpoint) + if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return "", fmt.Errorf("distribute.endpoint %q is not an http(s) URL", endpoint) + } + return strings.TrimRight(endpoint, "/") + "/" + uriEncode(bucket, true) + "/", nil +} + +type s3Store struct { + bucket string + base string + signer *sigv4 + http *http.Client + now func() time.Time +} + +func (s *s3Store) objectURL(key string) (*url.URL, error) { + return url.Parse(s.base + uriEncode(key, false)) +} + +func (s *s3Store) name(key string) string { return "s3://" + s.bucket + "/" + key } + +func (s *s3Store) do(ctx context.Context, method, rawURL string, header http.Header, body io.Reader, size int64) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, rawURL, body) + if err != nil { + return nil, err + } + for k, v := range header { + req.Header[k] = v + } + payload := emptySHA256 + if body != nil { + req.ContentLength = size + payload = unsignedPayload + } + s.signer.sign(req, payload, s.now()) + return s.http.Do(req) +} + +func (s *s3Store) put(ctx context.Context, key, contentType string, body io.Reader, size int64) error { + u, err := s.objectURL(key) + if err != nil { + return err + } + h := http.Header{"Content-Type": {contentType}, s3MarkerHeader: {markerValue}} + resp, err := s.do(ctx, "PUT", u.String(), h, body, size) + if err != nil { + return err + } + return s3Error(resp, "PUT", s.name(key)) +} + +func (s *s3Store) delete(ctx context.Context, key string) error { + u, err := s.objectURL(key) + if err != nil { + return err + } + resp, err := s.do(ctx, "DELETE", u.String(), nil, nil, 0) + if err != nil { + return err + } + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil + } + return s3Error(resp, "DELETE", s.name(key)) +} + +// listMarked pages through ListObjectsV2 and keeps the keys whose HEAD shows +// the marker; the listing does not carry user metadata. +func (s *s3Store) listMarked(ctx context.Context, prefix string) ([]string, error) { + var keys []string + token := "" + for { + q := url.Values{"list-type": {"2"}, "prefix": {prefix}} + if token != "" { + q.Set("continuation-token", token) + } + resp, err := s.do(ctx, "GET", s.base+"?"+canonicalQuery(q), nil, nil, 0) + if err != nil { + return nil, err + } + var page struct { + Contents []struct { + Key string `xml:"Key"` + } `xml:"Contents"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken"` + } + if err := decodeXML(resp, "list", "s3://"+s.bucket+"/"+prefix, s3Error, &page); err != nil { + return nil, err + } + for _, c := range page.Contents { + marked, err := s.marked(ctx, c.Key) + if err != nil { + return nil, err + } + if marked { + keys = append(keys, c.Key) + } + } + if !page.IsTruncated || page.NextContinuationToken == "" { + return keys, nil + } + token = page.NextContinuationToken + } +} + +func (s *s3Store) marked(ctx context.Context, key string) (bool, error) { + u, err := s.objectURL(key) + if err != nil { + return false, err + } + resp, err := s.do(ctx, "HEAD", u.String(), nil, nil, 0) + if err != nil { + return false, err + } + resp.Body.Close() + switch { + case resp.StatusCode == http.StatusNotFound: + return false, nil + case resp.StatusCode >= 300: + return false, fmt.Errorf("s3 HEAD %s: %s", s.name(key), resp.Status) + } + return resp.Header.Get(s3MarkerHeader) == markerValue, nil +} + +func (s *s3Store) presign(key string, ttl time.Duration, now time.Time) (string, error) { + u, err := s.objectURL(key) + if err != nil { + return "", err + } + return s.signer.presign("GET", u, ttl, now), nil +} + +// s3Error closes resp and turns a non-2xx answer into an error carrying S3's +// code and message (AccessDenied, NoSuchBucket, SignatureDoesNotMatch). +func s3Error(resp *http.Response, op, what string) error { + defer resp.Body.Close() + if resp.StatusCode < 300 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil + } + var e struct { + Code string `xml:"Code"` + Message string `xml:"Message"` + } + data, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if xml.Unmarshal(data, &e) == nil && e.Code != "" { + return fmt.Errorf("s3 %s %s: %s %s: %s", op, what, resp.Status, e.Code, e.Message) + } + return fmt.Errorf("s3 %s %s: %s", op, what, resp.Status) +} + +// decodeXML decodes a 2xx XML body into v, or returns errFn's error. +func decodeXML(resp *http.Response, op, what string, errFn func(*http.Response, string, string) error, v any) error { + if resp.StatusCode >= 300 { + return errFn(resp, op, what) + } + defer resp.Body.Close() + if err := xml.NewDecoder(resp.Body).Decode(v); err != nil { + return fmt.Errorf("%s %s: %w", op, what, err) + } + return nil +} + +// S3FromConfig builds the s3 backend from builder.json and the standard AWS +// environment: AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_SESSION_TOKEN, +// else the AWS_PROFILE (or default) section of ~/.aws/credentials +// (AWS_SHARED_CREDENTIALS_FILE); the region is distribute.region, else +// AWS_REGION, else AWS_DEFAULT_REGION. +func S3FromConfig(cfg *config.DistributeConfig, ttl time.Duration, getenv func(string) string) (*Bucket, error) { + if cfg == nil { + cfg = &config.DistributeConfig{} + } + creds, err := LoadAWSCredentials(getenv) + if err != nil { + return nil, err + } + region := cfg.Region + if region == "" { + region = getenv("AWS_REGION") + } + if region == "" { + region = getenv("AWS_DEFAULT_REGION") + } + return NewS3(&S3Options{Bucket: cfg.Bucket, Region: region, Endpoint: cfg.Endpoint, Prefix: cfg.Prefix, Credentials: creds, TTL: ttl}) +} + +// LoadAWSCredentials reads the environment, then the shared credentials file. +func LoadAWSCredentials(getenv func(string) string) (AWSCredentials, error) { + creds := AWSCredentials{AccessKeyID: getenv("AWS_ACCESS_KEY_ID"), SecretAccessKey: getenv("AWS_SECRET_ACCESS_KEY"), SessionToken: getenv("AWS_SESSION_TOKEN")} + if creds.AccessKeyID != "" && creds.SecretAccessKey != "" { + return creds, nil + } + if creds.AccessKeyID != "" || creds.SecretAccessKey != "" { + return AWSCredentials{}, errors.New("set both AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or neither to use ~/.aws/credentials") + } + path := getenv("AWS_SHARED_CREDENTIALS_FILE") + if path == "" { + home := getenv("HOME") + if home == "" { + home, _ = os.UserHomeDir() + } + path = filepath.Join(home, ".aws", "credentials") + } + profile := getenv("AWS_PROFILE") + if profile == "" { + profile = "default" + } + f, err := os.Open(path) + if err != nil { + return AWSCredentials{}, fmt.Errorf("no AWS credentials: set AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY, or AWS_PROFILE with a section in %s (%v)", path, err) + } + defer f.Close() + section := "" + sc := bufio.NewScanner(f) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" || line[0] == '#' || line[0] == ';' { + continue + } + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + section = strings.TrimSpace(line[1 : len(line)-1]) + continue + } + k, v, ok := strings.Cut(line, "=") + if !ok || section != profile { + continue + } + v = strings.TrimSpace(v) + switch strings.ToLower(strings.TrimSpace(k)) { + case "aws_access_key_id": + creds.AccessKeyID = v + case "aws_secret_access_key": + creds.SecretAccessKey = v + case "aws_session_token": + creds.SessionToken = v + } + } + if err := sc.Err(); err != nil { + return AWSCredentials{}, err + } + if creds.AccessKeyID == "" || creds.SecretAccessKey == "" { + return AWSCredentials{}, fmt.Errorf("no AWS credentials: %s has no aws_access_key_id and aws_secret_access_key in [%s] (SSO and credential_process profiles are not read; export AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY instead)", path, profile) + } + return creds, nil +} diff --git a/internal/otainstall/s3_test.go b/internal/otainstall/s3_test.go new file mode 100644 index 0000000..162d141 --- /dev/null +++ b/internal/otainstall/s3_test.go @@ -0,0 +1,455 @@ +package otainstall + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "html" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path/filepath" + "regexp" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" +) + +var fakeCreds = AWSCredentials{AccessKeyID: "AKIDFAKE", SecretAccessKey: "fake/secret+key", SessionToken: "sess"} + +type fakeObject struct { + data []byte + contentType string + marked bool +} + +// fakeS3 is a path-style S3 for bucket "b" that checks every signature the +// way S3 does: it rebuilds the canonical request from what arrived on the wire. +type fakeS3 struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + objects map[string]*fakeObject + log []string + // putFails answers PUTs with 403 AccessDenied; deleteFails DELETEs with 500. + putFails, deleteFails bool + // pageSize makes listings paginate. + pageSize int +} + +func newFakeS3(t *testing.T) *fakeS3 { + f := &fakeS3{t: t, objects: map[string]*fakeObject{}, pageSize: 2} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeS3) backend(t *testing.T, prefix string, ttl time.Duration) *Bucket { + t.Helper() + b, err := NewS3(&S3Options{Bucket: "b", Region: "auto", Endpoint: f.srv.URL, Prefix: prefix, Credentials: fakeCreds, TTL: ttl, HTTPClient: f.srv.Client()}) + if err != nil { + t.Fatal(err) + } + return b +} + +var authRe = regexp.MustCompile(`^AWS4-HMAC-SHA256 Credential=AKIDFAKE/(\d{8})/auto/s3/aws4_request, SignedHeaders=([a-z0-9;-]+), Signature=[0-9a-f]{64}$`) + +// verify reports why r's signature is wrong, or "". +func (f *fakeS3) verify(r *http.Request) string { + s := &sigv4{creds: fakeCreds, region: "auto", service: "s3"} + u := *r.URL + u.Scheme, u.Host = "http", r.Host + if q := r.URL.Query(); q.Get("X-Amz-Signature") != "" { + date, err := time.Parse(amzDateFormat, q.Get("X-Amz-Date")) + if err != nil { + return "bad X-Amz-Date" + } + secs, _ := strconv.Atoi(q.Get("X-Amz-Expires")) + if time.Now().After(date.Add(time.Duration(secs) * time.Second)) { + return "expired" + } + got := q.Get("X-Amz-Signature") + for _, k := range []string{"X-Amz-Signature", "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date", "X-Amz-Expires", "X-Amz-SignedHeaders", "X-Amz-Security-Token"} { + q.Del(k) + } + u.RawQuery = q.Encode() + want := s.presign(r.Method, &u, time.Duration(secs)*time.Second, date) + if !strings.HasSuffix(want, "X-Amz-Signature="+got) { + return "presigned signature mismatch" + } + return "" + } + m := authRe.FindStringSubmatch(r.Header.Get("Authorization")) + if m == nil { + return "no or malformed Authorization: " + r.Header.Get("Authorization") + } + date, err := time.Parse(amzDateFormat, r.Header.Get("X-Amz-Date")) + if err != nil { + return "bad X-Amz-Date" + } + if r.Header.Get("X-Amz-Security-Token") != "sess" { + return "session token missing" + } + req, _ := http.NewRequest(r.Method, u.String(), nil) + for _, h := range strings.Split(m[2], ";") { + if h != "host" && h != "x-amz-date" && h != "x-amz-content-sha256" && h != "x-amz-security-token" { + req.Header.Set(h, r.Header.Get(h)) + } + } + s.sign(req, r.Header.Get("X-Amz-Content-Sha256"), date) + if req.Header.Get("Authorization") != r.Header.Get("Authorization") { + return "signature mismatch" + } + return "" +} + +func (f *fakeS3) handle(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + f.log = append(f.log, r.Method+" "+r.URL.Path) + if why := f.verify(r); why != "" { + f.t.Errorf("%s %s: %s", r.Method, r.URL, why) + w.WriteHeader(403) + fmt.Fprint(w, `SignatureDoesNotMatchbad`) + return + } + if r.URL.Path == "/b/" || r.URL.Path == "/b" { + f.list(w, r) + return + } + key, ok := strings.CutPrefix(r.URL.Path, "/b/") + if !ok { + w.WriteHeader(404) + return + } + presigned := r.URL.Query().Get("X-Amz-Signature") != "" + switch { + case r.Method == "PUT" && !presigned: + if f.putFails { + w.WriteHeader(403) + fmt.Fprint(w, `AccessDeniedAccess Denied`) + return + } + if r.Header.Get("X-Amz-Content-Sha256") != unsignedPayload { + f.t.Errorf("PUT payload hash %q", r.Header.Get("X-Amz-Content-Sha256")) + } + data, _ := io.ReadAll(r.Body) + if int64(len(data)) != r.ContentLength { + f.t.Errorf("PUT Content-Length %d for %d bytes", r.ContentLength, len(data)) + } + f.objects[key] = &fakeObject{data: data, contentType: r.Header.Get("Content-Type"), marked: r.Header.Get(s3MarkerHeader) == markerValue} + case r.Method == "DELETE" && !presigned: + if f.deleteFails { + w.WriteHeader(500) + fmt.Fprint(w, `InternalErrorboom`) + return + } + delete(f.objects, key) + w.WriteHeader(204) + case r.Method == "HEAD" && !presigned: + o := f.objects[key] + if o == nil { + w.WriteHeader(404) + return + } + if o.marked { + w.Header().Set(s3MarkerHeader, markerValue) + } + case r.Method == "GET" && presigned: // the device + o := f.objects[key] + if o == nil { + w.WriteHeader(404) + return + } + w.Header().Set("Content-Type", o.contentType) + _, _ = w.Write(o.data) + default: + f.t.Errorf("unexpected %s %s", r.Method, r.URL) + w.WriteHeader(400) + } +} + +func (f *fakeS3) list(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + if q.Get("list-type") != "2" { + f.t.Errorf("list query %v", q) + } + var keys []string + for k := range f.objects { + if strings.HasPrefix(k, q.Get("prefix")) { + keys = append(keys, k) + } + } + sort.Strings(keys) + start, _ := strconv.Atoi(q.Get("continuation-token")) + end := min(start+f.pageSize, len(keys)) + type content struct { + Key string `xml:"Key"` + } + page := struct { + XMLName xml.Name `xml:"ListBucketResult"` + Contents []content `xml:"Contents"` + IsTruncated bool `xml:"IsTruncated"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + }{IsTruncated: end < len(keys)} + for _, k := range keys[start:end] { + page.Contents = append(page.Contents, content{k}) + } + if page.IsTruncated { + page.NextContinuationToken = strconv.Itoa(end) + } + _ = xml.NewEncoder(w).Encode(page) +} + +func (f *fakeS3) keys() []string { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.objects { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// install does what iOS does with a link: fetch the manifest, then the IPA it names. +func install(t *testing.T, client *http.Client, links Links) (manifest string, ipa []byte) { + t.Helper() + get := func(u string) []byte { + resp, err := client.Get(u) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + t.Fatalf("GET %s: %s", u, resp.Status) + } + return data + } + escaped := strings.TrimPrefix(links.Link, "itms-services://?action=download-manifest&url=") + manifestURL, err := url.QueryUnescape(strings.ReplaceAll(escaped, "+", "%2B")) + if err != nil || manifestURL != links.ManifestURL { + t.Fatalf("link does not unescape to the manifest URL: %q vs %q (%v)", manifestURL, links.ManifestURL, err) + } + manifest = string(get(manifestURL)) + m := regexp.MustCompile(`(http[^<]+\.ipa[^<]*)`).FindStringSubmatch(manifest) + if m == nil { + t.Fatalf("no IPA URL in manifest:\n%s", manifest) + } + return manifest, get(html.UnescapeString(m[1])) +} + +func TestS3SessionRefreshesInstallsAndCleansUp(t *testing.T) { + f := newFakeS3(t) + app := testApp(t) + var log, out syncBuffer + var progressCalls int + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + opts := &Options{App: app, Backend: f.backend(t, "team/ota", 2*time.Minute), Log: &log, JSON: &out, Timeout: time.Minute, + Progress: func(done, total int64) { progressCalls++ }, refreshLead: 119 * time.Second} + done := make(chan error, 1) + go func() { + _, err := Run(ctx, opts) + done <- err + }() + waitFor(t, "the automatic refresh", func() bool { return out.links() >= 2 }) + + links := decodeLinks(t, out.String()) + first, second := links[0], links[1] + if !strings.HasPrefix(first.ManifestURL, f.srv.URL+"/b/team/ota/"+BucketDir) || !strings.Contains(first.ManifestURL, "/m.plist?") { + t.Errorf("manifest URL = %s", first.ManifestURL) + } + if len(first.Objects) != 2 || !strings.HasPrefix(first.Objects[0], "s3://b/team/ota/ios-builder/") || !strings.HasSuffix(first.Objects[0], "/Tap-Dash-Runner.ipa") { + t.Errorf("objects = %v", first.Objects) + } + if d := time.Until(first.ExpiresAt); d < time.Minute || d > 2*time.Minute { + t.Errorf("expires in %s, want about two minutes", d) + } + // Both links install, the older one serving the newest manifest. + want, _ := os.ReadFile(app.Path) + for _, l := range []Links{first, second} { + manifest, ipa := install(t, f.srv.Client(), l) + if !bytes.Equal(ipa, want) || !strings.Contains(manifest, "run.mobai.tapdash") { + t.Errorf("install from %s fetched %d bytes", l.ManifestURL, len(ipa)) + } + } + if progressCalls == 0 { + t.Error("no upload progress") + } + if keys := f.keys(); len(keys) != 2 { + t.Errorf("objects during the session: %v", keys) + } + cancel() + if err := <-done; err != nil { + t.Fatalf("Run: %v", err) + } + if keys := f.keys(); len(keys) != 0 { + t.Errorf("leftovers: %v", keys) + } + if !strings.Contains(log.String(), "Removed the upload and the manifest.") { + t.Errorf("log: %s", log.String()) + } +} + +func TestS3SessionOnceLeavesObjects(t *testing.T) { + f := newFakeS3(t) + res, err := Run(context.Background(), &Options{App: testApp(t), Backend: f.backend(t, "", 7*24*time.Hour), Once: true}) + if err != nil { + t.Fatal(err) + } + if len(res.Leftovers) != 2 || !strings.HasSuffix(res.Leftovers[1], "/m.plist") || len(f.keys()) != 2 { + t.Errorf("leftovers = %v, objects = %v", res.Leftovers, f.keys()) + } + if !strings.Contains(res.ManifestURL, "X-Amz-Expires=604800") || time.Until(res.ExpiresAt) < 167*time.Hour { + t.Errorf("seven-day link: %s, expires %s", res.ManifestURL, res.ExpiresAt) + } +} + +func TestS3SessionReportsWhatCleanupCouldNotRemove(t *testing.T) { + f := newFakeS3(t) + f.deleteFails = true + var log syncBuffer + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + done := make(chan *Result, 1) + go func() { + res, err := Run(ctx, &Options{App: testApp(t), Backend: f.backend(t, "", 0), Log: &log, Timeout: time.Minute}) + if err != nil { + t.Error(err) + } + done <- res + }() + waitFor(t, "the first link", func() bool { return log.links() == 1 }) + cancel() + res := <-done + if len(res.Leftovers) != 2 || !strings.HasPrefix(res.Leftovers[0], "s3://b/ios-builder/") || !strings.Contains(log.String(), "Cleanup failed") { + t.Errorf("leftovers = %v\n%s", res.Leftovers, log.String()) + } +} + +func TestS3UploadErrorNamesTheS3Code(t *testing.T) { + f := newFakeS3(t) + f.putFails = true + _, err := Run(context.Background(), &Options{App: testApp(t), Backend: f.backend(t, "", 0)}) + if err == nil || !strings.Contains(err.Error(), "403 Forbidden AccessDenied: Access Denied") { + t.Fatalf("err = %v", err) + } + if len(f.keys()) != 0 { + t.Errorf("objects: %v", f.keys()) + } +} + +// Cleanup deletes only marked objects in the upload folder under the prefix. +func TestS3CleanupSweepsOnlyMarkedObjectsUnderThePrefix(t *testing.T) { + f := newFakeS3(t) + for key, marked := range map[string]bool{ + "ota/ios-builder/old1/App.ipa": true, + "ota/ios-builder/old1/m.plist": true, + "ota/ios-builder/old2/m.plist": true, + "ota/ios-builder/notes.txt": false, // the user's, no marker + "ota/release.ipa": true, // outside the upload folder + "ios-builder/x/m.plist": true, // outside the prefix + } { + f.objects[key] = &fakeObject{marked: marked} + } + n, err := f.backend(t, "ota/", 0).Cleanup(context.Background()) + if err != nil || n != 3 { + t.Fatalf("Cleanup = %d, %v", n, err) + } + if got := strings.Join(f.keys(), ","); got != "ios-builder/x/m.plist,ota/ios-builder/notes.txt,ota/release.ipa" { + t.Errorf("left: %s", got) + } +} + +// A code wider than 80 columns comes with a note, a normal one without. +func TestWideQRNote(t *testing.T) { + for _, tc := range []struct { + link string + note bool + }{ + {representativeLink, false}, + {Link("https://b.s3.us-east-1.amazonaws.com/ios-builder/x/m.plist?X-Amz-Security-Token=" + strings.Repeat("A", 800)), true}, + } { + var log bytes.Buffer + o := &Options{Log: &log, QR: true} + if err := o.print(&Result{Links: Links{Link: tc.link, ExpiresAt: time.Now()}}); err != nil { + t.Fatal(err) + } + if got := strings.Contains(log.String(), "columns wide"); got != tc.note { + t.Errorf("link of %d characters: note = %v\n%s", len(tc.link), got, log.String()) + } + } +} + +func TestBucketTTLLimits(t *testing.T) { + for _, ttl := range []time.Duration{time.Minute, 8 * 24 * time.Hour} { + _, err := NewS3(&S3Options{Bucket: "b", Credentials: fakeCreds, TTL: ttl}) + if err == nil || !strings.Contains(err.Error(), "--ttl must be between") { + t.Errorf("ttl %s: err = %v", ttl, err) + } + } + b, err := NewS3(&S3Options{Bucket: "b", Credentials: fakeCreds}) + if err != nil || b.ttl != DefaultBucketTTL { + t.Errorf("default ttl: %v, %v", b, err) + } +} + +func TestS3BaseURL(t *testing.T) { + for _, tc := range []struct{ bucket, region, endpoint, want string }{ + {"builds", "eu-west-1", "", "https://builds.s3.eu-west-1.amazonaws.com/"}, + {"my.builds", "us-east-1", "", "https://s3.us-east-1.amazonaws.com/my.builds/"}, + {"builds", "auto", "https://acct.r2.cloudflarestorage.com/", "https://acct.r2.cloudflarestorage.com/builds/"}, + {"builds", "us-east-1", "http://localhost:9000", "http://localhost:9000/builds/"}, + } { + if got, err := s3BaseURL(tc.bucket, tc.region, tc.endpoint); err != nil || got != tc.want { + t.Errorf("s3BaseURL(%q, %q, %q) = %q, %v", tc.bucket, tc.region, tc.endpoint, got, err) + } + } + if _, err := s3BaseURL("b", "r", "ftp://x"); err == nil { + t.Error("ftp endpoint accepted") + } +} + +func TestLoadAWSCredentials(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "credentials") + if err := os.WriteFile(file, []byte("[default]\naws_access_key_id = AKDEF\naws_secret_access_key = SKDEF\n\n# comment\n[ci]\naws_access_key_id=AKCI\naws_secret_access_key=SKCI\naws_session_token=TOK\n[sso]\nsso_start_url = https://x\n"), 0o600); err != nil { + t.Fatal(err) + } + env := func(m map[string]string) func(string) string { + return func(k string) string { return m[k] } + } + for _, tc := range []struct { + name string + env map[string]string + want AWSCredentials + wantErr string + }{ + {"env wins", map[string]string{"AWS_ACCESS_KEY_ID": "AKENV", "AWS_SECRET_ACCESS_KEY": "SKENV", "AWS_SESSION_TOKEN": "T", "AWS_SHARED_CREDENTIALS_FILE": file}, AWSCredentials{"AKENV", "SKENV", "T"}, ""}, + {"partial env", map[string]string{"AWS_ACCESS_KEY_ID": "AKENV"}, AWSCredentials{}, "set both"}, + {"default profile", map[string]string{"AWS_SHARED_CREDENTIALS_FILE": file}, AWSCredentials{"AKDEF", "SKDEF", ""}, ""}, + {"named profile", map[string]string{"AWS_SHARED_CREDENTIALS_FILE": file, "AWS_PROFILE": "ci"}, AWSCredentials{"AKCI", "SKCI", "TOK"}, ""}, + {"home", map[string]string{"HOME": dir, "AWS_PROFILE": "ci"}, AWSCredentials{}, "no AWS credentials"}, + {"sso profile", map[string]string{"AWS_SHARED_CREDENTIALS_FILE": file, "AWS_PROFILE": "sso"}, AWSCredentials{}, "SSO and credential_process profiles are not read"}, + } { + got, err := LoadAWSCredentials(env(tc.env)) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Errorf("%s: err = %v", tc.name, err) + } + continue + } + if err != nil || got != tc.want { + t.Errorf("%s: %+v, %v", tc.name, got, err) + } + } +} diff --git a/internal/otainstall/session.go b/internal/otainstall/session.go index 3f266a3..ebd5944 100644 --- a/internal/otainstall/session.go +++ b/internal/otainstall/session.go @@ -9,8 +9,13 @@ import ( "io" "strings" "time" + "unicode/utf8" ) +// wideQR is the widest code, quiet zone included, that fits a standard +// 80-column terminal. +const wideQR = 80 + // Options configures Run. type Options struct { App *App @@ -168,6 +173,9 @@ func (o *Options) print(res *Result) error { } fmt.Fprintln(o.Log) fmt.Fprint(o.Log, qr) + if width := utf8.RuneCountInString(qr[:strings.Index(qr, "\n")]); width > wideQR { + fmt.Fprintf(o.Log, "\nThe QR code is %d columns wide because the signed URL is long (temporary credentials add their session token); widen the terminal or zoom out to scan it, or open the link on the phone.\n", width) + } } fmt.Fprintln(o.Log) return nil diff --git a/internal/otainstall/sigv4.go b/internal/otainstall/sigv4.go new file mode 100644 index 0000000..564503c --- /dev/null +++ b/internal/otainstall/sigv4.go @@ -0,0 +1,177 @@ +package otainstall + +import ( + "crypto/hmac" + "crypto/sha256" + "encoding/hex" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" +) + +// AWSCredentials sign S3 requests; SessionToken is set for temporary ones. +type AWSCredentials struct { + AccessKeyID string + SecretAccessKey string + SessionToken string +} + +// sigv4 is AWS Signature Version 4 for one region and service, just what the +// S3 backend needs: header-signed requests and query-presigned URLs. +type sigv4 struct { + creds AWSCredentials + region string + service string +} + +const ( + sigv4Algorithm = "AWS4-HMAC-SHA256" + amzDateFormat = "20060102T150405Z" + // unsignedPayload lets a request stream a body it has not hashed. + unsignedPayload = "UNSIGNED-PAYLOAD" + // emptySHA256 is the hash of an empty body. + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +) + +// sign adds X-Amz-Date, X-Amz-Content-Sha256, X-Amz-Security-Token and +// Authorization to req. Every header already on req is signed, plus Host. +func (s *sigv4) sign(req *http.Request, payloadHash string, now time.Time) { + now = now.UTC() + req.Header.Set("X-Amz-Date", now.Format(amzDateFormat)) + req.Header.Set("X-Amz-Content-Sha256", payloadHash) + if s.creds.SessionToken != "" { + req.Header.Set("X-Amz-Security-Token", s.creds.SessionToken) + } + headers := map[string]string{"host": hostOf(req.URL)} + for k, v := range req.Header { + headers[strings.ToLower(k)] = strings.Join(trimAll(v), ",") + } + names := make([]string, 0, len(headers)) + for k := range headers { + names = append(names, k) + } + sort.Strings(names) + var canonHeaders strings.Builder + for _, k := range names { + canonHeaders.WriteString(k + ":" + headers[k] + "\n") + } + signed := strings.Join(names, ";") + canonical := strings.Join([]string{ + req.Method, canonicalPath(req.URL), canonicalQuery(req.URL.Query()), canonHeaders.String(), signed, payloadHash, + }, "\n") + scope := s.scope(now) + sig := s.signature(now, scope, canonical) + req.Header.Set("Authorization", sigv4Algorithm+" Credential="+s.creds.AccessKeyID+"/"+scope+", SignedHeaders="+signed+", Signature="+sig) +} + +// presign returns u with the query parameters that let anyone holding the +// URL make method on it until now+expires, signing only the host header. +func (s *sigv4) presign(method string, u *url.URL, expires time.Duration, now time.Time) string { + now = now.UTC() + scope := s.scope(now) + q := u.Query() + q.Set("X-Amz-Algorithm", sigv4Algorithm) + q.Set("X-Amz-Credential", s.creds.AccessKeyID+"/"+scope) + q.Set("X-Amz-Date", now.Format(amzDateFormat)) + q.Set("X-Amz-Expires", strconv.FormatInt(int64(expires/time.Second), 10)) + q.Set("X-Amz-SignedHeaders", "host") + if s.creds.SessionToken != "" { + q.Set("X-Amz-Security-Token", s.creds.SessionToken) + } + query := canonicalQuery(q) + canonical := strings.Join([]string{ + method, canonicalPath(u), query, "host:" + hostOf(u) + "\n", "host", unsignedPayload, + }, "\n") + sig := s.signature(now, scope, canonical) + return u.Scheme + "://" + u.Host + canonicalPath(u) + "?" + query + "&X-Amz-Signature=" + sig +} + +func (s *sigv4) scope(now time.Time) string { + return now.Format("20060102") + "/" + s.region + "/" + s.service + "/aws4_request" +} + +func (s *sigv4) signature(now time.Time, scope, canonical string) string { + sum := sha256.Sum256([]byte(canonical)) + toSign := sigv4Algorithm + "\n" + now.Format(amzDateFormat) + "\n" + scope + "\n" + hex.EncodeToString(sum[:]) + key := hmacSHA256([]byte("AWS4"+s.creds.SecretAccessKey), now.Format("20060102")) + key = hmacSHA256(key, s.region) + key = hmacSHA256(key, s.service) + key = hmacSHA256(key, "aws4_request") + return hex.EncodeToString(hmacSHA256(key, toSign)) +} + +func hmacSHA256(key []byte, data string) []byte { + m := hmac.New(sha256.New, key) + m.Write([]byte(data)) + return m.Sum(nil) +} + +// hostOf is the Host header Go sends: the port is dropped when it is the +// scheme's default. +func hostOf(u *url.URL) string { + host := u.Host + if (u.Scheme == "https" && strings.HasSuffix(host, ":443")) || (u.Scheme == "http" && strings.HasSuffix(host, ":80")) { + host = host[:strings.LastIndex(host, ":")] + } + return host +} + +// canonicalPath is the URI-encoded path, slashes kept, as S3 wants it (S3 is +// the one service that does not double-encode). +func canonicalPath(u *url.URL) string { + p := u.Path + if p == "" { + return "/" + } + return uriEncode(p, false) +} + +// canonicalQuery sorts by key, then value, and encodes both. +func canonicalQuery(q url.Values) string { + keys := make([]string, 0, len(q)) + for k := range q { + keys = append(keys, k) + } + sort.Strings(keys) + var parts []string + for _, k := range keys { + vs := append([]string(nil), q[k]...) + sort.Strings(vs) + for _, v := range vs { + parts = append(parts, uriEncode(k, true)+"="+uriEncode(v, true)) + } + } + return strings.Join(parts, "&") +} + +// uriEncode percent-encodes every byte but A-Z a-z 0-9 - . _ ~ (and '/' +// unless encodeSlash), in upper-case hex. +func uriEncode(s string, encodeSlash bool) string { + const hexDigits = "0123456789ABCDEF" + var b strings.Builder + for i := 0; i < len(s); i++ { + c := s[i] + switch { + case c >= 'A' && c <= 'Z', c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-', c == '.', c == '_', c == '~': + b.WriteByte(c) + case c == '/' && !encodeSlash: + b.WriteByte(c) + default: + b.WriteByte('%') + b.WriteByte(hexDigits[c>>4]) + b.WriteByte(hexDigits[c&15]) + } + } + return b.String() +} + +func trimAll(vs []string) []string { + out := make([]string, len(vs)) + for i, v := range vs { + out[i] = strings.Join(strings.Fields(v), " ") + } + return out +} diff --git a/internal/otainstall/sigv4_test.go b/internal/otainstall/sigv4_test.go new file mode 100644 index 0000000..8acdf32 --- /dev/null +++ b/internal/otainstall/sigv4_test.go @@ -0,0 +1,98 @@ +package otainstall + +import ( + "net/http" + "net/url" + "strings" + "testing" + "time" +) + +// The vectors below are the worked examples of the Amazon S3 API Reference, +// "Authenticating Requests (AWS Signature Version 4)": the header-signed +// examples and the presigned URL example, all for examplebucket in us-east-1 +// on 24 May 2013 with the documentation's example key. +var ( + exampleCreds = AWSCredentials{AccessKeyID: "AKIAIOSFODNN7EXAMPLE", SecretAccessKey: "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY"} + exampleTime = time.Date(2013, 5, 24, 0, 0, 0, 0, time.UTC) +) + +func exampleSigner() *sigv4 { return &sigv4{creds: exampleCreds, region: "us-east-1", service: "s3"} } + +func TestSigV4PresignAWSVector(t *testing.T) { + u, _ := url.Parse("https://examplebucket.s3.amazonaws.com/test.txt") + got := exampleSigner().presign("GET", u, 86400*time.Second, exampleTime) + want := "https://examplebucket.s3.amazonaws.com/test.txt" + + "?X-Amz-Algorithm=AWS4-HMAC-SHA256" + + "&X-Amz-Credential=AKIAIOSFODNN7EXAMPLE%2F20130524%2Fus-east-1%2Fs3%2Faws4_request" + + "&X-Amz-Date=20130524T000000Z&X-Amz-Expires=86400&X-Amz-SignedHeaders=host" + + "&X-Amz-Signature=aeeed9bbccd4d02ee5c0109b86d86835f995330da4c265957d157751f604d404" + if got != want { + t.Errorf("presign\n got %s\nwant %s", got, want) + } +} + +func TestSigV4HeaderAWSVectors(t *testing.T) { + for _, tc := range []struct { + name, method, url, payload string + headers map[string]string + wantSigned, wantSig string + }{ + { + name: "GET Object", method: "GET", url: "https://examplebucket.s3.amazonaws.com/test.txt", payload: emptySHA256, + headers: map[string]string{"Range": "bytes=0-9"}, + wantSigned: "host;range;x-amz-content-sha256;x-amz-date", + wantSig: "f0e8bdb87c964420e857bd35b5d6ed310bd44f0170aba48dd91039c6036bdb41", + }, + { + name: "PUT Object", method: "PUT", url: "https://examplebucket.s3.amazonaws.com/test$file.text", + payload: "44ce7dd67c959e0d3524ffac1771dfbba87d2b6b4b4e99e42034a8b803f8b072", + headers: map[string]string{"Date": "Fri, 24 May 2013 00:00:00 GMT", "X-Amz-Storage-Class": "REDUCED_REDUNDANCY"}, + wantSigned: "date;host;x-amz-content-sha256;x-amz-date;x-amz-storage-class", + wantSig: "98ad721746da40c64f1a55b78f14c238d841ea1380cd77a1b5971af0ece108bd", + }, + { + name: "GET Bucket Lifecycle", method: "GET", url: "https://examplebucket.s3.amazonaws.com/?lifecycle", payload: emptySHA256, + wantSigned: "host;x-amz-content-sha256;x-amz-date", + wantSig: "fea454ca298b7da1c68078a5d1bdbfbbe0d65c699e0f91ac7a200a0136783543", + }, + { + name: "Get Bucket (List Objects)", method: "GET", url: "https://examplebucket.s3.amazonaws.com/?max-keys=2&prefix=J", payload: emptySHA256, + wantSigned: "host;x-amz-content-sha256;x-amz-date", + wantSig: "34b48302e7b5fa45bde8084f4b7868a86f0a534bc59db6670ed5711ef69dc6f7", + }, + } { + t.Run(tc.name, func(t *testing.T) { + req, err := http.NewRequest(tc.method, tc.url, nil) + if err != nil { + t.Fatal(err) + } + for k, v := range tc.headers { + req.Header.Set(k, v) + } + exampleSigner().sign(req, tc.payload, exampleTime) + want := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request, SignedHeaders=" + tc.wantSigned + ", Signature=" + tc.wantSig + if got := req.Header.Get("Authorization"); got != want { + t.Errorf("Authorization\n got %s\nwant %s", got, want) + } + }) + } +} + +// A session token is signed into the query, and a non-default port stays in +// the signed host. +func TestSigV4PresignSessionTokenAndPort(t *testing.T) { + s := &sigv4{creds: AWSCredentials{AccessKeyID: "AK", SecretAccessKey: "SK", SessionToken: "tok/en+="}, region: "auto", service: "s3"} + u, _ := url.Parse("http://127.0.0.1:9000/bucket/a b.ipa") + got := s.presign("GET", u, time.Hour, exampleTime) + if !strings.HasPrefix(got, "http://127.0.0.1:9000/bucket/a%20b.ipa?") || !strings.Contains(got, "&X-Amz-Security-Token=tok%2Fen%2B%3D&") { + t.Errorf("presign = %s", got) + } + if hostOf(u) != "127.0.0.1:9000" { + t.Errorf("host = %s", hostOf(u)) + } + u443, _ := url.Parse("https://h:443/x") + if hostOf(u443) != "h" { + t.Errorf("host = %s", hostOf(u443)) + } +} From 89e3f334f3239526ff764485d03f848560597298 Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:44:05 +0200 Subject: [PATCH 4/9] otainstall: azure backend with service SAS The same bucket backend on Azure Blob Storage: every request, Builder's own included, carries a service SAS signed with the account key, so one signer covers upload, delete, list and the install links. The SAS keeps its colons, slashes and padding unescaped, which leaves a version 10 code (57 modules). TestBucketQRSizes pins each backend's size. --- internal/otainstall/azure.go | 229 +++++++++++++++++++++++ internal/otainstall/azure_test.go | 295 ++++++++++++++++++++++++++++++ internal/otainstall/qr_test.go | 41 +++++ 3 files changed, 565 insertions(+) create mode 100644 internal/otainstall/azure.go create mode 100644 internal/otainstall/azure_test.go diff --git a/internal/otainstall/azure.go b/internal/otainstall/azure.go new file mode 100644 index 0000000..b427b4e --- /dev/null +++ b/internal/otainstall/azure.go @@ -0,0 +1,229 @@ +package otainstall + +import ( + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "time" + + "github.com/MobAI-App/ios-builder/internal/config" +) + +const ( + // azureVersion is the storage service version the SAS is signed for and + // requests are made with. + azureVersion = "2022-11-02" + // azureMarker is the metadata name every Builder blob carries (metadata + // names must be C# identifiers, so no dash). + azureMarker = "iosbuilder" + // azureOpTTL bounds the SAS of Builder's own requests; an IPA upload on a + // slow link must finish within it. + azureOpTTL = 2 * time.Hour +) + +// AzureOptions configures NewAzure. +type AzureOptions struct { + Account string + Container string + // Key is the storage account key (base64), which signs service SAS tokens. + Key string + // Endpoint is the blob endpoint, default https://.blob.core.windows.net + // (Azurite: http://127.0.0.1:10000/devstoreaccount1). + Endpoint string + Prefix string + TTL time.Duration + HTTPClient *http.Client +} + +// NewAzure returns the bucket backend for Azure Blob Storage. Every request, +// Builder's own included, carries a service SAS signed with the account key, +// so one signer covers upload, delete, list and the install links. +func NewAzure(opts *AzureOptions) (*Bucket, error) { + if opts.Account == "" || opts.Container == "" { + return nil, errors.New(`the azure backend needs an account and a container: set "distribute": {"account": "...", "container": "..."} in builder.json (or AZURE_STORAGE_ACCOUNT)`) + } + key, err := base64.StdEncoding.DecodeString(opts.Key) + if err != nil || len(key) == 0 { + return nil, errors.New("the azure backend needs the storage account key in AZURE_STORAGE_KEY (base64), or AZURE_STORAGE_CONNECTION_STRING") + } + endpoint := opts.Endpoint + if endpoint == "" { + endpoint = "https://" + opts.Account + ".blob.core.windows.net" + } + if u, err := url.Parse(endpoint); err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return nil, fmt.Errorf("azure endpoint %q is not an http(s) URL", endpoint) + } + client := opts.HTTPClient + if client == nil { + client = &http.Client{} + } + store := &azureStore{account: opts.Account, container: opts.Container, key: key, base: strings.TrimRight(endpoint, "/") + "/" + uriEncode(opts.Container, true), http: client, now: time.Now} + return newBucket(store, opts.Prefix, opts.TTL) +} + +type azureStore struct { + account, container string + key []byte + base string // endpoint/container + http *http.Client + now func() time.Time +} + +func (a *azureStore) name(key string) string { + return "azure://" + a.account + "/" + a.container + "/" + key +} + +// sas is a service SAS query string. blob empty signs the container (sr=c), +// else that blob (sr=b). Only what a query parser would misread is escaped +// (the signature's +): se keeps its colons and sig its / and =, because the +// install link escapes every % again and each one costs three characters of +// QR code. +func (a *azureStore) sas(blob, perms string, expiry time.Time) string { + resource, canonical := "c", "/blob/"+a.account+"/"+a.container + if blob != "" { + resource, canonical = "b", canonical+"/"+blob + } + se := expiry.UTC().Format(time.RFC3339) + toSign := strings.Join([]string{ + perms, "", se, canonical, "", "", "", azureVersion, resource, "", "", "", "", "", "", "", + }, "\n") + m := hmac.New(sha256.New, a.key) + m.Write([]byte(toSign)) + sig := base64.StdEncoding.EncodeToString(m.Sum(nil)) + return "sv=" + azureVersion + "&se=" + se + "&sr=" + resource + "&sp=" + perms + "&sig=" + strings.ReplaceAll(sig, "+", "%2B") +} + +func (a *azureStore) blobURL(key string) string { return a.base + "/" + uriEncode(key, false) } + +func (a *azureStore) do(ctx context.Context, method, rawURL string, header http.Header, body io.Reader, size int64) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, method, rawURL, body) + if err != nil { + return nil, err + } + for k, v := range header { + req.Header[k] = v + } + req.Header.Set("X-Ms-Version", azureVersion) + if body != nil { + req.ContentLength = size + } + return a.http.Do(req) +} + +func (a *azureStore) put(ctx context.Context, key, contentType string, body io.Reader, size int64) error { + h := http.Header{"Content-Type": {contentType}, "X-Ms-Blob-Type": {"BlockBlob"}, "X-Ms-Meta-" + azureMarker: {markerValue}} + resp, err := a.do(ctx, "PUT", a.blobURL(key)+"?"+a.sas(key, "cw", a.now().Add(azureOpTTL)), h, body, size) + if err != nil { + return err + } + return azureError(resp, "PUT", a.name(key)) +} + +func (a *azureStore) delete(ctx context.Context, key string) error { + resp, err := a.do(ctx, "DELETE", a.blobURL(key)+"?"+a.sas(key, "d", a.now().Add(azureOpTTL)), nil, nil, 0) + if err != nil { + return err + } + if resp.StatusCode == http.StatusNotFound { + resp.Body.Close() + return nil + } + return azureError(resp, "DELETE", a.name(key)) +} + +// listMarked pages through List Blobs with metadata, which carries the marker. +func (a *azureStore) listMarked(ctx context.Context, prefix string) ([]string, error) { + var keys []string + marker := "" + for { + q := url.Values{"restype": {"container"}, "comp": {"list"}, "prefix": {prefix}, "include": {"metadata"}} + if marker != "" { + q.Set("marker", marker) + } + resp, err := a.do(ctx, "GET", a.base+"?"+q.Encode()+"&"+a.sas("", "l", a.now().Add(azureOpTTL)), nil, nil, 0) + if err != nil { + return nil, err + } + var page struct { + Blobs []struct { + Name string `xml:"Name"` + Metadata struct { + Marker string `xml:"iosbuilder"` + } `xml:"Metadata"` + } `xml:"Blobs>Blob"` + NextMarker string `xml:"NextMarker"` + } + if err := decodeXML(resp, "list", a.name(prefix), azureError, &page); err != nil { + return nil, err + } + for _, b := range page.Blobs { + if b.Metadata.Marker == markerValue { + keys = append(keys, b.Name) + } + } + if page.NextMarker == "" { + return keys, nil + } + marker = page.NextMarker + } +} + +func (a *azureStore) presign(key string, ttl time.Duration, now time.Time) (string, error) { + return a.blobURL(key) + "?" + a.sas(key, "r", now.Add(ttl)), nil +} + +// azureError closes resp and turns a non-2xx answer into an error with the +// storage error code (AuthenticationFailed, ContainerNotFound, ...). +func azureError(resp *http.Response, op, what string) error { + defer resp.Body.Close() + if resp.StatusCode < 300 { + _, _ = io.Copy(io.Discard, resp.Body) + return nil + } + code := resp.Header.Get("X-Ms-Error-Code") + if code == "" { + return fmt.Errorf("azure %s %s: %s", op, what, resp.Status) + } + return fmt.Errorf("azure %s %s: %s %s", op, what, resp.Status, code) +} + +// AzureFromConfig builds the azure backend from builder.json and the +// environment the Azure CLI uses: AZURE_STORAGE_ACCOUNT and AZURE_STORAGE_KEY, +// or AZURE_STORAGE_CONNECTION_STRING (AccountName, AccountKey, BlobEndpoint). +func AzureFromConfig(cfg *config.DistributeConfig, ttl time.Duration, getenv func(string) string) (*Bucket, error) { + if cfg == nil { + cfg = &config.DistributeConfig{} + } + opts := &AzureOptions{Account: cfg.Account, Container: cfg.Container, Endpoint: cfg.Endpoint, Prefix: cfg.Prefix, TTL: ttl} + if cs := getenv("AZURE_STORAGE_CONNECTION_STRING"); cs != "" { + for _, part := range strings.Split(cs, ";") { + k, v, _ := strings.Cut(part, "=") + switch k { + case "AccountName": + if opts.Account == "" { + opts.Account = v + } + case "AccountKey": + opts.Key = v + case "BlobEndpoint": + if opts.Endpoint == "" { + opts.Endpoint = v + } + } + } + } + if opts.Account == "" { + opts.Account = getenv("AZURE_STORAGE_ACCOUNT") + } + if opts.Key == "" { + opts.Key = getenv("AZURE_STORAGE_KEY") + } + return NewAzure(opts) +} diff --git a/internal/otainstall/azure_test.go b/internal/otainstall/azure_test.go new file mode 100644 index 0000000..cb457a8 --- /dev/null +++ b/internal/otainstall/azure_test.go @@ -0,0 +1,295 @@ +package otainstall + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/sha256" + "encoding/base64" + "fmt" + "github.com/MobAI-App/ios-builder/internal/config" + "io" + "net/http" + "net/http/httptest" + "net/url" + "os" + "sort" + "strings" + "sync" + "testing" + "time" +) + +const fakeAzureKey = "ZmFrZS1henVyZS1hY2NvdW50LWtleQ==" + +// azureSig is the service SAS signature computed straight from the string-to- +// sign of "Create a service SAS" (version 2020-12-06 and later): sixteen +// fields, the optional ones empty. +func azureSig(perms, expiry, resource, canonical string) string { + key, _ := base64.StdEncoding.DecodeString(fakeAzureKey) + toSign := perms + "\n" + // signedPermissions + "\n" + // signedStart + expiry + "\n" + // signedExpiry + canonical + "\n" + // canonicalizedResource + "\n" + // signedIdentifier + "\n" + // signedIP + "\n" + // signedProtocol + "2022-11-02\n" + // signedVersion + resource + "\n" + // signedResource + "\n" + // signedSnapshotTime + "\n" + // signedEncryptionScope + "\n\n\n\n" // rscc, rscd, rsce, rscl, then rsct (empty, no newline) + m := hmac.New(sha256.New, key) + m.Write([]byte(toSign)) + return base64.StdEncoding.EncodeToString(m.Sum(nil)) +} + +func TestAzureSASMatchesTheDocumentedStringToSign(t *testing.T) { + b, err := NewAzure(&AzureOptions{Account: "acct", Container: "builds", Key: fakeAzureKey}) + if err != nil { + t.Fatal(err) + } + store := b.store.(*azureStore) + exp := time.Date(2026, 10, 4, 13, 0, 0, 0, time.UTC) + got, _ := url.ParseQuery(store.sas("ios-builder/x/m.plist", "r", exp)) + if got.Get("sig") != azureSig("r", "2026-10-04T13:00:00Z", "b", "/blob/acct/builds/ios-builder/x/m.plist") || + got.Get("sv") != "2022-11-02" || got.Get("sr") != "b" || got.Get("sp") != "r" || got.Get("se") != "2026-10-04T13:00:00Z" { + t.Errorf("blob SAS = %v", got) + } + got, _ = url.ParseQuery(store.sas("", "l", exp)) + if got.Get("sig") != azureSig("l", "2026-10-04T13:00:00Z", "c", "/blob/acct/builds") || got.Get("sr") != "c" { + t.Errorf("container SAS = %v", got) + } + u, _ := store.presign("ios-builder/x/m.plist", time.Hour, exp.Add(-time.Hour)) + if !strings.HasPrefix(u, "https://acct.blob.core.windows.net/builds/ios-builder/x/m.plist?sv=2022-11-02&se=2026-10-04T13:00:00Z&sr=b&sp=r&sig=") { + t.Errorf("presign = %s", u) + } +} + +// fakeAzure is Blob Storage for account "acct", container "c", served under +// /acct like Azurite, checking every SAS. +type fakeAzure struct { + t *testing.T + srv *httptest.Server + mu sync.Mutex + blobs map[string]*fakeObject + deleteFails bool +} + +func newFakeAzure(t *testing.T) *fakeAzure { + f := &fakeAzure{t: t, blobs: map[string]*fakeObject{}} + f.srv = httptest.NewServer(http.HandlerFunc(f.handle)) + t.Cleanup(f.srv.Close) + return f +} + +func (f *fakeAzure) backend(t *testing.T, prefix string, ttl time.Duration) *Bucket { + t.Helper() + b, err := NewAzure(&AzureOptions{Account: "acct", Container: "c", Key: fakeAzureKey, Endpoint: f.srv.URL + "/acct", Prefix: prefix, TTL: ttl, HTTPClient: f.srv.Client()}) + if err != nil { + t.Fatal(err) + } + return b +} + +func (f *fakeAzure) handle(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + q := r.URL.Query() + blob, _ := strings.CutPrefix(r.URL.Path, "/acct/c/") + canonical, resource := "/blob/acct/c/"+blob, "b" + if r.URL.Path == "/acct/c" { + canonical, resource = "/blob/acct/c", "c" + } + exp, err := time.Parse(time.RFC3339, q.Get("se")) + if err != nil || time.Now().After(exp) || q.Get("sr") != resource || q.Get("sig") != azureSig(q.Get("sp"), q.Get("se"), resource, canonical) { + f.t.Errorf("%s %s: bad SAS", r.Method, r.URL) + w.Header().Set("X-Ms-Error-Code", "AuthenticationFailed") + w.WriteHeader(403) + return + } + need := map[string]string{"PUT": "cw", "DELETE": "d", "GET": "r"}[r.Method] + if resource == "c" { + need = "l" + } + if q.Get("sp") != need { + f.t.Errorf("%s %s with sp=%s, want %s", r.Method, r.URL.Path, q.Get("sp"), need) + } + if r.Method != "GET" || resource == "c" { + if r.Header.Get("X-Ms-Version") != azureVersion { + f.t.Errorf("%s without x-ms-version", r.Method) + } + } + switch { + case resource == "c": + if q.Get("restype") != "container" || q.Get("comp") != "list" || q.Get("include") != "metadata" { + f.t.Errorf("list query %v", q) + } + var names []string + for k := range f.blobs { + if strings.HasPrefix(k, q.Get("prefix")) { + names = append(names, k) + } + } + sort.Strings(names) + // One blob per page, to exercise NextMarker. + start := 0 + if m := q.Get("marker"); m != "" { + start = sort.SearchStrings(names, m) + } + fmt.Fprint(w, ``) + if start < len(names) { + meta := "" + if f.blobs[names[start]].marked { + meta = "distribute" + } + fmt.Fprintf(w, `%s%s`, names[start], meta) + } + fmt.Fprint(w, ``) + if start+1 < len(names) { + fmt.Fprintf(w, `%s`, names[start+1]) + } else { + fmt.Fprint(w, ``) + } + fmt.Fprint(w, ``) + case r.Method == "PUT": + if r.Header.Get("X-Ms-Blob-Type") != "BlockBlob" { + f.t.Errorf("PUT without x-ms-blob-type") + } + data, _ := io.ReadAll(r.Body) + f.blobs[blob] = &fakeObject{data: data, contentType: r.Header.Get("Content-Type"), marked: r.Header.Get("X-Ms-Meta-Iosbuilder") == markerValue} + w.WriteHeader(201) + case r.Method == "DELETE": + if f.deleteFails { + w.Header().Set("X-Ms-Error-Code", "InternalError") + w.WriteHeader(500) + return + } + if f.blobs[blob] == nil { + w.Header().Set("X-Ms-Error-Code", "BlobNotFound") + w.WriteHeader(404) + return + } + delete(f.blobs, blob) + w.WriteHeader(202) + case r.Method == "GET": + o := f.blobs[blob] + if o == nil { + w.WriteHeader(404) + return + } + _, _ = w.Write(o.data) + } +} + +func (f *fakeAzure) names() []string { + f.mu.Lock() + defer f.mu.Unlock() + var out []string + for k := range f.blobs { + out = append(out, k) + } + sort.Strings(out) + return out +} + +func TestAzureSessionInstallsAndCleansUp(t *testing.T) { + f := newFakeAzure(t) + app := testApp(t) + var log syncBuffer + stdin, keys := io.Pipe() + done := make(chan *Result, 1) + go func() { + res, err := Run(context.Background(), &Options{App: app, Backend: f.backend(t, "ota", 0), Log: &log, Stdin: stdin, Timeout: time.Minute}) + if err != nil { + t.Error(err) + } + done <- res + }() + waitFor(t, "the first link", func() bool { return log.links() == 1 }) + if _, err := io.WriteString(keys, "\n"); err != nil { + t.Fatal(err) + } + waitFor(t, "the refresh", func() bool { return log.links() == 2 }) + if n := len(f.names()); n != 2 { + t.Errorf("blobs: %v", f.names()) + } + if _, err := io.WriteString(keys, "q\n"); err != nil { + t.Fatal(err) + } + res := <-done + if !strings.HasPrefix(res.ManifestURL, f.srv.URL+"/acct/c/ota/ios-builder/") || !strings.HasPrefix(res.Objects[0], "azure://acct/c/ota/ios-builder/") { + t.Errorf("links = %+v", res.Links) + } + if d := time.Until(res.ExpiresAt); d < 59*time.Minute { + t.Errorf("default TTL: expires in %s", d) + } + if len(f.names()) != 0 || len(res.Leftovers) != 0 { + t.Errorf("left: %v %v", f.names(), res.Leftovers) + } +} + +func TestAzureLinkInstalls(t *testing.T) { + f := newFakeAzure(t) + app := testApp(t) + res, err := Run(context.Background(), &Options{App: app, Backend: f.backend(t, "", 0), Once: true}) + if err != nil { + t.Fatal(err) + } + if strings.Count(res.Link, "%25") != strings.Count(res.Link, "%252B") { + t.Errorf("link has double-escaped characters, which cost QR versions: %s", res.Link) + } + _, ipa := install(t, f.srv.Client(), res.Links) + want, _ := os.ReadFile(app.Path) + if !bytes.Equal(ipa, want) { + t.Error("installed IPA differs") + } + if len(res.Leftovers) != 2 || !strings.HasPrefix(res.Leftovers[0], "azure://acct/c/ios-builder/") { + t.Errorf("leftovers = %v", res.Leftovers) + } +} + +func TestAzureCleanupAndLeftovers(t *testing.T) { + f := newFakeAzure(t) + for name, marked := range map[string]bool{ + "ios-builder/a/App.ipa": true, "ios-builder/a/m.plist": true, "ios-builder/mine.txt": false, "other/m.plist": true, + } { + f.blobs[name] = &fakeObject{marked: marked} + } + b := f.backend(t, "", 0) + n, err := b.Cleanup(context.Background()) + if err != nil || n != 2 { + t.Fatalf("Cleanup = %d, %v", n, err) + } + if got := strings.Join(f.names(), ","); got != "ios-builder/mine.txt,other/m.plist" { + t.Errorf("left: %s", got) + } + + f.deleteFails = true + up, err := b.Upload(context.Background(), testApp(t), nil) + if err != nil { + t.Fatal(err) + } + if err := up.Close(context.Background()); err == nil || !strings.Contains(err.Error(), "500 Internal Server Error InternalError") { + t.Errorf("Close: %v", err) + } + if left := up.Leftovers(); len(left) != 1 || !strings.HasSuffix(left[0], ".ipa") { + t.Errorf("leftovers = %v", left) + } +} + +func TestAzureFromConfigReadsTheEnvironment(t *testing.T) { + env := map[string]string{"AZURE_STORAGE_CONNECTION_STRING": "DefaultEndpointsProtocol=https;AccountName=fromcs;AccountKey=" + fakeAzureKey + ";EndpointSuffix=core.windows.net"} + b, err := AzureFromConfig(nil, 0, func(k string) string { return env[k] }) + if err == nil || !strings.Contains(err.Error(), "container") { + t.Fatalf("no container: %v, %v", b, err) + } + cfgB, err := AzureFromConfig(&config.DistributeConfig{Container: "c"}, 0, func(k string) string { return env[k] }) + if err != nil || cfgB.store.(*azureStore).account != "fromcs" { + t.Fatalf("connection string: %v", err) + } + env = map[string]string{"AZURE_STORAGE_ACCOUNT": "acct"} + if _, err := AzureFromConfig(&config.DistributeConfig{Container: "c"}, 0, func(k string) string { return env[k] }); err == nil || !strings.Contains(err.Error(), "AZURE_STORAGE_KEY") { + t.Errorf("no key: %v", err) + } +} diff --git a/internal/otainstall/qr_test.go b/internal/otainstall/qr_test.go index 0f9e208..cacd222 100644 --- a/internal/otainstall/qr_test.go +++ b/internal/otainstall/qr_test.go @@ -3,6 +3,7 @@ package otainstall import ( "strings" "testing" + "time" "unicode/utf8" ) @@ -20,6 +21,46 @@ func TestQRFitsATerminal(t *testing.T) { } } +// TestBucketQRSizes pins the code size of each bucket backend's link, as +// documented: a SigV4 presigned manifest URL is ~345 characters, so S3 needs +// version 13 (69 modules, 73 columns with the quiet zone) where a gist needs +// version 6; R2's longer host is version 14 (73); Azure's SAS is short, +// version 10 (57). Temporary AWS credentials put their session token in the +// URL (~105 modules), which the session flags as too wide for 80 columns. +func TestBucketQRSizes(t *testing.T) { + now := time.Date(2026, 10, 4, 12, 0, 0, 0, time.UTC) + aws := AWSCredentials{AccessKeyID: "AKIAIOSFODNN7EXAMPLE", SecretAccessKey: "secret"} + s3, _ := NewS3(&S3Options{Bucket: "my-app-builds", Region: "eu-central-1", Credentials: aws}) + r2, _ := NewS3(&S3Options{Bucket: "builds", Region: "auto", Endpoint: "https://0123456789abcdef0123456789abcdef.r2.cloudflarestorage.com", + Credentials: AWSCredentials{AccessKeyID: "0123456789abcdef0123456789abcdef", SecretAccessKey: "secret"}}) + sts, _ := NewS3(&S3Options{Bucket: "my-app-builds", Region: "us-east-1", + Credentials: AWSCredentials{AccessKeyID: "ASIAIOSFODNN7EXAMPLE", SecretAccessKey: "secret", SessionToken: strings.Repeat("A", 800)}}) + azure, _ := NewAzure(&AzureOptions{Account: "myappbuilds", Container: "builds", Key: "c2VjcmV0"}) + for _, tc := range []struct { + name string + b *Bucket + max int + fits bool + }{ + {"s3", s3, 69, true}, {"r2", r2, 73, true}, {"azure", azure, 57, true}, {"s3 session token", sts, 105, false}, + } { + u, err := tc.b.store.presign(BucketDir+"0123abcd/"+manifestObject, time.Hour, now) + if err != nil { + t.Fatal(err) + } + modules, err := qrModules(Link(u)) + if err != nil { + t.Fatal(err) + } + if n := len(modules); n > tc.max { + t.Errorf("%s: QR code is %d modules wide, documented as %d", tc.name, n, tc.max) + } + if fits := len(modules)+2*quietZone <= wideQR; fits != tc.fits { + t.Errorf("%s: %d modules, fits 80 columns = %v", tc.name, len(modules), fits) + } + } +} + func TestQRRendersHalfBlocks(t *testing.T) { modules, err := qrModules("hello") if err != nil { From 4664ed0997aaf9acc3b6ca77f822af5a6ce51c94 Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:46:49 +0200 Subject: [PATCH 5/9] distribute: --backend, --ttl and --group for ios distribute and ios build --backend (else distribute.backend, else github) picks the store; the target is resolved before a build is pushed, so a missing bucket, credential or group fails first. testflight uploads an App Store IPA and adds it to the internal group --group; ios build --distribute --backend testflight runs ios release to that one internal group, which also picks the next build number. --ttl sets the s3/azure link lifetime; the backend flags on ios build need --distribute. --- cmd/builder/distribute.go | 193 ++++++++++++++++++++++++++++----- cmd/builder/distribute_test.go | 103 ++++++++++++++++++ cmd/builder/root.go | 15 ++- 3 files changed, 277 insertions(+), 34 deletions(-) diff --git a/cmd/builder/distribute.go b/cmd/builder/distribute.go index ce924e7..b8bdf72 100644 --- a/cmd/builder/distribute.go +++ b/cmd/builder/distribute.go @@ -3,6 +3,7 @@ package main import ( "context" "encoding/json" + "errors" "fmt" "io" "os" @@ -12,8 +13,10 @@ import ( "time" "github.com/MobAI-App/ios-builder/internal/config" + "github.com/MobAI-App/ios-builder/internal/distribute" "github.com/MobAI-App/ios-builder/internal/ipa" "github.com/MobAI-App/ios-builder/internal/otainstall" + "github.com/MobAI-App/ios-builder/internal/release" "github.com/spf13/cobra" "golang.org/x/term" ) @@ -27,13 +30,29 @@ TestFlight, no cable, no Mac. The IPA must be signed with a development or ad-hoc profile that lists the phone (builder signing setup --device or --devices-from-mobai) or an -enterprise profile; App Store builds cannot be installed this way. +enterprise profile; App Store builds go through --backend testflight. -The IPA goes to a draft release in the project's GitHub repository (no tag, -not visible on the repository page) and the manifest to a secret gist. Links -live five minutes; the command refreshes them while it runs and removes both -uploads when it ends (q, Ctrl-C or --timeout). --once prints one link and -leaves them; --cleanup removes what earlier sessions left behind. +Backends (--backend, else distribute.backend in builder.json, else github): + + github a draft release in the project's repository (no tag, not on the + repository page) holds the IPA, a secret gist the manifest. Links + live five minutes and are refreshed while the command runs. + s3 an S3 bucket, or an S3-compatible one through distribute.endpoint + (Cloudflare R2, MinIO, Google Cloud Storage with HMAC keys): + distribute.bucket, .region, .prefix. Credentials from + AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY / AWS_SESSION_TOKEN or + AWS_PROFILE in ~/.aws/credentials. Presigned links live --ttl. + azure an Azure Blob container: distribute.account, .container, .prefix; + the key from AZURE_STORAGE_KEY or AZURE_STORAGE_CONNECTION_STRING. + Service SAS links live --ttl. + testflight uploads an App Store signed IPA to App Store Connect, waits for + processing and adds it to the internal TestFlight group --group + (or distribute.group; created if missing, external groups refused). + Testers install from the TestFlight app; no link, no QR code. + +The uploads are removed when the session ends (q, Ctrl-C or --timeout). --once +prints one link and leaves them (with s3/azure the link lives --ttl, up to +seven days); --cleanup removes what earlier sessions left behind. Takes the newest IPA in --output, or --ipa. builder ios build --distribute builds first.`, @@ -45,16 +64,29 @@ func init() { f := iosDistributeCmd.Flags() f.String("ipa", "", "IPA to distribute (default: the newest in the output directory)") f.StringP("output", "o", "dist", "Directory holding the IPA") - f.Duration("timeout", time.Hour, "How long to keep the link alive") + f.Duration("timeout", time.Hour, "How long to keep the link alive (testflight: how long to wait for processing)") f.Bool("once", false, "Print one link and leave the upload in place (no refresh, no cleanup)") - f.Bool("cleanup", false, "Remove the draft releases and manifest gists earlier sessions left, then exit") + f.Bool("cleanup", false, "Remove the uploads earlier sessions left behind, then exit") f.Bool("qr", false, "Print the QR code even when stdout is not a terminal") f.Bool("no-qr", false, "Never print the QR code") f.Bool("qr-invert", false, "Render the QR code for a dark-on-light terminal") f.Bool("json", false, "Print one JSON object per link (progress goes to stderr); no QR code, no prompt") + addDistributeFlags(f.String, f.Duration) + f.String("notes", "", "What to Test notes for the build (testflight)") + f.Bool("no-encryption", false, "Declare the app uses no non-exempt encryption (testflight)") iosCmd.AddCommand(iosDistributeCmd) - iosBuildCmd.Flags().Bool("distribute", false, "After the build, print an over-the-air install link and QR code (see: ios distribute)") + bf := iosBuildCmd.Flags() + bf.Bool("distribute", false, "After the build, print an over-the-air install link and QR code (see: ios distribute)") + addDistributeFlags(bf.String, bf.Duration) +} + +// addDistributeFlags declares the backend flags ios distribute and ios build +// --distribute share. +func addDistributeFlags(str func(string, string, string) *string, dur func(string, time.Duration, string) *time.Duration) { + str("backend", "", "Where the install goes: github, s3, azure or testflight (default: distribute.backend in builder.json, else github)") + dur("ttl", 0, "Link lifetime for s3 and azure (2m to 168h; default 1h)") + str("group", "", "Internal TestFlight group for --backend testflight (default: distribute.group in builder.json)") } func runIOSDistribute(cmd *cobra.Command, _ []string) error { @@ -65,8 +97,12 @@ func runIOSDistribute(cmd *cobra.Command, _ []string) error { if err := cfg.Validate(); err != nil { return fmt.Errorf("invalid configuration: %w", err) } + target, err := distributeTarget(cmd, cfg) + if err != nil { + return err + } if cleanup, _ := cmd.Flags().GetBool("cleanup"); cleanup { - return runDistributeCleanup(cmd, cfg) + return runDistributeCleanup(cmd, target) } path, _ := cmd.Flags().GetString("ipa") if path == "" { @@ -75,17 +111,72 @@ func runIOSDistribute(cmd *cobra.Command, _ []string) error { return err } } - return runDistribute(cmd, cfg, path) + return runDistribute(cmd, target, path) } -func runDistributeCleanup(cmd *cobra.Command, cfg *config.Config) error { - backend, err := distributeBackend(cfg) +// target is a resolved --backend: Backend for the over-the-air ones, Group +// for testflight. +type target struct { + Name string + Backend otainstall.Backend + Group string +} + +// distributeTarget resolves the backend and its settings, so a missing bucket, +// credential or group fails before a build is pushed. +func distributeTarget(cmd *cobra.Command, cfg *config.Config) (*target, error) { + flag, _ := cmd.Flags().GetString("backend") + name, err := otainstall.BackendName(flag, cfg) if err != nil { - return err + return nil, err + } + ttl, _ := cmd.Flags().GetDuration("ttl") + group, _ := cmd.Flags().GetString("group") + dc := cfg.Distribute + if dc == nil { + dc = &config.DistributeConfig{} + } + t := &target{Name: name} + if ttl != 0 && name != otainstall.BackendS3 && name != otainstall.BackendAzure { + return nil, fmt.Errorf("--ttl applies to the s3 and azure backends; %s links have a fixed lifetime", name) + } + if group != "" && name != otainstall.BackendTestFlight { + return nil, fmt.Errorf("--group applies to --backend testflight") + } + switch name { + case otainstall.BackendGitHub: + gh, err := getGitHubClient() + if err != nil { + return nil, err + } + t.Backend = otainstall.NewGitHub(gh, cfg.GitHub.Owner, cfg.GitHub.Repo) + case otainstall.BackendS3: + if t.Backend, err = otainstall.S3FromConfig(dc, ttl, os.Getenv); err != nil { + return nil, err + } + case otainstall.BackendAzure: + if t.Backend, err = otainstall.AzureFromConfig(dc, ttl, os.Getenv); err != nil { + return nil, err + } + case otainstall.BackendTestFlight: + t.Group = group + if t.Group == "" { + t.Group = dc.Group + } + if t.Group == "" { + return nil, errors.New("--backend testflight needs an internal TestFlight group: pass --group or set distribute.group in builder.json (a missing group is created)") + } + } + return t, nil +} + +func runDistributeCleanup(cmd *cobra.Command, t *target) error { + if t.Backend == nil { + return fmt.Errorf("--backend %s leaves nothing to clean up; expire old builds with builder asc builds expire", t.Name) } ctx, cancel := commandContext(cmd, false) defer cancel() - n, err := backend.Cleanup(ctx) + n, err := t.Backend.Cleanup(ctx) if err != nil { return err } @@ -96,26 +187,17 @@ func runDistributeCleanup(cmd *cobra.Command, cfg *config.Config) error { return nil } -func distributeBackend(cfg *config.Config) (otainstall.Backend, error) { - gh, err := getGitHubClient() - if err != nil { - return nil, err - } - return otainstall.NewGitHub(gh, cfg.GitHub.Owner, cfg.GitHub.Repo), nil -} - // runDistribute is the flow behind `ios distribute` and `ios build --distribute`. -func runDistribute(cmd *cobra.Command, cfg *config.Config, ipaPath string) error { - app, err := otainstall.Inspect(ipaPath, otainstall.BackendGitHub) +func runDistribute(cmd *cobra.Command, t *target, ipaPath string) error { + app, err := otainstall.Inspect(ipaPath, t.Name) if err != nil { return err } - backend, err := distributeBackend(cfg) - if err != nil { - return err + if t.Name == otainstall.BackendTestFlight { + return runDistributeTestFlight(cmd, t, ipaPath) } out := newOutput(cmd) - opts := &otainstall.Options{App: app, Backend: backend, Log: out.log, Stdin: cmd.InOrStdin()} + opts := &otainstall.Options{App: app, Backend: t.Backend, Log: out.log, Stdin: cmd.InOrStdin()} if cmd.Name() == "distribute" { // ios build's --timeout bounds the build, not the link opts.Timeout, _ = cmd.Flags().GetDuration("timeout") } @@ -151,6 +233,59 @@ func runDistribute(cmd *cobra.Command, cfg *config.Config, ipaPath string) error return nil } +// runDistributeTestFlight uploads an App Store IPA and hands it to the +// internal group: the TestFlight app is the install link. +func runDistributeTestFlight(cmd *cobra.Command, t *target, ipaPath string) error { + client, err := getASCClient() + if err != nil { + return err + } + out := newOutput(cmd) + notes, _ := cmd.Flags().GetString("notes") + noEncryption, _ := cmd.Flags().GetBool("no-encryption") + ctx, cancel := commandContext(cmd, true) + defer cancel() + res, err := distribute.ToInternalGroup(ctx, client, &distribute.InternalGroupOptions{IPAPath: ipaPath, Group: t.Group, Notes: notes, NoEncryption: noEncryption, Log: out.log}) + return finish(out, cmd, res, err, func() { + w := cmd.OutOrStdout() + tf := res.TestFlight + fmt.Fprintln(w) + fmt.Fprintf(w, "Build: %s (build %s)\n", tf.Build.ID, tf.Build.BuildNumber) + for _, g := range tf.Groups { + fmt.Fprintf(w, "Group: %s (internal)\n", g.Name) + } + fmt.Fprintf(w, "Link: %s\n", tf.Link) + fmt.Fprintf(w, "Testers in %s install it from the TestFlight app on their iPhone.\n", t.Group) + }) +} + +// buildDistributeTarget is the preflight of `ios build --distribute`: the +// profile must sign the way the backend needs and the backend must be +// configured, all before anything is pushed. +func buildDistributeTarget(cmd *cobra.Command, cfg *config.Config, profile string) (*target, error) { + flag, _ := cmd.Flags().GetString("backend") + name, err := otainstall.BackendName(flag, cfg) + if err != nil { + return nil, err + } + s, err := cfg.ResolveProfile(profile) + if err != nil { + return nil, err + } + if err := otainstall.CheckDistribution(&s, name); err != nil { + return nil, err + } + return distributeTarget(cmd, cfg) +} + +// runBuildTestFlight is `ios build --distribute --backend testflight`: ios +// release to the one internal group, which also picks the next build number +// App Store Connect will accept. +func runBuildTestFlight(cmd *cobra.Command, cfg *config.Config, t *target, opts *release.Options) error { + opts.Groups, opts.InternalGroups = []string{t.Group}, true + return runRelease(cmd, cfg, opts) +} + // uploadProgress draws the IPA upload as a bar on one line, redrawn when the // percentage changes, and ends the line when the upload does. func uploadProgress(w io.Writer) func(done, total int64) { diff --git a/cmd/builder/distribute_test.go b/cmd/builder/distribute_test.go index 0172439..089e0af 100644 --- a/cmd/builder/distribute_test.go +++ b/cmd/builder/distribute_test.go @@ -1,6 +1,9 @@ package main import ( + "archive/zip" + "os" + "path/filepath" "strings" "testing" @@ -50,3 +53,103 @@ func TestDistributeNeedsAnIPA(t *testing.T) { t.Errorf("err = %v", err) } } + +// TestBuildDistributeBackendPreflight: the backend's needs are checked before +// anything is pushed, and the backend flags need --distribute. +func TestBuildDistributeBackendPreflight(t *testing.T) { + t.Chdir(t.TempDir()) + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + t.Setenv("AZURE_STORAGE_CONNECTION_STRING", "") + t.Setenv("AZURE_STORAGE_ACCOUNT", "") + t.Setenv("AZURE_STORAGE_KEY", "") + cfg := &config.Config{Project: "App", Platform: "ios", GitHub: config.GitHubConfig{Owner: "o", Repo: "r"}, Profiles: map[string]config.Profile{ + "store": {Distribution: "store"}, + "dev": {Distribution: "development"}, + }} + if err := config.NewManager().Save(cfg); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + args []string + want string + }{ + {[]string{"--backend", "s3"}, "--backend goes with --distribute"}, + {[]string{"--submit", "--group", "Team"}, "--group goes with --distribute"}, + {[]string{"--distribute", "--backend", "ftp", "--profile", "dev"}, `unknown distribute backend "ftp"`}, + {[]string{"--distribute", "--backend", "testflight", "--profile", "dev"}, "TestFlight takes only App Store builds"}, + {[]string{"--distribute", "--backend", "testflight", "--profile", "store"}, "needs an internal TestFlight group"}, + {[]string{"--distribute", "--backend", "s3", "--profile", "store"}, `profile "store" has distribution store`}, + {[]string{"--distribute", "--backend", "s3", "--profile", "dev"}, "needs a bucket"}, + {[]string{"--distribute", "--backend", "s3", "--profile", "dev", "--group", "Team"}, "--group applies to --backend testflight"}, + {[]string{"--distribute", "--backend", "azure", "--profile", "dev"}, "needs an account and a container"}, + {[]string{"--distribute", "--profile", "dev", "--ttl", "1h"}, "--ttl applies to the s3 and azure backends"}, + } { + _, _, err := run(t, append([]string{"ios", "build"}, tc.args...)...) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%v: err = %v, want %q", tc.args, err, tc.want) + } + } + + // distribute.backend in builder.json applies without the flag. + cfg.Distribute = &config.DistributeConfig{Backend: "s3", Bucket: "b"} + if err := config.NewManager().Save(cfg); err != nil { + t.Fatal(err) + } + _, _, err := run(t, "ios", "build", "--distribute", "--profile", "dev", "--ttl", "200h") + if err == nil || !strings.Contains(err.Error(), "--ttl must be between") { + t.Errorf("config backend: err = %v", err) + } +} + +func writeSignedIPA(t *testing.T, profileBody string) string { + t.Helper() + path := filepath.Join(t.TempDir(), "App.ipa") + f, err := os.Create(path) + if err != nil { + t.Fatal(err) + } + zw := zip.NewWriter(f) + for name, body := range map[string]string{ + "Payload/App.app/Info.plist": `CFBundleIdentifiercom.example.appCFBundleShortVersionString1.0CFBundleVersion3`, + "Payload/App.app/embedded.mobileprovision": "\x30\x82" + `` + profileBody + ``, + } { + w, _ := zw.Create(name) + _, _ = w.Write([]byte(body)) + } + if err := zw.Close(); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } + return path +} + +func TestDistributeBackendChecksTheIPA(t *testing.T) { + t.Chdir(t.TempDir()) + cfg := &config.Config{Project: "App", Platform: "ios", GitHub: config.GitHubConfig{Owner: "o", Repo: "r"}, Distribute: &config.DistributeConfig{Group: "Team"}} + if err := config.NewManager().Save(cfg); err != nil { + t.Fatal(err) + } + dev := writeSignedIPA(t, `ProvisionedDevicesu1Entitlementsget-task-allow`) + store := writeSignedIPA(t, `Entitlementsget-task-allow`) + _, _, err := run(t, "ios", "distribute", "--backend", "testflight", "--ipa", dev) + if err == nil || !strings.Contains(err.Error(), "TestFlight takes only App Store signed builds") { + t.Errorf("dev IPA to testflight: %v", err) + } + t.Setenv("AWS_ACCESS_KEY_ID", "AK") + t.Setenv("AWS_SECRET_ACCESS_KEY", "SK") + cfg.Distribute.Bucket = "b" + if err := config.NewManager().Save(cfg); err != nil { + t.Fatal(err) + } + _, _, err = run(t, "ios", "distribute", "--backend", "s3", "--ipa", store) + if err == nil || !strings.Contains(err.Error(), "use --backend testflight") { + t.Errorf("store IPA over the air: %v", err) + } + _, _, err = run(t, "ios", "distribute", "--backend", "testflight", "--cleanup") + if err == nil || !strings.Contains(err.Error(), "leaves nothing to clean up") { + t.Errorf("testflight cleanup: %v", err) + } +} diff --git a/cmd/builder/root.go b/cmd/builder/root.go index 74d2d49..7352f78 100644 --- a/cmd/builder/root.go +++ b/cmd/builder/root.go @@ -678,22 +678,27 @@ func runIOSBuild(cmd *cobra.Command, args []string) error { if submit && distribute { return fmt.Errorf("pass only one of --submit (TestFlight) or --distribute (over-the-air install)") } + for _, name := range []string{"backend", "ttl", "group"} { + if cmd.Flags().Changed(name) && !distribute { + return fmt.Errorf("--%s goes with --distribute", name) + } + } if submit { if opts.Unsigned { return fmt.Errorf("--submit uploads to App Store Connect, which needs a signed build; drop --unsigned") } return runRelease(cmd, cfg, &release.Options{Build: opts}) } + var dist *target if distribute { if opts.Unsigned { return fmt.Errorf("--distribute installs on a device, which needs a signed build; drop --unsigned") } - s, err := cfg.ResolveProfile(opts.Profile) - if err != nil { + if dist, err = buildDistributeTarget(cmd, cfg, opts.Profile); err != nil { return err } - if err := otainstall.CheckDistribution(&s, otainstall.BackendGitHub); err != nil { - return err + if dist.Name == otainstall.BackendTestFlight { + return runBuildTestFlight(cmd, cfg, dist, &release.Options{Build: opts}) } } @@ -715,7 +720,7 @@ func runIOSBuild(cmd *cobra.Command, args []string) error { if err != nil || !distribute { return err } - return runDistribute(cmd, cfg, result.IPAPath) + return runDistribute(cmd, dist, result.IPAPath) } func runIOSShare(cmd *cobra.Command, args []string) error { From 4b794d8e9c0fecc4621124f311328c7b01f0d984 Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:48:58 +0200 Subject: [PATCH 6/9] otainstall: satisfy the linter in the bucket backends and their tests --- internal/otainstall/azure_test.go | 15 +++++++++++---- internal/otainstall/s3_test.go | 4 ++-- internal/otainstall/session.go | 3 ++- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/internal/otainstall/azure_test.go b/internal/otainstall/azure_test.go index cb457a8..3dd4143 100644 --- a/internal/otainstall/azure_test.go +++ b/internal/otainstall/azure_test.go @@ -7,7 +7,6 @@ import ( "crypto/sha256" "encoding/base64" "fmt" - "github.com/MobAI-App/ios-builder/internal/config" "io" "net/http" "net/http/httptest" @@ -18,6 +17,8 @@ import ( "sync" "testing" "time" + + "github.com/MobAI-App/ios-builder/internal/config" ) const fakeAzureKey = "ZmFrZS1henVyZS1hY2NvdW50LWtleQ==" @@ -49,7 +50,10 @@ func TestAzureSASMatchesTheDocumentedStringToSign(t *testing.T) { if err != nil { t.Fatal(err) } - store := b.store.(*azureStore) + store, ok := b.store.(*azureStore) + if !ok { + t.Fatal("not an azure store") + } exp := time.Date(2026, 10, 4, 13, 0, 0, 0, time.UTC) got, _ := url.ParseQuery(store.sas("ios-builder/x/m.plist", "r", exp)) if got.Get("sig") != azureSig("r", "2026-10-04T13:00:00Z", "b", "/blob/acct/builds/ios-builder/x/m.plist") || @@ -239,7 +243,7 @@ func TestAzureLinkInstalls(t *testing.T) { if strings.Count(res.Link, "%25") != strings.Count(res.Link, "%252B") { t.Errorf("link has double-escaped characters, which cost QR versions: %s", res.Link) } - _, ipa := install(t, f.srv.Client(), res.Links) + _, ipa := install(t, f.srv.Client(), &res.Links) want, _ := os.ReadFile(app.Path) if !bytes.Equal(ipa, want) { t.Error("installed IPA differs") @@ -285,7 +289,10 @@ func TestAzureFromConfigReadsTheEnvironment(t *testing.T) { t.Fatalf("no container: %v, %v", b, err) } cfgB, err := AzureFromConfig(&config.DistributeConfig{Container: "c"}, 0, func(k string) string { return env[k] }) - if err != nil || cfgB.store.(*azureStore).account != "fromcs" { + if err != nil { + t.Fatalf("connection string: %v", err) + } + if store, ok := cfgB.store.(*azureStore); !ok || store.account != "fromcs" { t.Fatalf("connection string: %v", err) } env = map[string]string{"AZURE_STORAGE_ACCOUNT": "acct"} diff --git a/internal/otainstall/s3_test.go b/internal/otainstall/s3_test.go index 162d141..d5f648a 100644 --- a/internal/otainstall/s3_test.go +++ b/internal/otainstall/s3_test.go @@ -220,7 +220,7 @@ func (f *fakeS3) keys() []string { } // install does what iOS does with a link: fetch the manifest, then the IPA it names. -func install(t *testing.T, client *http.Client, links Links) (manifest string, ipa []byte) { +func install(t *testing.T, client *http.Client, links *Links) (manifest string, ipa []byte) { t.Helper() get := func(u string) []byte { resp, err := client.Get(u) @@ -276,7 +276,7 @@ func TestS3SessionRefreshesInstallsAndCleansUp(t *testing.T) { } // Both links install, the older one serving the newest manifest. want, _ := os.ReadFile(app.Path) - for _, l := range []Links{first, second} { + for _, l := range []*Links{&first, &second} { manifest, ipa := install(t, f.srv.Client(), l) if !bytes.Equal(ipa, want) || !strings.Contains(manifest, "run.mobai.tapdash") { t.Errorf("install from %s fetched %d bytes", l.ManifestURL, len(ipa)) diff --git a/internal/otainstall/session.go b/internal/otainstall/session.go index ebd5944..2536722 100644 --- a/internal/otainstall/session.go +++ b/internal/otainstall/session.go @@ -173,7 +173,8 @@ func (o *Options) print(res *Result) error { } fmt.Fprintln(o.Log) fmt.Fprint(o.Log, qr) - if width := utf8.RuneCountInString(qr[:strings.Index(qr, "\n")]); width > wideQR { + firstLine, _, _ := strings.Cut(qr, "\n") + if width := utf8.RuneCountInString(firstLine); width > wideQR { fmt.Fprintf(o.Log, "\nThe QR code is %d columns wide because the signed URL is long (temporary credentials add their session token); widen the terminal or zoom out to scan it, or open the link on the phone.\n", width) } } From 1485412c77343d1e2475ddebb4d80e28992cde7b Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:49:25 +0200 Subject: [PATCH 7/9] docs: distribute backends (s3, azure, testflight) in the README and CLAUDE.md --- CLAUDE.md | 35 ++++++++++++++++++++++++++-- README.md | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 103 insertions(+), 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 9c2b1e1..f05107a 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,6 +43,8 @@ go install ./cmd/builder ./builder ios build --profile store --submit # Short for: ios release (no groups) ./builder ios build --profile development --distribute # Build, then print an over-the-air install link + QR code ./builder ios distribute [--ipa x.ipa] [--once] [--json] # Same for an existing IPA; --cleanup removes leftovers +./builder ios distribute --backend s3|azure [--ttl 168h] # Bucket backends (distribute.* in builder.json) +./builder ios distribute --backend testflight --group # App Store IPA to an internal group ./builder asc apps|builds|groups|testers|users # App Store Connect listings (--json) ./builder asc groups create [--external] # also: groups delete, groups add-build ./builder asc testers add ... --group # also: testers remove, users invite @@ -197,6 +199,12 @@ builder ios distribute ──► otainstall.Inspect: Info.plist + embedded.mobil ▼ Print link + QR (half blocks); re-mint a minute before expiry or on Enter; q / Ctrl-C / --timeout → DELETE gist + release + --backend s3|azure (otainstall.Bucket): PUT IPA + m.plist under + ios-builder// (marker metadata) → SigV4 presign / service + SAS for --ttl → DELETE both on exit + --backend testflight: Inspect wants a store IPA → + distribute.ToInternalGroup (group check → Upload wait → SubmitTestFlight + Internal); ios build --distribute → release.Run with InternalGroups ``` ### Module Layout @@ -214,7 +222,8 @@ internal/ # beta groups, beta testers, team users/invitations, review) distribute/ # Upload / TestFlight / App Store / tester flows on top of asc ipa/ # Info.plist and embedded.mobileprovision reading from .ipa archives - otainstall/ # ios distribute: over-the-air install links (manifest, QR, GitHub draft release + gist backend) + otainstall/ # ios distribute: over-the-air install links (manifest, QR; backends: GitHub draft + # release + gist, S3/S3-compatible via SigV4, Azure Blob via service SAS) build/ # Build coordination (snapshot + trigger + poll + download) signing/ # CSR generation, .p12 assembly, and Auto (portal-free provisioning on top of asc) snapshot/ # Working-tree snapshot as a throwaway commit on a remote ref @@ -407,7 +416,7 @@ internal/ interface a foreign build backend implements. - **OTA Install, Not OTA Updates** (`internal/otainstall`): `ios distribute` serves a whole signed IPA through an `itms-services://` link; iOS installs only development/ad-hoc (device on the profile) or - enterprise builds, so `Inspect` refuses unsigned and App Store IPAs and `CheckDistribution` refuses + enterprise builds, so `Inspect` refuses unsigned and App Store IPAs (except for `--backend testflight`) and `CheckDistribution` refuses the profile of `ios build --distribute` before the snapshot push. `--distribute` excludes `--submit`. - **Draft Releases Create No Tag**: the IPA is an asset of a draft release tagged `ios-builder/distribute-` (nothing in `refs/tags`, nothing on the repo page). `GET releases/assets/{id}` with `Accept: @@ -428,6 +437,28 @@ internal/ - **QR Rendering**: `skip2/go-qrcode` at error-correction Low, Unicode half blocks (two module rows per line, 2-module quiet zone), light modules as `█` so it scans on a dark terminal (`--qr-invert` for light); `TestQRFitsATerminal` pins a representative link at 41 modules (version 6). Printed only on a TTY or `--qr`. +- **Distribute Backends**: `--backend`, else `distribute.backend`, else github (`otainstall.BackendName`). + `cmd/builder` resolves the whole target (`distributeTarget`: client, bucket, credentials, group) before a + build is pushed; `--ttl` is s3/azure only, `--group` testflight only, and on `ios build` all three need + `--distribute`. `Inspect`/`CheckDistribution` take the backend: testflight is the inverse of the OTA check + (store only; an empty profile passes so `release.Preflight` can pick the only store profile). +- **Bucket Backends** (`otainstall.Bucket` over `objectStore`): one folder `ios-builder//` with the + IPA and `m.plist` (short: the manifest URL is the QR code). Mint presigns the IPA, rewrites the manifest in + place (an older, still valid link serves the newest one) and presigns it; `ExpiresAt` is now + `--ttl` + (2m to 168h, SigV4's cap, kept for azure too). Cleanup lists the folder and deletes only marked objects + (S3: `x-amz-meta-ios-builder` via HEAD, since listings carry no metadata; Azure: `iosbuilder` via + `include=metadata`). Missing keys on DELETE are success. +- **SigV4 Without The SDK** (`sigv4.go`): header signing signs every header on the request plus host; query + presign signs host only with `UNSIGNED-PAYLOAD`; PUTs stream with `UNSIGNED-PAYLOAD`. `TestSigV4*` holds AWS's + published S3 vectors, and the fake S3 re-signs what arrives on the wire. Custom endpoints are path-style; + AWS is virtual-hosted unless the bucket has a dot. Credentials: `AWS_*` env, else `AWS_PROFILE`/`default` in + the shared credentials file (static keys only: SSO/`credential_process` are refused with a hint). +- **Azure Service SAS**: every request, Builder's own included, carries a SAS (`sv=2022-11-02`, the 16-field + string-to-sign of 2020-12-06+); only the signature's `+` is escaped, since `Link` doubles every `%`. +- **QR Size Per Backend**: github 41 modules, azure 57, s3 69 (R2 73), s3 with a session token ~105. + `TestBucketQRSizes` pins them; `Options.print` adds a note when the code is wider than 80 columns. +- **Hosted Short Links** (not built): a MobAI-hosted link would be one more `Backend` whose `Mint` returns a + short manifest URL; it needs a server that does not exist yet. - **Signing Sets As A Library**: `signing.Setup` and `signing.EnsureSecrets` (internal/signing/sets.go) hold the non-interactive core of `signing setup` and on-demand provisioning; cmd/builder keeps the prompts, the plan and the diff --git a/README.md b/README.md index cc24fe8..3093818 100644 --- a/README.md +++ b/README.md @@ -256,6 +256,8 @@ builder ios build --profile development --distribute # Build, then print an ins builder ios distribute # Same for the newest IPA in ./dist/ (or --ipa) builder ios distribute --once # One link, no refresh, uploads left in place builder ios distribute --cleanup # Remove uploads earlier sessions left behind +builder ios distribute --backend s3 --once --ttl 168h # A week-long link from an S3/R2 bucket +builder ios distribute --backend testflight --group Team # App Store build to an internal TestFlight group # App Store Connect management (needs builder auth apple) builder asc apps # Apps the API key can see @@ -872,6 +874,74 @@ drops the code, `--qr` prints it off a terminal and `--qr-invert` renders it for a dark-on-light one. Codemagic and Bitrise builds work the same way, since the uploads always go to the GitHub repository in `builder.json`. +### Other backends + +`--backend` picks where the install goes; without it `distribute.backend` in +`builder.json` decides, else GitHub as above. Every flag works the same on +`ios build --distribute`, and the backend is checked before the build is pushed. + +| Backend | Holds | Link lifetime | QR code | +| --- | --- | --- | --- | +| `github` (default) | draft release + secret gist | 5 minutes, refreshed | 41 modules | +| `s3` | S3 or S3-compatible bucket | `--ttl`, 2m to 7 days (default 1h) | 69 modules (R2: 73) | +| `azure` | Azure Blob container | `--ttl`, 2m to 7 days (default 1h) | 57 modules | +| `testflight` | App Store Connect, internal group | as long as the build | none | + +```json +"distribute": { + "backend": "s3", + "bucket": "my-app-builds", + "region": "eu-central-1", + "prefix": "ota/" +} +``` + +**s3** works with Amazon S3 and with S3-compatible stores through `endpoint` +(addressed path-style): Cloudflare R2 (`"endpoint": +"https://.r2.cloudflarestorage.com", "region": "auto"`), MinIO, or +Google Cloud Storage with HMAC keys (`"endpoint": +"https://storage.googleapis.com"`). Credentials come from +`AWS_ACCESS_KEY_ID`/`AWS_SECRET_ACCESS_KEY` (plus `AWS_SESSION_TOKEN`), else the +`AWS_PROFILE` (or `default`) section of `~/.aws/credentials`; the region from +`region`, `AWS_REGION` or `AWS_DEFAULT_REGION`, else `us-east-1`. The key needs +`s3:PutObject`, `s3:GetObject`, `s3:DeleteObject` and, for `--cleanup`, +`s3:ListBucket`. The bucket stays private: the IPA and a small manifest go to +`ios-builder//` and are linked with SigV4 presigned URLs. A +presigned URL is about 350 characters, so the QR code is larger than GitHub's +(69 modules, 73 columns with its border, still inside an 80-column terminal). +Temporary credentials (SSO, assumed roles) put their session token in the URL +and the code grows to about 105 modules; Builder says so when it does not fit, +and the link itself works either way. A presigned URL also never outlives the +credentials that signed it. + +**azure** needs `"account"` and `"container"` (and `"endpoint"` for Azurite or +a sovereign cloud) and the account key in `AZURE_STORAGE_KEY` or +`AZURE_STORAGE_CONNECTION_STRING`; links are service SAS URLs signed with that +key. The container stays private. + +With either bucket backend `--ttl` sets how long a link lives; the session +re-mints a minute before expiry as usual, and `--once --ttl 168h` gives a link +to send around that stays valid for a week. Ending a session deletes both +objects; `--once` leaves them, and `--cleanup` deletes every object under +`ios-builder/` that carries Builder's marker metadata, nothing else. + +**testflight** is the route for App Store signed builds, which cannot be +installed over the air: + +```bash +builder ios distribute --backend testflight --group Team # an existing App Store IPA +builder ios build --profile store --distribute --backend testflight --group Team +``` + +It uploads the IPA (needs `builder auth apple`), waits until App Store Connect +has processed it and adds it to the **internal** group `--group` (or +`distribute.group`), which is created when missing. Internal groups need no +beta review, so testers install from the TestFlight app within minutes; an +external group is refused before the upload. `ios build --distribute +--backend testflight` is `ios release` to that one group, so it also picks +the next build number. Instead of a QR code it prints the build and its App +Store Connect link; `--notes` and `--no-encryption` work as for `ios submit`. + Alternatively, [MobAI](https://mobai.run) installs an IPA over the cable, signed or not: an unsigned IPA can be re-signed on install with a free Apple ID (MobAI asks for the account). From 9f00185bed9e35f0ab9b8d815bb178a50e7996ed Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 09:52:59 +0200 Subject: [PATCH 8/9] distribute: find the IPA before resolving the backend ios distribute without a GitHub login reported the missing token instead of the missing IPA, as it did before backends existed. --- cmd/builder/distribute.go | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cmd/builder/distribute.go b/cmd/builder/distribute.go index b8bdf72..cf2c962 100644 --- a/cmd/builder/distribute.go +++ b/cmd/builder/distribute.go @@ -97,19 +97,26 @@ func runIOSDistribute(cmd *cobra.Command, _ []string) error { if err := cfg.Validate(); err != nil { return fmt.Errorf("invalid configuration: %w", err) } - target, err := distributeTarget(cmd, cfg) - if err != nil { - return err - } if cleanup, _ := cmd.Flags().GetBool("cleanup"); cleanup { + target, err := distributeTarget(cmd, cfg) + if err != nil { + return err + } return runDistributeCleanup(cmd, target) } + // The IPA is found before any client or credential is needed. path, _ := cmd.Flags().GetString("ipa") if path == "" { dir, _ := cmd.Flags().GetString("output") if path, err = ipa.Newest(dir); err != nil { return err } + } else if _, err := os.Stat(path); err != nil { + return fmt.Errorf("open IPA: %w", err) + } + target, err := distributeTarget(cmd, cfg) + if err != nil { + return err } return runDistribute(cmd, target, path) } From b29a5a0ff4a40440a285d9d1852b3cdf8dfffaed Mon Sep 17 00:00:00 2001 From: Interlap Date: Sun, 4 Oct 2026 11:40:56 +0200 Subject: [PATCH 9/9] docs: drop the hosted short link note --- CLAUDE.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index f05107a..fec2366 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -457,8 +457,6 @@ internal/ string-to-sign of 2020-12-06+); only the signature's `+` is escaped, since `Link` doubles every `%`. - **QR Size Per Backend**: github 41 modules, azure 57, s3 69 (R2 73), s3 with a session token ~105. `TestBucketQRSizes` pins them; `Options.print` adds a note when the code is wider than 80 columns. -- **Hosted Short Links** (not built): a MobAI-hosted link would be one more `Backend` whose `Mint` returns a - short manifest URL; it needs a server that does not exist yet. - **Signing Sets As A Library**: `signing.Setup` and `signing.EnsureSecrets` (internal/signing/sets.go) hold the non-interactive core of `signing setup` and on-demand provisioning; cmd/builder keeps the prompts, the plan and the