Adding a token for deploy-from-url LMCROSSITXSADEPLOY-3391

clients/mtaclient/fakes/fake_mta_client_builder.go

@@ -56,8 +56,8 @@ func (fb *FakeMtaClientBuilder) UploadMtaFile(file *os.File, result *models.File
 	fb.FakeMtaClient.UploadMtaFileReturns(result, resultError)
 	return fb
 }
-func (fb *FakeMtaClientBuilder) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string, result http.Header, resultError error) *FakeMtaClientBuilder {
-	fb.FakeMtaClient.StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl, namespace, result, resultError)
+func (fb *FakeMtaClientBuilder) StartUploadMtaArchiveFromUrl(fileUrl string, token string, namespace *string, result http.Header, resultError error) *FakeMtaClientBuilder {
+	fb.FakeMtaClient.StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl, token, namespace, result, resultError)
 	return fb
 }
 func (fb *FakeMtaClientBuilder) GetAsyncUploadJob(jobId string, namespace *string, result mtaclient.AsyncUploadJobResult, resultErr error) *FakeMtaClientBuilder {

clients/mtaclient/fakes/fake_mta_client_operations.go

@@ -680,18 +680,18 @@ func (fake *FakeMtaClientOperations) UploadMtaFileReturnsOnCall(i int, result1 *
 	}{result1, result2}
 }
 
-func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) {
+func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrl(fileUrl string, token string, namespace *string) (http.Header, error) {
 	fake.startUploadMtaArchiveFromUrlMutex.Lock()
 	result, specificReturn := fake.startUploadMtaArchiveFromUrlReturnsOnCall[fileUrl]
-	fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, namespace})
+	fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, token, namespace})
 	fake.startUploadMtaArchiveFromUrlMutex.Unlock()
 	if specificReturn {
 		return result.headers, result.err
 	}
 	return fake.startUploadMtaArchiveFromUrlReturns.result1, fake.startUploadMtaArchiveFromUrlReturns.result2
 }
 
-func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl string, namespace *string, headers http.Header, err error) {
+func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrlReturnsOnCall(fileUrl string, token string, namespace *string, headers http.Header, err error) {
 	fake.startUploadMtaArchiveFromUrlMutex.Lock()
 	if fake.startUploadMtaArchiveFromUrlReturnsOnCall == nil {
 		fake.startUploadMtaArchiveFromUrlReturnsOnCall = make(map[string]struct {
@@ -703,7 +703,7 @@ func (fake *FakeMtaClientOperations) StartUploadMtaArchiveFromUrlReturnsOnCall(f
 		headers http.Header
 		err     error
 	}{headers, err}
-	fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, namespace})
+	fake.recordInvocation("StartUploadMtaArchiveFromUrl", []interface{}{fileUrl, token, namespace})
 	fake.startUploadMtaArchiveFromUrlMutex.Unlock()
 }
 

clients/mtaclient/mta_client_operations.go

@@ -19,7 +19,7 @@ type MtaClientOperations interface {
 	GetOperationActions(operationID string) ([]string, error)
 	StartMtaOperation(operation models.Operation) (ResponseHeader, error)
 	UploadMtaFile(file util.NamedReadSeeker, fileSize int64, namespace *string) (*models.FileMetadata, error)
-	StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error)
+	StartUploadMtaArchiveFromUrl(fileUrl string, token string, namespace *string) (http.Header, error)
 	GetAsyncUploadJob(jobId string, namespace *string) (AsyncUploadJobResult, error)
 	GetMtaOperationLogContent(operationID, logID string) (string, error)
 }

clients/mtaclient/mta_rest_client.go

@@ -49,6 +49,19 @@ type AsyncUploadJobResult struct {
 	ClientActions  []string             `json:"client_actions,omitempty"`
 }
 
+// startUploadFromUrlBody is the JSON request body for POST /files/async. Credentials is optional and
+// omitted entirely when no token is supplied, so existing credential-less callers are unaffected.
+type startUploadFromUrlBody struct {
+	FileUrl     string                         `json:"file_url"`
+	Credentials *startUploadFromUrlCredentials `json:"credentials,omitempty"`
+}
+
+// startUploadFromUrlCredentials carries the bearer token used to authenticate the controller's
+// outbound fetch of the mtar from the remote URL (e.g. a GitHub token). It is never persisted.
+type startUploadFromUrlCredentials struct {
+	Token string `json:"token,omitempty"`
+}
+
 func NewMtaClient(host, spaceID string, rt http.RoundTripper, tokenFactory baseclient.TokenFactory) MtaClientOperations {
 	restURL := restBaseURL + spacesURL + spaceID
 	t := baseclient.NewHTTPTransport(host, restURL, rt)
@@ -310,15 +323,16 @@ func escapeQuotes(s string) string {
 	return strings.NewReplacer("\\", "\\\\", `"`, "\\\"").Replace(s)
 }
 
-func (c MtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) {
+func (c MtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, token string, namespace *string) (http.Header, error) {
 	requestUrl := "https://" + c.dsHost + "/" + restBaseURL + spacesURL + c.spaceGuid + "/files/async"
 	if namespace != nil && len(*namespace) != 0 {
 		requestUrl += "?namespace=" + *namespace
 	}
 
-	body := struct {
-		FileUrl string `json:"file_url"`
-	}{fileUrl}
+	body := startUploadFromUrlBody{FileUrl: fileUrl}
+	if token != "" {
+		body.Credentials = &startUploadFromUrlCredentials{Token: token}
+	}
 
 	bodyBytes, err := json.Marshal(body)
 	if err != nil {
@@ -330,12 +344,12 @@ func (c MtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *s
 		return nil, fmt.Errorf("could not create start async file upload request: %v", err)
 	}
 
-	token, err := c.TokenFactory.NewRawToken()
+	authToken, err := c.TokenFactory.NewRawToken()
 	if err != nil {
 		return nil, fmt.Errorf("could not get authentication token: %v", err)
 	}
 
-	req.Header.Set("Authorization", "Bearer "+token)
+	req.Header.Set("Authorization", "Bearer "+authToken)
 	req.Header.Set("Content-Type", "application/json")
 
 	cl := c.client.Transport.(*client.Runtime)

clients/mtaclient/retryable_mta_rest_client.go

@@ -136,9 +136,9 @@ func (c RetryableMtaRestClient) UploadMtaFile(file util.NamedReadSeeker, fileSiz
 	return resp.(*models.FileMetadata), nil
 }
 
-func (c RetryableMtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, namespace *string) (http.Header, error) {
+func (c RetryableMtaRestClient) StartUploadMtaArchiveFromUrl(fileUrl string, token string, namespace *string) (http.Header, error) {
 	uploadMtaArchiveFromUrlCb := func() (interface{}, error) {
-		return c.mtaClient.StartUploadMtaArchiveFromUrl(fileUrl, namespace)
+		return c.mtaClient.StartUploadMtaArchiveFromUrl(fileUrl, token, namespace)
 	}
 	resp, err := baseclient.CallWithRetry(uploadMtaArchiveFromUrlCb, c.MaxRetriesCount, c.RetryInterval)
 	if err != nil {

commands/deploy_command.go

@@ -58,8 +58,14 @@ const (
 	shouldBackupPreviousVersionOpt   = "backup-previous-version"
 	requireSecureParameters          = "require-secure-parameters"
 	disposableUserProvidedServiceOpt = "disposable-user-provided-service"
+	urlTokenOpt                      = "url-token"
 )
 
+// urlTokenEnv is the environment variable fallback for the deploy-from-URL bearer token. Preferring
+// an env var keeps the secret out of shell history and the process list; the --url-token flag
+// overrides it when both are set.
+const urlTokenEnv = "MULTIAPPS_URL_TOKEN"
+
 type listFlag struct {
 	elements []string
 }
@@ -123,7 +129,7 @@ func (c *DeployCommand) GetPluginCommand() plugin.Command {
    cf deploy -i OPERATION_ID -a ACTION [-u URL]
 
    Deploy a multi-target app archive referenced by a remote URL
-   <write MTA archive URL to STDOUT> | cf deploy [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u MTA_CONTROLLER_URL] [--retries RETRIES] [--no-start] [--namespace NAMESPACE] [--apply-namespace-app-names true/false] [--apply-namespace-service-names true/false] [--apply-namespace-app-routes true/false] [--apply-namespace-as-suffix true/false ] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--strategy STRATEGY] [--skip-testing-phase] [--skip-idle-start] [require-secure-parameters] [--disposable-user-provided-service] [--apps-start-timeout TIMEOUT] [--apps-stage-timeout TIMEOUT] [--apps-upload-timeout TIMEOUT] [--apps-task-execution-timeout TIMEOUT]` + util.UploadEnvHelpText,
+   <write MTA archive URL to STDOUT> | cf deploy [-e EXT_DESCRIPTOR[,...]] [-t TIMEOUT] [--version-rule VERSION_RULE] [-u MTA_CONTROLLER_URL] [--retries RETRIES] [--no-start] [--namespace NAMESPACE] [--apply-namespace-app-names true/false] [--apply-namespace-service-names true/false] [--apply-namespace-app-routes true/false] [--apply-namespace-as-suffix true/false ] [--delete-services] [--delete-service-keys] [--delete-service-brokers] [--keep-files] [--no-restart-subscribed-apps] [--do-not-fail-on-missing-permissions] [--abort-on-error] [--strategy STRATEGY] [--skip-testing-phase] [--skip-idle-start] [require-secure-parameters] [--disposable-user-provided-service] [--url-token TOKEN] [--apps-start-timeout TIMEOUT] [--apps-stage-timeout TIMEOUT] [--apps-upload-timeout TIMEOUT] [--apps-task-execution-timeout TIMEOUT]` + util.UploadEnvHelpText,
 
 			Options: map[string]string{
 				extDescriptorsOpt:                 "Extension descriptors",
@@ -161,6 +167,7 @@ func (c *DeployCommand) GetPluginCommand() plugin.Command {
 				util.GetShortOption(dependencyAwareStopOrderOpt):                "(EXPERIMENTAL) (STRATEGY: BLUE-GREEN, INCREMENTAL-BLUE-GREEN) Stop apps in a dependency-aware order during the resume phase of a blue-green deployment",
 				util.GetShortOption(requireSecureParameters):                    "(EXPERIMENTAL) Pass secrets to the deploy service in a secure way",
 				util.GetShortOption(disposableUserProvidedServiceOpt):           "Deploy when --require-secure-parameters flag is active for disposable UPS to be created and then deleted at the of the operation",
+				util.GetShortOption(urlTokenOpt):                                "(URL DEPLOY) Bearer token used to authenticate the fetch of the MTA archive from the remote URL (e.g. a GitHub token). Falls back to the MULTIAPPS_URL_TOKEN environment variable, which is preferred so the token does not appear in shell history",
 			},
 		},
 	}
@@ -217,6 +224,7 @@ func (c *DeployCommand) defineCommandOptions(flags *flag.FlagSet) {
 	flags.Bool(deleteServicesOpt, false, "")
 	flags.Bool(noStartOpt, false, "")
 	flags.String(namespaceOpt, "", "")
+	flags.String(urlTokenOpt, "", "")
 	flags.String(applyNamespaceAppNamesOpt, "", "")
 	flags.String(applyNamespaceServiceNamesOpt, "", "")
 	flags.String(applyNamespaceAppRoutesOpt, "", "")
@@ -298,7 +306,8 @@ func (c *DeployCommand) executeInternal(positionalArgs []string, dsHost string,
 		var fileId string
 		var schemaVersion string
 
-		asyncUploadJobResult := c.uploadFromUrl(mtaArchive, mtaClient, namespace, disableProgressBar)
+		urlToken := resolveUrlToken(GetStringOpt(urlTokenOpt, flags))
+		asyncUploadJobResult := c.uploadFromUrl(mtaArchive, urlToken, mtaClient, namespace, disableProgressBar)
 		if asyncUploadJobResult.ExecutionStatus == Failure {
 			return Failure
 		}
@@ -498,21 +507,31 @@ func parseMtaArchiveArgument(rawMtaArchive interface{}) (bool, string) {
 	return false, ""
 }
 
-func (c *DeployCommand) uploadFromUrl(url string, mtaClient mtaclient.MtaClientOperations, namespace string,
+// resolveUrlToken returns the deploy-from-URL bearer token, preferring the --url-token flag value and
+// falling back to the MULTIAPPS_URL_TOKEN environment variable. Returns "" when neither is set, in
+// which case no credentials are sent and the fetch behaves as before.
+func resolveUrlToken(flagValue string) string {
+	if flagValue != "" {
+		return flagValue
+	}
+	return os.Getenv(urlTokenEnv)
+}
+
+func (c *DeployCommand) uploadFromUrl(url string, token string, mtaClient mtaclient.MtaClientOperations, namespace string,
 	disableProgressBar bool) UploadFromUrlStatus {
 	encodedFileUrl := base64.URLEncoding.EncodeToString([]byte(url))
 	uploadStatus, _ := retrier.Execute[UploadFromUrlStatus](3, func() (UploadFromUrlStatus, error) {
 		progressBar := c.tryFetchMtarSize(url, disableProgressBar)
-		uploadFromUrlStatus := c.doUploadFromUrl(encodedFileUrl, mtaClient, namespace, progressBar)
+		uploadFromUrlStatus := c.doUploadFromUrl(encodedFileUrl, token, mtaClient, namespace, progressBar)
 		return uploadFromUrlStatus, nil
 	}, func(result UploadFromUrlStatus, err error) bool {
 		return shouldRetryUpload(result)
 	})
 	return uploadStatus
 }
 
-func (c *DeployCommand) doUploadFromUrl(encodedFileUrl string, mtaClient mtaclient.MtaClientOperations, namespace string, progressBar *pb.ProgressBar) UploadFromUrlStatus {
-	responseHeaders, err := mtaClient.StartUploadMtaArchiveFromUrl(encodedFileUrl, &namespace)
+func (c *DeployCommand) doUploadFromUrl(encodedFileUrl string, token string, mtaClient mtaclient.MtaClientOperations, namespace string, progressBar *pb.ProgressBar) UploadFromUrlStatus {
+	responseHeaders, err := mtaClient.StartUploadMtaArchiveFromUrl(encodedFileUrl, token, &namespace)
 	if err != nil {
 		var retryErr *baseclient.RetryAfterError
 		if errors.As(err, &retryErr) {

commands/deploy_command_test.go

@@ -70,6 +70,17 @@ func (f *mockFile) Close() error {
 	return nil
 }
 
+// lastUploadFromUrlToken returns the token argument of the most recent StartUploadMtaArchiveFromUrl
+// invocation recorded by the fake client. The recorded args are {fileUrl, token, namespace}.
+func lastUploadFromUrlToken(fake *mtafake.FakeMtaClientOperations) string {
+	calls := fake.Invocations()["StartUploadMtaArchiveFromUrl"]
+	if len(calls) == 0 {
+		return ""
+	}
+	lastCall := calls[len(calls)-1]
+	return lastCall[1].(string)
+}
+
 var _ = Describe("DeployCommand", func() {
 	Describe("Execute", func() {
 		const org = "test-org"
@@ -226,8 +237,8 @@ var _ = Describe("DeployCommand", func() {
 				GetMtaFiles([]*models.FileMetadata{&testutil.SimpleFile}, nil).
 				UploadMtaFile(mtaArchiveFile, mtaArchive, nil).
 				UploadMtaFile(extDescriptorFile, extDescriptor, nil).
-				StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(correctMtaUrl)), nil, fileUploadJobId, nil).
-				StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(incorrectMtaUrl)), nil, nil, fmt.Errorf("connection refused")).
+				StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(correctMtaUrl)), "", nil, fileUploadJobId, nil).
+				StartUploadMtaArchiveFromUrl(base64.URLEncoding.EncodeToString([]byte(incorrectMtaUrl)), "", nil, nil, fmt.Errorf("connection refused")).
 				GetAsyncUploadJob(jobId, nil, jobResult, nil).
 				StartMtaOperation(testutil.OperationResult, mtaclient.ResponseHeader{Location: "operations/1000?embed=messages"}, nil).
 				GetMtaOperation("1000", "messages", &testutil.OperationResult, nil).
@@ -283,6 +294,43 @@ var _ = Describe("DeployCommand", func() {
 			})
 		})
 
+		Context("with a correct URL argument and a --url-token flag", func() {
+			It("should pass the token through to the upload-from-URL client call", func() {
+				command.FileUrlReader = newMockFileReader(correctMtaUrl)
+				_, status := oc.CaptureOutputAndStatus(func() int {
+					return command.Execute([]string{"--url-token", "ghp_flag_token"}).ToInt()
+				})
+				Expect(status).To(Equal(0))
+				Expect(lastUploadFromUrlToken(mtaClient)).To(Equal("ghp_flag_token"))
+			})
+		})
+
+		Context("with a correct URL argument and the MULTIAPPS_URL_TOKEN env var", func() {
+			It("should fall back to the env var when no flag is given", func() {
+				os.Setenv("MULTIAPPS_URL_TOKEN", "ghp_env_token")
+				defer os.Unsetenv("MULTIAPPS_URL_TOKEN")
+				command.FileUrlReader = newMockFileReader(correctMtaUrl)
+				_, status := oc.CaptureOutputAndStatus(func() int {
+					return command.Execute([]string{}).ToInt()
+				})
+				Expect(status).To(Equal(0))
+				Expect(lastUploadFromUrlToken(mtaClient)).To(Equal("ghp_env_token"))
+			})
+		})
+
+		Context("with both a --url-token flag and the MULTIAPPS_URL_TOKEN env var", func() {
+			It("should prefer the flag over the env var", func() {
+				os.Setenv("MULTIAPPS_URL_TOKEN", "ghp_env_token")
+				defer os.Unsetenv("MULTIAPPS_URL_TOKEN")
+				command.FileUrlReader = newMockFileReader(correctMtaUrl)
+				_, status := oc.CaptureOutputAndStatus(func() int {
+					return command.Execute([]string{"--url-token", "ghp_flag_token"}).ToInt()
+				})
+				Expect(status).To(Equal(0))
+				Expect(lastUploadFromUrlToken(mtaClient)).To(Equal("ghp_flag_token"))
+			})
+		})
+
 		Context("with an incorrect URL argument", func() {
 			It("should fail with the error returned from the server", func() {
 				command.FileUrlReader = newMockFileReader(incorrectMtaUrl)