Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add config value to make optional outputs required for remote execution #3189

Merged
merged 4 commits into from
Jul 4, 2024
Merged

Add config value to make optional outputs required for remote execution #3189

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
ffc312a
Add config value to make optional outputs required for remote execution
peterebden Jul 3, 2024
d9a3a6d
gofmt
peterebden Jul 3, 2024
7a86649
Update docs
peterebden Jul 3, 2024
e716720
use NeedCoverage
peterebden Jul 3, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions docs/config.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1014,6 +1014,12 @@ <h3 class="mt1 f6 lh-title" id="remote.uploaddirs">UploadDirs <span class="norma
<p>{{ index .ConfigHelpText "remote.uploaddirs" }}</p>
</div>
</li>
<li>
<div>
<h3 class="mt1 f6 lh-title" id="remote.optionaloutputsrequired">OptionalOutputsRequired <span class="normal">(bool)</span></h3>
<p>{{ index .ConfigHelpText "remote.optionaloutputsrequired" }}</p>
</div>
</li>
<li>
<div>
<h3 class="mt1 f6 lh-title" id="remote.shell">Shell </h3>
Expand Down
33 changes: 17 additions & 16 deletions src/core/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -558,22 +558,23 @@ type Configuration struct {
ExcludeableTargets []BuildLabel `help:"If set, only targets that match these wildcards will be allowed to opt out of the sandbox"`
} `help:"A config section describing settings relating to sandboxing of build actions."`
Remote struct {
URL string `help:"URL for the remote server."`
CASURL string `help:"URL for the CAS service, if it is different to the main one."`
AssetURL string `help:"URL for the remote asset server, if it is different to the main one."`
NumExecutors int `help:"Maximum number of remote executors to use simultaneously."`
Instance string `help:"Remote instance name to request; depending on the server this may be required."`
Name string `help:"A name for this worker instance. This is attached to artifacts uploaded to remote storage." example:"agent-001"`
DisplayURL string `help:"A URL to browse the remote server with (e.g. using buildbarn-browser). Only used when printing hashes."`
TokenFile string `help:"A file containing a token that is attached to outgoing RPCs to authenticate them. This is somewhat bespoke; we are still investigating further options for authentication."`
Timeout cli.Duration `help:"Timeout for connections made to the remote server."`
Secure bool `help:"Whether to use TLS for communication or not."`
VerifyOutputs bool `help:"Whether to verify all outputs are present after a cached remote execution action. Depending on your server implementation, you may require this to ensure files are really present."`
UploadDirs bool `help:"Uploads individual directory blobs after build actions. This might not be necessary with some servers, but if you aren't sure, you should leave it on."`
Shell string `help:"Path to the shell to use to execute actions in. Default is 'bash' which will be looked up by the server."`
Platform []string `help:"Platform properties to request from remote workers, in the format key=value."`
CacheDuration cli.Duration `help:"Length of time before we re-check locally cached build actions. Default is unlimited."`
BuildID string `help:"ID of the build action that's being run, to attach to remote requests. If not set then one is automatically generated."`
URL string `help:"URL for the remote server."`
CASURL string `help:"URL for the CAS service, if it is different to the main one."`
AssetURL string `help:"URL for the remote asset server, if it is different to the main one."`
NumExecutors int `help:"Maximum number of remote executors to use simultaneously."`
Instance string `help:"Remote instance name to request; depending on the server this may be required."`
Name string `help:"A name for this worker instance. This is attached to artifacts uploaded to remote storage." example:"agent-001"`
DisplayURL string `help:"A URL to browse the remote server with (e.g. using buildbarn-browser). Only used when printing hashes."`
TokenFile string `help:"A file containing a token that is attached to outgoing RPCs to authenticate them. This is somewhat bespoke; we are still investigating further options for authentication."`
Timeout cli.Duration `help:"Timeout for connections made to the remote server."`
Secure bool `help:"Whether to use TLS for communication or not."`
VerifyOutputs bool `help:"Whether to verify all outputs are present after a cached remote execution action. Depending on your server implementation, you may require this to ensure files are really present."`
UploadDirs bool `help:"Uploads individual directory blobs after build actions. This might not be necessary with some servers, but if you aren't sure, you should leave it on."`
OptionalOutputsRequired bool `help:"Requires that any optional outputs of build actions (optional test outputs, coverage when not opted out of) are produced. By default this is a non-fatal failure, but the actions may not cache remotely."`
Shell string `help:"Path to the shell to use to execute actions in. Default is 'bash' which will be looked up by the server."`
Platform []string `help:"Platform properties to request from remote workers, in the format key=value."`
CacheDuration cli.Duration `help:"Length of time before we re-check locally cached build actions. Default is unlimited."`
BuildID string `help:"ID of the build action that's being run, to attach to remote requests. If not set then one is automatically generated."`
} `help:"Settings related to remote execution & caching using the Google remote execution APIs. This section is still experimental and subject to change."`
Size map[string]*Size `help:"Named sizes of targets; these are the definitions of what can be passed to the 'size' argument."`
Cover struct {
Expand Down
12 changes: 11 additions & 1 deletion src/remote/action.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -446,9 +446,19 @@ func (c *Client) verifyActionResult(target *core.BuildTarget, command *pb.Comman
outs := outputsForActionResult(ar)
// Test outputs are optional
if isTest {
if !outs[core.TestResultsFile] && !target.Test.NoOutput {
if !target.Test.NoOutput && !outs[core.TestResultsFile] {
return fmt.Errorf("Remote build action for %s failed to produce output %s%s", target, core.TestResultsFile, c.actionURL(actionDigest, true))
}
if c.state.Config.Remote.OptionalOutputsRequired {
if target.NeedCoverage(c.state) && !outs[core.CoverageFile] {
return fmt.Errorf("Remote build action for %s failed to produce output %s%s", target, core.CoverageFile, c.actionURL(actionDigest, true))
}
for _, out := range target.Test.Outputs {
if !outs[out] {
return fmt.Errorf("Remote build action for %s failed to produce output %s%s", target, out, c.actionURL(actionDigest, true))
}
}
}
} else {
for _, out := range command.OutputPaths {
if !outs[out] {
Expand Down
Loading