justan0th3rstud3nt
/
garble
mirror of https://github.com/burrowers/garble
1
0
Fork 0
Code Issues 1 Projects Releases Wiki Activity

use "go env -json" to collect env info all at once

Browse Source 
In the worst case scenario, when GOPRIVATE isn't set at all, we would
run these three commands:

* "go env GOPRIVATE", to fetch GOPRIVATE itself
* "go list -m", for GOPRIVATE's fallback
* "go version", to check the version of Go being used

Now that we support Go 1.16 and later, all these three can be obtained
via "go env -json":

	$ go env -json GOPRIVATE GOMOD GOVERSION
	{
		"GOMOD": "/home/mvdan/src/garble/go.mod",
		"GOPRIVATE": "",
		"GOVERSION": "go1.16.3"
	}

Note that we don't get the module path directly, but we can use the
x/mod/modfile Go API to parse it from the GOMOD file cheaply.

Notably, this also simplifies our Go version checking logic, as now we
get just the version string without the "go version" prefix and
"GOOS/GOARCH" suffix we don't care about.

This makes our code a bit more maintainable and robust. When running a
short incremental build, we can also see a small speed-up, as saving two
"go" invocations can save a few milliseconds:

	name           old time/op       new time/op       delta
	Build/Cache-8        168ms ± 0%        166ms ± 1%  -1.26%  (p=0.009 n=6+6)

	name           old bin-B         new bin-B         delta
	Build/Cache-8        6.36M ± 0%        6.36M ± 0%  +0.12%  (p=0.002 n=6+6)

	name           old sys-time/op   new sys-time/op   delta
	Build/Cache-8        222ms ± 2%        219ms ± 3%    ~     (p=0.589 n=6+6)

	name           old user-time/op  new user-time/op  delta
	Build/Cache-8        857ms ± 1%        846ms ± 1%  -1.31%  (p=0.041 n=6+6)
pull/314/head
Daniel Martí 4 years ago committed by lu4p
parent 24d5ff362c
commit 3afc993266
4 changed files with 77 additions and 57 deletions
Show Stats Download Patch File Download Diff File

4
hash.go
Unescape Escape View File

@ -92,8 +92,8 @@ func addGarbleToBuildIDComponent(inputHash []byte) []byte {
// We also need to add the selected options to the full version string, // We also need to add the selected options to the full version string,
// because all of them result in different output. We use spaces to // because all of them result in different output. We use spaces to
// separate the env vars and flags, to reduce the chances of collisions. // separate the env vars and flags, to reduce the chances of collisions.
if envGoPrivate != "" { if cache.GoEnv.GOPRIVATE != "" {
fmt.Fprintf(h, " GOPRIVATE=%s", envGoPrivate) fmt.Fprintf(h, " GOPRIVATE=%s", cache.GoEnv.GOPRIVATE)
} }
if opts.GarbleLiterals { if opts.GarbleLiterals {
fmt.Fprintf(h, " -literals") fmt.Fprintf(h, " -literals")

95
main.go
Unescape Escape View File

@ -7,6 +7,7 @@ import (
"bytes" "bytes"
"encoding/base64" "encoding/base64"
"encoding/binary" "encoding/binary"
"encoding/json"
"errors" "errors"
"flag" "flag"
"fmt" "fmt"
@ -16,6 +17,7 @@ import (
"go/token" "go/token"
"go/types" "go/types"
"io" "io"
"io/ioutil"
"log" "log"
mathrand "math/rand" mathrand "math/rand"
"os" "os"
@ -29,6 +31,7 @@ import (
"unicode" "unicode"
"unicode/utf8" "unicode/utf8"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module" "golang.org/x/mod/module"
"golang.org/x/mod/semver" "golang.org/x/mod/semver"
"golang.org/x/tools/go/ast/astutil" "golang.org/x/tools/go/ast/astutil"
@ -114,8 +117,6 @@ var (
}).(types.ImporterFrom) }).(types.ImporterFrom)
opts *flagOptions opts *flagOptions
envGoPrivate = os.Getenv("GOPRIVATE") // complemented by 'go env' later
) )
func obfuscatedTypesPackage(path string) *types.Package { func obfuscatedTypesPackage(path string) *types.Package {
@ -175,7 +176,7 @@ var errJustExit = errors.New("")
func goVersionOK() bool { func goVersionOK() bool {
const ( const (
minGoVersion = "v1.16.0" minGoVersionSemver = "v1.16.0"
suggestedGoVersion = "1.16.x" suggestedGoVersion = "1.16.x"
gitTimeFormat = "Mon Jan 2 15:04:05 2006 -0700" gitTimeFormat = "Mon Jan 2 15:04:05 2006 -0700"
@ -183,32 +184,30 @@ func goVersionOK() bool {
// Go 1.16 was released on Febuary 16th, 2021. // Go 1.16 was released on Febuary 16th, 2021.
minGoVersionDate := time.Date(2021, 2, 16, 0, 0, 0, 0, time.UTC) minGoVersionDate := time.Date(2021, 2, 16, 0, 0, 0, 0, time.UTC)
out, err := exec.Command("go", "version").CombinedOutput() version := cache.GoEnv.GOVERSION
rawVersion := strings.TrimSpace(string(out)) if version == "" {
if err != nil || !strings.HasPrefix(rawVersion, "go version ") { // Go 1.15.x and older do not have GOVERSION yet.
fmt.Fprintf(os.Stderr, `Can't get Go version: %v // We could go the extra mile and fetch it via 'go version',
// but we'd have to error anyway.
This is likely due to go not being installed/setup correctly. fmt.Fprintf(os.Stderr, "Go version is too old; please upgrade to Go %s or a newer devel version\n", suggestedGoVersion)
How to install Go: https://golang.org/doc/install
`, err)
return false return false
} }
rawVersion = strings.TrimPrefix(rawVersion, "go version ") if strings.HasPrefix(version, "devel ") {
commitAndDate := strings.TrimPrefix(version, "devel ")
tagIdx := strings.IndexByte(rawVersion, ' ')
tag := rawVersion[:tagIdx]
if tag == "devel" {
commitAndDate := rawVersion[tagIdx+1:]
// Remove commit hash and architecture from version // Remove commit hash and architecture from version
startDateIdx := strings.IndexByte(commitAndDate, ' ') + 1 startDateIdx := strings.IndexByte(commitAndDate, ' ') + 1
endDateIdx := strings.LastIndexByte(commitAndDate, ' ') if startDateIdx < 0 {
if endDateIdx <= 0 || endDateIdx <= startDateIdx {
// Custom version; assume the user knows what they're doing. // Custom version; assume the user knows what they're doing.
return true return true
} }
date := commitAndDate[startDateIdx:endDateIdx]
// TODO: once we support Go 1.17 and later, use the major Go
// version included in its devel versions:
//
// go version devel go1.17-8518aac314 ...
date := commitAndDate[startDateIdx:]
versionDate, err := time.Parse(gitTimeFormat, date) versionDate, err := time.Parse(gitTimeFormat, date)
if err != nil { if err != nil {
@ -220,13 +219,13 @@ How to install Go: https://golang.org/doc/install
return true return true
} }
fmt.Fprintf(os.Stderr, "Go version %q is too old; please upgrade to Go %s or a newer devel version\n", rawVersion, suggestedGoVersion) fmt.Fprintf(os.Stderr, "Go version %q is too old; please upgrade to Go %s or a newer devel version\n", version, suggestedGoVersion)
return false return false
} }
version := "v" + strings.TrimPrefix(tag, "go") versionSemver := "v" + strings.TrimPrefix(version, "go")
if semver.Compare(version, minGoVersion) < 0 { if semver.Compare(versionSemver, minGoVersionSemver) < 0 {
fmt.Fprintf(os.Stderr, "Go version %q is too old; please upgrade to Go %s\n", rawVersion, suggestedGoVersion) fmt.Fprintf(os.Stderr, "Go version %q is too old; please upgrade to Go %s\n", version, suggestedGoVersion)
return false return false
} }
@ -346,9 +345,6 @@ func mainErr(args []string) error {
// Note that it uses and modifies global state; in general, it should only be // Note that it uses and modifies global state; in general, it should only be
// called once from mainErr in the top-level garble process. // called once from mainErr in the top-level garble process.
func toolexecCmd(command string, args []string) (*exec.Cmd, error) { func toolexecCmd(command string, args []string) (*exec.Cmd, error) {
if !goVersionOK() {
return nil, errJustExit
}
// Split the flags from the package arguments, since we'll need // Split the flags from the package arguments, since we'll need
// to run 'go list' on the same set of packages. // to run 'go list' on the same set of packages.
flags, args := splitFlagsFromArgs(args) flags, args := splitFlagsFromArgs(args)
@ -374,10 +370,14 @@ func toolexecCmd(command string, args []string) (*exec.Cmd, error) {
cache.BuildFlags = append(cache.BuildFlags, "-test") cache.BuildFlags = append(cache.BuildFlags, "-test")
} }
if err := setGoPrivate(); err != nil { if err := fetchGoEnv(); err != nil {
return nil, err return nil, err
} }
if !goVersionOK() {
return nil, errJustExit
}
var err error var err error
cache.ExecPath, err = os.Executable() cache.ExecPath, err = os.Executable()
if err != nil { if err != nil {
@ -834,7 +834,7 @@ func isPrivate(path string) bool {
if path == "command-line-arguments" || strings.HasPrefix(path, "plugin/unnamed") { if path == "command-line-arguments" || strings.HasPrefix(path, "plugin/unnamed") {
return true return true
} }
return module.MatchPrefixPatterns(envGoPrivate, path) return module.MatchPrefixPatterns(cache.GoEnv.GOPRIVATE, path)
} }
// processImportCfg initializes importCfgEntries via the supplied flags, and // processImportCfg initializes importCfgEntries via the supplied flags, and
@ -1452,26 +1452,33 @@ func flagSetValue(flags []string, name, value string) []string {
return append(flags, name+"="+value) return append(flags, name+"="+value)
} }
func setGoPrivate() error { func fetchGoEnv() error {
if envGoPrivate == "" { out, err := exec.Command("go", "env", "-json",
// Try 'go env' too, to query ${CONFIG}/go/env as well. "GOPRIVATE", "GOMOD", "GOVERSION",
out, err := exec.Command("go", "env", "GOPRIVATE").CombinedOutput() ).CombinedOutput()
if err != nil { if err != nil {
return fmt.Errorf("%v: %s", err, out) fmt.Fprintf(os.Stderr, `Can't find Go toolchain: %v
This is likely due to go not being installed/setup correctly.
How to install Go: https://golang.org/doc/install
`, err)
return errJustExit
} }
envGoPrivate = string(bytes.TrimSpace(out)) if err := json.Unmarshal(out, &cache.GoEnv); err != nil {
return err
} }
// If GOPRIVATE isn't set and we're in a module, use its module // If GOPRIVATE isn't set and we're in a module, use its module
// path as a GOPRIVATE default. Include a _test variant too. // path as a GOPRIVATE default. Include a _test variant too.
if envGoPrivate == "" { // TODO(mvdan): we shouldn't need the _test variant here,
modpath, err := exec.Command("go", "list", "-m").Output() // as the import path should not include it; only the package name.
if err == nil { if cache.GoEnv.GOPRIVATE == "" {
path := string(bytes.TrimSpace(modpath)) if mod, err := ioutil.ReadFile(cache.GoEnv.GOMOD); err == nil {
envGoPrivate = path + "," + path + "_test" modpath := modfile.ModulePath(mod)
if modpath != "" {
cache.GoEnv.GOPRIVATE = modpath + "," + modpath + "_test"
}
} }
} }
// Explicitly set GOPRIVATE, since future garble processes won't
// query 'go env' again.
os.Setenv("GOPRIVATE", envGoPrivate)
return nil return nil
} }

7
shared.go
Unescape Escape View File

@ -37,6 +37,13 @@ type sharedCache struct {
// Once https://github.com/golang/go/issues/37475 is fixed, we // Once https://github.com/golang/go/issues/37475 is fixed, we
// can likely just use that. // can likely just use that.
BinaryContentID []byte BinaryContentID []byte
// From "go env", primarily.
GoEnv struct {
GOPRIVATE string // Set to the module path as a fallback.
GOMOD string
GOVERSION string
}
} }
var cache *sharedCache var cache *sharedCache

24
testdata/scripts/goversion.txt vendored
Unescape Escape View File

@ -5,33 +5,33 @@ go build -o .bin/go$exe ./fakego
env PATH=${WORK}/.bin${:}${PATH} env PATH=${WORK}/.bin${:}${PATH}
# An empty go version. # An empty go version.
env GO_VERSION='' env GOVERSION=''
! garble build ! garble build
stderr 'Can''t get Go version' stderr 'Go version is too old'
# We should error on a devel version that's too old. # We should error on a devel version that's too old.
env GO_VERSION='go version devel +afb5fca Sun Aug 07 00:00:00 2020 +0000 linux/amd64' env GOVERSION='devel +afb5fca Sun Aug 07 00:00:00 2020 +0000'
! garble build ! garble build
stderr 'Go version.*Aug 07.*too old; please upgrade to Go 1.16.x or a newer devel version' stderr 'Go version.*Aug 07.*too old; please upgrade to Go 1.16.x or a newer devel version'
# A future devel timestamp should be fine. # A future devel timestamp should be fine.
env GO_VERSION='go version devel +afb5fca Sun Sep 13 07:54:42 2021 +0000 linux/amd64' env GOVERSION='devel +afb5fca Sun Sep 13 07:54:42 2021 +0000'
! garble build ! garble build
stderr 'mocking the real build' stderr 'mocking the real build'
# We should error on a stable version that's too old. # We should error on a stable version that's too old.
env GO_VERSION='go version go1.14 windows/amd64' env GOVERSION='go1.14'
! garble build ! garble build
stderr 'Go version.*go1.14.*too old; please upgrade to Go 1.16.x' stderr 'Go version.*go1.14.*too old; please upgrade to Go 1.16.x'
! stderr 'or a newer devel version' ! stderr 'or a newer devel version'
# We should accept a future stable version. # We should accept a future stable version.
env GO_VERSION='go version go1.16.2 windows/amd64' env GOVERSION='go1.16.2'
! garble build ! garble build
stderr 'mocking the real build' stderr 'mocking the real build'
# We should accept custom devel strings. # We should accept custom devel strings.
env GO_VERSION='go version devel somecustomversion linux/amd64' env GOVERSION='devel somecustomversion'
! garble build ! garble build
stderr 'mocking the real build' stderr 'mocking the real build'
@ -48,13 +48,19 @@ func main() {}
package main package main
import ( import (
"encoding/json"
"fmt" "fmt"
"os" "os"
) )
func main() { func main() {
if len(os.Args) > 0 && os.Args[1] == "version" { if len(os.Args) > 0 && os.Args[1] == "env" {
fmt.Println(os.Getenv("GO_VERSION")) enc, _ := json.Marshal(struct{
GOVERSION string
} {
GOVERSION: os.Getenv("GOVERSION"),
})
fmt.Printf("%s\n", enc)
return return
} }
fmt.Fprintln(os.Stderr, "mocking the real build") fmt.Fprintln(os.Stderr, "mocking the real build")

Write Preview
Loading…
Cancel
Save