From 7bc891e00be4263311d75aa2b2ee6a3b7b75355f Mon Sep 17 00:00:00 2001 From: Alex Kube Date: Wed, 23 Oct 2019 21:18:12 +0430 Subject: [PATCH] cmd/dist: separate host and target builds Upstream-Status: Inappropriate [OE specific] Change the dist tool to allow for OE-style cross- and cross-canadian builds: - command flags --host-only and --target only are added; if one is present, the other changes mentioned below take effect, and arguments may also be specified on the command line to enumerate the package(s) to be built. - for OE cross builds, go_bootstrap is always built for the current build host, and is moved, along with the supporting toolchain (asm, compile, etc.) to a separate 'native_native' directory under GOROOT/pkg/tool. - go_bootstrap is not automatically removed after the build, so it can be reused later (e.g., building both static and shared runtime). Note that for --host-only builds, it would be nice to specify just the "cmd" package to build only the go commands/tools, the staleness checks in the dist tool will fail if the "std" library has not also been built. So host-only builds have to build everything anyway. Adapted to Go 1.13 from patches originally submitted to the meta/recipes-devtools/go tree by Matt Madison . Signed-off-by: Alexander J Kube --- src/cmd/dist/build.go | 156 ++++++++++++++++++++++++++++++------------ 1 file changed, 113 insertions(+), 43 deletions(-) --- a/src/cmd/dist/build.go +++ b/src/cmd/dist/build.go @@ -44,6 +44,7 @@ var ( goexperiment string workdir string tooldir string + build_tooldir string oldgoos string oldgoarch string exe string @@ -54,6 +55,7 @@ var ( rebuildall bool defaultclang bool + crossBuild bool vflag int // verbosity ) @@ -254,6 +256,8 @@ func xinit() { if tooldir = os.Getenv("GOTOOLDIR"); tooldir == "" { tooldir = pathf("%s/pkg/tool/%s_%s", goroot, gohostos, gohostarch) } + + build_tooldir = pathf("%s/pkg/tool/native_native", goroot) } // compilerEnv returns a map from "goos/goarch" to the @@ -499,8 +503,10 @@ func setup() { p := pathf("%s/pkg/%s_%s", goroot, gohostos, gohostarch) if rebuildall { xremoveall(p) + xremoveall(build_tooldir) } xmkdirall(p) + xmkdirall(build_tooldir) if goos != gohostos || goarch != gohostarch { p := pathf("%s/pkg/%s_%s", goroot, goos, goarch) @@ -1252,17 +1258,35 @@ func cmdbootstrap() { var noBanner, noClean bool var debug bool + var hostOnly bool + var targetOnly bool + var toBuild = []string{"std", "cmd"} + flag.BoolVar(&rebuildall, "a", rebuildall, "rebuild all") flag.BoolVar(&debug, "d", debug, "enable debugging of bootstrap process") flag.BoolVar(&noBanner, "no-banner", noBanner, "do not print banner") flag.BoolVar(&noClean, "no-clean", noClean, "print deprecation warning") + flag.BoolVar(&hostOnly, "host-only", hostOnly, "build only host binaries, not target") + flag.BoolVar(&targetOnly, "target-only", targetOnly, "build only target binaries, not host") - xflagparse(0) + xflagparse(-1) if noClean { xprintf("warning: --no-clean is deprecated and has no effect; use 'go install std cmd' instead\n") } + if hostOnly && targetOnly { + fatalf("specify only one of --host-only or --target-only\n") + } + crossBuild = hostOnly || targetOnly + if flag.NArg() > 0 { + if crossBuild { + toBuild = flag.Args() + } else { + fatalf("package names not permitted without --host-only or --target-only\n") + } + } + // Set GOPATH to an internal directory. We shouldn't actually // need to store files here, since the toolchain won't // depend on modules outside of vendor directories, but if @@ -1330,8 +1354,13 @@ func cmdbootstrap() { xprintf("\n") } - gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now - goldflags = os.Getenv("GO_LDFLAGS") // we were using $BOOT_GO_LDFLAGS until now + // For split host/target cross/cross-canadian builds, we don't + // want to be setting these flags until after we have compiled + // the toolchain that runs on the build host. + if !crossBuild { + gogcflags = os.Getenv("GO_GCFLAGS") // we were using $BOOT_GO_GCFLAGS until now + goldflags = os.Getenv("GO_LDFLAGS") // we were using $BOOT_GO_LDFLAGS until now + } goBootstrap := pathf("%s/go_bootstrap", tooldir) cmdGo := pathf("%s/go", gobin) if debug { @@ -1360,7 +1389,11 @@ func cmdbootstrap() { xprintf("\n") } xprintf("Building Go toolchain2 using go_bootstrap and Go toolchain1.\n") - os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + if crossBuild { + os.Setenv("CC", defaultcc[""]) + } else { + os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + } // Now that cmd/go is in charge of the build process, enable GOEXPERIMENT. os.Setenv("GOEXPERIMENT", goexperiment) goInstall(goBootstrap, append([]string{"-i"}, toolchain...)...) @@ -1399,50 +1432,84 @@ func cmdbootstrap() { } checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...) - if goos == oldgoos && goarch == oldgoarch { - // Common case - not setting up for cross-compilation. - timelog("build", "toolchain") - if vflag > 0 { - xprintf("\n") + if crossBuild { + gogcflags = os.Getenv("GO_GCFLAGS") + goldflags = os.Getenv("GO_LDFLAGS") + tool_files, _ := filepath.Glob(pathf("%s/*", tooldir)) + for _, f := range tool_files { + copyfile(pathf("%s/%s", build_tooldir, filepath.Base(f)), f, writeExec) + xremove(f) + } + os.Setenv("GOTOOLDIR", build_tooldir) + goBootstrap = pathf("%s/go_bootstrap", build_tooldir) + if hostOnly { + timelog("build", "host toolchain") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building %s for host, %s/%s.\n", strings.Join(toBuild, ","), goos, goarch) + goInstall(goBootstrap, toBuild...) + checkNotStale(goBootstrap, toBuild...) + // Skip cmdGo staleness checks here, since we can't necessarily run the cmdGo binary + + timelog("build", "target toolchain") + if vflag > 0 { + xprintf("\n") + } + } else if targetOnly { + goos = oldgoos + goarch = oldgoarch + os.Setenv("GOOS", goos) + os.Setenv("GOARCH", goarch) + os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + xprintf("Building %s for target, %s/%s.\n", strings.Join(toBuild, ","), goos, goarch) + goInstall(goBootstrap, toBuild...) + checkNotStale(goBootstrap, toBuild...) + // Skip cmdGo staleness checks here, since we can't run the target's cmdGo binary } - xprintf("Building packages and commands for %s/%s.\n", goos, goarch) } else { - // GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH. - // Finish GOHOSTOS/GOHOSTARCH installation and then - // run GOOS/GOARCH installation. - timelog("build", "host toolchain") - if vflag > 0 { - xprintf("\n") + + if goos == oldgoos && goarch == oldgoarch { + // Common case - not setting up for cross-compilation. + timelog("build", "toolchain") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building packages and commands for %s/%s.\n", goos, goarch) + } else { + // GOOS/GOARCH does not match GOHOSTOS/GOHOSTARCH. + // Finish GOHOSTOS/GOHOSTARCH installation and then + // run GOOS/GOARCH installation. + timelog("build", "host toolchain") + if vflag > 0 { + xprintf("\n") + } + xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch) + goInstall(goBootstrap, "std", "cmd") + checkNotStale(goBootstrap, "std", "cmd") + checkNotStale(cmdGo, "std", "cmd") + + timelog("build", "target toolchain") + if vflag > 0 { + xprintf("\n") + } + goos = oldgoos + goarch = oldgoarch + os.Setenv("GOOS", goos) + os.Setenv("GOARCH", goarch) + os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) + xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch) } - xprintf("Building packages and commands for host, %s/%s.\n", goos, goarch) goInstall(goBootstrap, "std", "cmd") checkNotStale(goBootstrap, "std", "cmd") checkNotStale(cmdGo, "std", "cmd") - timelog("build", "target toolchain") - if vflag > 0 { - xprintf("\n") - } - goos = oldgoos - goarch = oldgoarch - os.Setenv("GOOS", goos) - os.Setenv("GOARCH", goarch) - os.Setenv("CC", compilerEnvLookup(defaultcc, goos, goarch)) - xprintf("Building packages and commands for target, %s/%s.\n", goos, goarch) - } - targets := []string{"std", "cmd"} - if goos == "js" && goarch == "wasm" { - // Skip the cmd tools for js/wasm. They're not usable. - targets = targets[:1] - } - goInstall(goBootstrap, targets...) - checkNotStale(goBootstrap, targets...) - checkNotStale(cmdGo, targets...) - if debug { - run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") - run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch)) - checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...) - copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec) + if debug { + run("", ShowOutput|CheckExit, pathf("%s/compile", tooldir), "-V=full") + run("", ShowOutput|CheckExit, pathf("%s/buildid", tooldir), pathf("%s/pkg/%s_%s/runtime/internal/sys.a", goroot, goos, goarch)) + checkNotStale(goBootstrap, append(toolchain, "runtime/internal/sys")...) + copyfile(pathf("%s/compile4", tooldir), pathf("%s/compile", tooldir), writeExec) + } } // Check that there are no new files in $GOROOT/bin other than @@ -1459,8 +1526,11 @@ func cmdbootstrap() { } } - // Remove go_bootstrap now that we're done. - xremove(pathf("%s/go_bootstrap", tooldir)) + // Except that for split host/target cross-builds, we need to + // keep it. + if !crossBuild { + xremove(pathf("%s/go_bootstrap", tooldir)) + } if goos == "android" { // Make sure the exec wrapper will sync a fresh $GOROOT to the device.