fietsje: implement
This introduces Fietsje, a little Go dependency manager.
For more information, see third_party/go/fietsje/README.md.
We also bump some dependencies while we're at it, notably, sqliboiler
now uses Go modules. If we weren't to do that, we'd have to add more
heuristics to Fietsje to handle the old version correctly.
Test Plan: fietsje is untested - I'll add some tests to it. Everything else is just regenerating basically the same repositories.bzl file, but with some bumped dependencies.
X-Origin-Diff: phab/D535
GitOrigin-RevId: 4fc919e1bd386bc3f3c1c53e672b1e3b9da17dfc
diff --git a/build/fietsje/transitive.go b/build/fietsje/transitive.go
new file mode 100644
index 0000000..f4a6e1d
--- /dev/null
+++ b/build/fietsje/transitive.go
@@ -0,0 +1,192 @@
+// Copyright 2020 The Monogon Project Authors.
+//
+// SPDX-License-Identifier: Apache-2.0
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+ "fmt"
+ "io/ioutil"
+ "log"
+ "os"
+ "strings"
+
+ "golang.org/x/mod/modfile"
+)
+
+// getTransitiveDeps is a hairy ball of heuristic used to find all recursively transitive dependencies of a given
+// dependency.
+// It downloads a given dependency using `go get`, and performs analysis of standard (go.mod/go.sum) and project-
+// specific dependency management configuration/lock files in order to build a full view of all known, versioned
+// transitive dependencies.
+func (d *dependency) getTransitiveDeps() (map[string]*dependency, error) {
+ // First, lock the dependency. Downloading it later will also return a sum, and we want to ensure both are the
+ // same.
+ err := d.lock()
+ if err != nil {
+ return nil, fmt.Errorf("could not lock: %v", err)
+ }
+
+ _, path, sum, err := d.download()
+ if err != nil {
+ return nil, fmt.Errorf("could not download: %v", err)
+ }
+
+ if sum != d.locked.sum {
+ return nil, fmt.Errorf("inconsistent sum: %q downloaded, %q in shelf/lock", sum, d.locked.sum)
+ }
+
+ exists := func(p string) bool {
+ full := fmt.Sprintf("%s/%s", path, p)
+ if _, err := os.Stat(full); err == nil {
+ return true
+ }
+ if err != nil && !os.IsExist(err) {
+ panic(fmt.Sprintf("checking file %q: %v", full, err))
+ }
+ return false
+ }
+
+ read := func(p string) []byte {
+ full := fmt.Sprintf("%s/%s", path, p)
+ data, err := ioutil.ReadFile(full)
+ if err != nil {
+ panic(fmt.Sprintf("reading file %q: %v", full, err))
+ }
+ return data
+ }
+
+ requirements := make(map[string]*dependency)
+
+ // Read & parse go.mod if present.
+ var mf *modfile.File
+ if exists("go.mod") {
+ log.Printf("%q: parsing go.mod\n", d.importpath)
+ data := read("go.mod")
+ mf, err = modfile.Parse("go.mod", data, nil)
+ if err != nil {
+ return nil, fmt.Errorf("parsing go.mod in %s: %v", d.importpath, err)
+ }
+ }
+
+ // If a go.mod file was present, interpret it to populate dependencies.
+ if mf != nil {
+ for _, req := range mf.Require {
+ requirements[req.Mod.Path] = d.child(req.Mod.Path, req.Mod.Version)
+ }
+ for _, rep := range mf.Replace {
+ // skip filesystem rewrites
+ if rep.New.Version == "" {
+ continue
+ }
+
+ requirements[rep.New.Path] = d.child(rep.New.Path, rep.New.Version)
+ }
+ }
+
+ // Read parse, and interpret. go.sum if present.
+ // This should bring into view all recursively transitive dependencies.
+ if exists("go.sum") {
+ log.Printf("%q: parsing go.sum", d.importpath)
+ data := read("go.sum")
+ for _, line := range strings.Split(string(data), "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+
+ parts := strings.Fields(line)
+ if len(parts) != 3 {
+ return nil, fmt.Errorf("parsing go.sum: unparseable line %q", line)
+ }
+
+ importpath, version := parts[0], parts[1]
+
+ // Skip if already created from go.mod.
+ // TODO(q3k): error if go.sum and go.mod disagree?
+ if _, ok := requirements[importpath]; ok {
+ continue
+ }
+
+ if strings.HasSuffix(version, "/go.mod") {
+ version = strings.TrimSuffix(version, "/go.mod")
+ }
+ requirements[importpath] = d.child(importpath, version)
+ }
+ }
+
+ // Special case: root Kubernetes repo - rewrite staging/ deps to k8s.io/ at correct versions, quit early.
+ // Kubernetes vendors all dependencies into vendor/, and also contains sub-projects (components) in staging/.
+ // This converts all staging dependencies into appropriately versioned k8s.io/<dep> paths.
+ if d.importpath == "k8s.io/kubernetes" {
+ log.Printf("%q: special case for Kubernetes main repository", d.importpath)
+ if mf == nil {
+ return nil, fmt.Errorf("k8s.io/kubernetes needs a go.mod")
+ }
+ // extract the version, turn into component version
+ version := d.version
+ if !strings.HasPrefix(version, "v") {
+ return nil, fmt.Errorf("invalid version format for k8s: %q", version)
+ }
+ version = version[1:]
+ componentVersion := fmt.Sprintf("kubernetes-%s", version)
+
+ // find all k8s.io 'components'
+ components := make(map[string]bool)
+ for _, rep := range mf.Replace {
+ if !strings.HasPrefix(rep.Old.Path, "k8s.io/") || !strings.HasPrefix(rep.New.Path, "./staging/src/") {
+ continue
+ }
+ components[rep.Old.Path] = true
+ }
+
+ // add them to planner at the 'kubernetes-$ver' tag
+ for component, _ := range components {
+ requirements[component] = d.child(component, componentVersion)
+ }
+ return requirements, nil
+ }
+
+ // Special case: github.com/containerd/containerd: read vendor.conf.
+ if d.importpath == "github.com/containerd/containerd" {
+ log.Printf("%q: special case for containerd", d.importpath)
+ if !exists("vendor.conf") {
+ panic("containerd needs vendor.conf")
+ }
+ data := read("vendor.conf")
+ for _, line := range strings.Split(string(data), "\n") {
+ // strip comments
+ parts := strings.SplitN(line, "#", 2)
+ line = parts[0]
+
+ // skip empty contents
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+
+ // read dep/version pairs
+ parts = strings.Fields(line)
+ if len(parts) < 2 {
+ return nil, fmt.Errorf("unparseable line in containerd vendor.conf: %q", line)
+ }
+ importpath, version := parts[0], parts[1]
+ requirements[importpath] = d.child(importpath, version)
+ }
+ return requirements, nil
+ }
+
+ return requirements, nil
+}