Add all dependencies for Kubernetes worker
Adds Kubelet, CNI plugins, containerd, runc and gVisor using a
pre-baked list of dependencies generated using scripts/gazelle-deps/sh.
This moves all dependencies of gVisor, Kubernetes, runc, etc into the
same 'namespace' of Bazel external repositories, giving us ease of
accessing code as libraries, and benefits when it comes to version
auditing.
The gazelle-deps.sh script is a temporary solution that will be replaced
ASAP, see T725.
This unblocks T486.
This is an alternative to D389.
Test Plan: `bazel build //core:image` runs and picks up the new binaries
X-Origin-Diff: phab/D487
GitOrigin-RevId: a28a25071fa2ae76b272d237ce9af777485065ff
diff --git a/third_party/go/patches/BUILD b/third_party/go/patches/BUILD
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/third_party/go/patches/BUILD
diff --git a/third_party/go/patches/cadvisor-build.patch b/third_party/go/patches/cadvisor-build.patch
new file mode 100644
index 0000000..5e7a11f
--- /dev/null
+++ b/third_party/go/patches/cadvisor-build.patch
@@ -0,0 +1,47 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This patch updates BUILD files to reflect changes in Go sources. This only needs to be applied because Gazelle applies patches after BUILDfile generation.
+
+diff -ur com_github_google_cadvisor.orig/fs/fs.go com_github_google_cadvisor/fs/fs.go
+--- com_github_google_cadvisor.orig/fs/BUILD.bazel 2020-04-15 17:54:23.086143468 +0200
++++ com_github_google_cadvisor/fs/BUILD.bazel 2020-04-15 17:56:17.098546899 +0200
+@@ -12,14 +12,12 @@
+ "@io_bazel_rules_go//go/platform:android": [
+ "//devicemapper:go_default_library",
+ "//utils:go_default_library",
+- "@com_github_mistifyio_go_zfs//:go_default_library",
+ "@io_k8s_klog//:go_default_library",
+ "@io_k8s_utils//mount:go_default_library",
+ ],
+ "@io_bazel_rules_go//go/platform:linux": [
+ "//devicemapper:go_default_library",
+ "//utils:go_default_library",
+- "@com_github_mistifyio_go_zfs//:go_default_library",
+ "@io_k8s_klog//:go_default_library",
+ "@io_k8s_utils//mount:go_default_library",
+ ],
+diff -ur com_github_google_cadvisor.orig/fs/fs.go com_github_google_cadvisor/fs/fs.go
+--- com_github_google_cadvisor.orig/container/docker/BUILD.bazel 2020-04-15 17:54:23.086143468 +0200
++++ com_github_google_cadvisor/container/docker/BUILD.bazel 2020-04-15 17:56:17.098546899 +0200
+@@ -21,7 +21,6 @@
+ "//info/v1:go_default_library",
+ "//machine:go_default_library",
+ "//watcher:go_default_library",
+- "//zfs:go_default_library",
+ "@com_github_blang_semver//:go_default_library",
+ "@com_github_docker_docker//api/types:go_default_library",
+ "@com_github_docker_docker//api/types/container:go_default_library",
+
diff --git a/third_party/go/patches/cadvisor.patch b/third_party/go/patches/cadvisor.patch
new file mode 100644
index 0000000..9761dee
--- /dev/null
+++ b/third_party/go/patches/cadvisor.patch
@@ -0,0 +1,415 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This rips out ZFS support from cadvisor. This should be turned into an upstream change for allowing zfs to be disabled via a build tag.
+
+diff -ur com_github_google_cadvisor.orig/container/docker/factory.go com_github_google_cadvisor/container/docker/factory.go
+--- com_github_google_cadvisor.orig/container/docker/factory.go 2020-04-15 17:54:23.082143453 +0200
++++ com_github_google_cadvisor/container/docker/factory.go 2020-04-16 15:08:47.394501543 +0200
+@@ -34,7 +34,6 @@
+ info "github.com/google/cadvisor/info/v1"
+ "github.com/google/cadvisor/machine"
+ "github.com/google/cadvisor/watcher"
+- "github.com/google/cadvisor/zfs"
+
+ docker "github.com/docker/docker/client"
+ "golang.org/x/net/context"
+@@ -102,7 +101,6 @@
+ aufsStorageDriver storageDriver = "aufs"
+ overlayStorageDriver storageDriver = "overlay"
+ overlay2StorageDriver storageDriver = "overlay2"
+- zfsStorageDriver storageDriver = "zfs"
+ )
+
+ type dockerFactory struct {
+@@ -127,8 +125,6 @@
+
+ thinPoolName string
+ thinPoolWatcher *devicemapper.ThinPoolWatcher
+-
+- zfsWatcher *zfs.ZfsWatcher
+ }
+
+ func (self *dockerFactory) String() string {
+@@ -157,7 +153,6 @@
+ self.includedMetrics,
+ self.thinPoolName,
+ self.thinPoolWatcher,
+- self.zfsWatcher,
+ )
+ return
+ }
+@@ -246,21 +241,6 @@
+ return thinPoolWatcher, nil
+ }
+
+-func startZfsWatcher(dockerInfo *dockertypes.Info) (*zfs.ZfsWatcher, error) {
+- filesystem, err := dockerutil.DockerZfsFilesystem(*dockerInfo)
+- if err != nil {
+- return nil, err
+- }
+-
+- zfsWatcher, err := zfs.NewZfsWatcher(filesystem)
+- if err != nil {
+- return nil, err
+- }
+-
+- go zfsWatcher.Start()
+- return zfsWatcher, nil
+-}
+-
+ func ensureThinLsKernelVersion(kernelVersion string) error {
+ // kernel 4.4.0 has the proper bug fixes to allow thin_ls to work without corrupting the thin pool
+ minKernelVersion := semver.MustParse("4.4.0")
+@@ -358,14 +338,6 @@
+ thinPoolName = status.DriverStatus[dockerutil.DriverStatusPoolName]
+ }
+
+- var zfsWatcher *zfs.ZfsWatcher
+- if storageDriver(dockerInfo.Driver) == zfsStorageDriver {
+- zfsWatcher, err = startZfsWatcher(dockerInfo)
+- if err != nil {
+- klog.Errorf("zfs filesystem stats will not be reported: %v", err)
+- }
+- }
+-
+ klog.V(1).Infof("Registering Docker factory")
+ f := &dockerFactory{
+ cgroupSubsystems: cgroupSubsystems,
+@@ -379,7 +351,6 @@
+ includedMetrics: includedMetrics,
+ thinPoolName: thinPoolName,
+ thinPoolWatcher: thinPoolWatcher,
+- zfsWatcher: zfsWatcher,
+ }
+
+ container.RegisterContainerHandlerFactory(f, []watcher.ContainerWatchSource{watcher.Raw})
+diff -ur com_github_google_cadvisor.orig/container/docker/handler.go com_github_google_cadvisor/container/docker/handler.go
+--- com_github_google_cadvisor.orig/container/docker/handler.go 2020-04-15 17:54:23.082143453 +0200
++++ com_github_google_cadvisor/container/docker/handler.go 2020-04-16 15:45:56.432489633 +0200
+@@ -25,12 +25,10 @@
+
+ "github.com/google/cadvisor/container"
+ "github.com/google/cadvisor/container/common"
+- dockerutil "github.com/google/cadvisor/container/docker/utils"
+ containerlibcontainer "github.com/google/cadvisor/container/libcontainer"
+ "github.com/google/cadvisor/devicemapper"
+ "github.com/google/cadvisor/fs"
+ info "github.com/google/cadvisor/info/v1"
+- "github.com/google/cadvisor/zfs"
+
+ dockercontainer "github.com/docker/docker/api/types/container"
+ docker "github.com/docker/docker/client"
+@@ -87,9 +85,6 @@
+ // the devicemapper poolname
+ poolName string
+
+- // zfsParent is the parent for docker zfs
+- zfsParent string
+-
+ // Reference to the container
+ reference info.ContainerReference
+
+@@ -130,7 +125,6 @@
+ includedMetrics container.MetricSet,
+ thinPoolName string,
+ thinPoolWatcher *devicemapper.ThinPoolWatcher,
+- zfsWatcher *zfs.ZfsWatcher,
+ ) (container.ContainerHandler, error) {
+ // Create the cgroup paths.
+ cgroupPaths := common.MakeCgroupPaths(cgroupSubsystems.MountPoints, name)
+@@ -164,8 +158,6 @@
+ // For devicemapper, we only need the thin pool name, and that is passed in to this call
+ var (
+ rootfsStorageDir string
+- zfsFilesystem string
+- zfsParent string
+ )
+ switch storageDriver {
+ case aufsStorageDriver:
+@@ -174,13 +166,6 @@
+ rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlayRWLayer)
+ case overlay2StorageDriver:
+ rootfsStorageDir = path.Join(storageDir, string(storageDriver), rwLayerID, overlay2RWLayer)
+- case zfsStorageDriver:
+- status, err := Status()
+- if err != nil {
+- return nil, fmt.Errorf("unable to determine docker status: %v", err)
+- }
+- zfsParent = status.DriverStatus[dockerutil.DriverStatusParentDataset]
+- zfsFilesystem = path.Join(zfsParent, rwLayerID)
+ }
+
+ // We assume that if Inspect fails then the container is not known to docker.
+@@ -200,7 +185,6 @@
+ envs: make(map[string]string),
+ labels: ctnr.Config.Labels,
+ includedMetrics: includedMetrics,
+- zfsParent: zfsParent,
+ }
+ // Timestamp returned by Docker is in time.RFC3339Nano format.
+ handler.creationTime, err = time.Parse(time.RFC3339Nano, ctnr.Created)
+@@ -244,9 +228,6 @@
+ handler.fsHandler = &dockerFsHandler{
+ fsHandler: common.NewFsHandler(common.DefaultPeriod, rootfsStorageDir, otherStorageDir, fsInfo),
+ thinPoolWatcher: thinPoolWatcher,
+- zfsWatcher: zfsWatcher,
+- deviceID: ctnr.GraphDriver.Data["DeviceId"],
+- zfsFilesystem: zfsFilesystem,
+ }
+ }
+
+@@ -271,7 +252,7 @@
+ }
+
+ // dockerFsHandler is a composite FsHandler implementation the incorporates
+-// the common fs handler, a devicemapper ThinPoolWatcher, and a zfsWatcher
++// the common fs handler and a devicemapper ThinPoolWatcher
+ type dockerFsHandler struct {
+ fsHandler common.FsHandler
+
+@@ -279,11 +260,6 @@
+ thinPoolWatcher *devicemapper.ThinPoolWatcher
+ // deviceID is the id of the container's fs device
+ deviceID string
+-
+- // zfsWatcher is the zfs filesystem watcher
+- zfsWatcher *zfs.ZfsWatcher
+- // zfsFilesystem is the docker zfs filesystem
+- zfsFilesystem string
+ }
+
+ var _ common.FsHandler = &dockerFsHandler{}
+@@ -316,16 +292,6 @@
+ usage.TotalUsageBytes += thinPoolUsage
+ }
+ }
+-
+- if h.zfsWatcher != nil {
+- zfsUsage, err := h.zfsWatcher.GetUsage(h.zfsFilesystem)
+- if err != nil {
+- klog.V(5).Infof("unable to get fs usage from zfs for filesystem %s: %v", h.zfsFilesystem, err)
+- } else {
+- usage.BaseUsageBytes = zfsUsage
+- usage.TotalUsageBytes += zfsUsage
+- }
+- }
+ return usage
+ }
+
+@@ -389,8 +355,6 @@
+ return fmt.Errorf("unable to determine device info for dir: %v: %v", self.rootfsStorageDir, err)
+ }
+ device = deviceInfo.Device
+- case zfsStorageDriver:
+- device = self.zfsParent
+ default:
+ return nil
+ }
+diff -ur com_github_google_cadvisor.orig/container/docker/utils/docker.go com_github_google_cadvisor/container/docker/utils/docker.go
+--- com_github_google_cadvisor.orig/container/docker/utils/docker.go 2020-04-15 17:54:23.082143453 +0200
++++ com_github_google_cadvisor/container/docker/utils/docker.go 2020-04-16 15:10:30.379817831 +0200
+@@ -69,12 +69,3 @@
+
+ return metadataDevice, nil
+ }
+-
+-func DockerZfsFilesystem(info dockertypes.Info) (string, error) {
+- filesystem := DriverStatusValue(info.DriverStatus, DriverStatusParentDataset)
+- if len(filesystem) == 0 {
+- return "", fmt.Errorf("Could not get zfs filesystem")
+- }
+-
+- return filesystem, nil
+-}
+diff -ur com_github_google_cadvisor.orig/fs/fs.go com_github_google_cadvisor/fs/fs.go
+--- com_github_google_cadvisor.orig/fs/fs.go 2020-04-15 17:54:23.086143468 +0200
++++ com_github_google_cadvisor/fs/fs.go 2020-04-16 15:39:57.294333566 +0200
+@@ -32,7 +32,6 @@
+
+ "github.com/google/cadvisor/devicemapper"
+ "github.com/google/cadvisor/utils"
+- zfs "github.com/mistifyio/go-zfs"
+
+ "k8s.io/klog"
+ "k8s.io/utils/mount"
+@@ -167,11 +166,9 @@
+
+ supportedFsType := map[string]bool{
+ // all ext systems are checked through prefix.
+- "btrfs": true,
+ "overlay": true,
+ "tmpfs": true,
+ "xfs": true,
+- "zfs": true,
+ }
+
+ for _, mount := range mounts {
+@@ -200,17 +197,6 @@
+ if mount.FsType == "tmpfs" {
+ mount.Source = mount.MountPoint
+ }
+- // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
+- // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
+- if mount.FsType == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") {
+- major, minor, err := getBtrfsMajorMinorIds(&mount)
+- if err != nil {
+- klog.Warningf("%s", err)
+- } else {
+- mount.Major = major
+- mount.Minor = minor
+- }
+- }
+
+ // overlay fix: Making mount source unique for all overlay mounts, using the mount's major and minor ids.
+ if mount.FsType == "overlay" {
+@@ -312,7 +298,7 @@
+
+ // TODO(rjnagal): Detect docker root and graphdriver directories from docker info.
+ dockerRoot := context.Docker.Root
+- for _, dir := range []string{"devicemapper", "btrfs", "aufs", "overlay", "overlay2", "zfs"} {
++ for _, dir := range []string{"devicemapper", "overlay", "overlay2"} {
+ dockerImagePaths[path.Join(dockerRoot, dir)] = struct{}{}
+ }
+ for dockerRoot != "/" && dockerRoot != "." {
+@@ -390,14 +376,6 @@
+ fs.Capacity, fs.Free, fs.Available, err = getDMStats(device, partition.blockSize)
+ klog.V(5).Infof("got devicemapper fs capacity stats: capacity: %v free: %v available: %v:", fs.Capacity, fs.Free, fs.Available)
+ fs.Type = DeviceMapper
+- case ZFS.String():
+- if _, devzfs := os.Stat("/dev/zfs"); os.IsExist(devzfs) {
+- fs.Capacity, fs.Free, fs.Available, err = getZfstats(device)
+- fs.Type = ZFS
+- break
+- }
+- // if /dev/zfs is not present default to VFS
+- fallthrough
+ default:
+ var inodes, inodesFree uint64
+ if utils.FileExists(partition.mountpoint) {
+@@ -521,30 +499,6 @@
+ }
+ }
+
+- mount, found := self.mounts[dir]
+- // try the parent dir if not found until we reach the root dir
+- // this is an issue on btrfs systems where the directory is not
+- // the subvolume
+- for !found {
+- pathdir, _ := filepath.Split(dir)
+- // break when we reach root
+- if pathdir == "/" {
+- break
+- }
+- // trim "/" from the new parent path otherwise the next possible
+- // filepath.Split in the loop will not split the string any further
+- dir = strings.TrimSuffix(pathdir, "/")
+- mount, found = self.mounts[dir]
+- }
+-
+- if found && mount.FsType == "btrfs" && mount.Major == 0 && strings.HasPrefix(mount.Source, "/dev/") {
+- major, minor, err := getBtrfsMajorMinorIds(&mount)
+- if err != nil {
+- klog.Warningf("%s", err)
+- } else {
+- return &DeviceInfo{mount.Source, uint(major), uint(minor)}, nil
+- }
+- }
+ return nil, fmt.Errorf("could not find device with major: %d, minor: %d in cached partitions map", major, minor)
+ }
+
+@@ -715,18 +669,6 @@
+ return used, total, nil
+ }
+
+-// getZfstats returns ZFS mount stats using zfsutils
+-func getZfstats(poolName string) (uint64, uint64, uint64, error) {
+- dataset, err := zfs.GetDataset(poolName)
+- if err != nil {
+- return 0, 0, 0, err
+- }
+-
+- total := dataset.Used + dataset.Avail + dataset.Usedbydataset
+-
+- return total, dataset.Avail, dataset.Avail, nil
+-}
+-
+ // Simple io.Writer implementation that counts how many bytes were written.
+ type byteCounter struct{ bytesWritten uint64 }
+
+@@ -734,33 +676,3 @@
+ b.bytesWritten += uint64(len(p))
+ return len(p), nil
+ }
+-
+-// Get major and minor Ids for a mount point using btrfs as filesystem.
+-func getBtrfsMajorMinorIds(mount *mount.MountInfo) (int, int, error) {
+- // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
+- // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
+-
+- buf := new(syscall.Stat_t)
+- err := syscall.Stat(mount.Source, buf)
+- if err != nil {
+- err = fmt.Errorf("stat failed on %s with error: %s", mount.Source, err)
+- return 0, 0, err
+- }
+-
+- klog.V(4).Infof("btrfs mount %#v", mount)
+- if buf.Mode&syscall.S_IFMT == syscall.S_IFBLK {
+- err := syscall.Stat(mount.MountPoint, buf)
+- if err != nil {
+- err = fmt.Errorf("stat failed on %s with error: %s", mount.MountPoint, err)
+- return 0, 0, err
+- }
+-
+- // The type Dev and Rdev in Stat_t are 32bit on mips.
+- klog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev)))) // nolint: unconvert
+- klog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(uint64(buf.Rdev))), int(minor(uint64(buf.Rdev)))) // nolint: unconvert
+-
+- return int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev))), nil // nolint: unconvert
+- } else {
+- return 0, 0, fmt.Errorf("%s is not a block device", mount.Source)
+- }
+-}
+diff -ur com_github_google_cadvisor.orig/fs/fs_test.go com_github_google_cadvisor/fs/fs_test.go
+--- com_github_google_cadvisor.orig/fs/fs_test.go 2020-04-15 17:54:23.086143468 +0200
++++ com_github_google_cadvisor/fs/fs_test.go 2020-04-16 15:10:42.829856068 +0200
+@@ -516,7 +516,6 @@
+ {Root: "/", MountPoint: "/b", Source: "/dev/sdb", FsType: "ext4", Major: 253, Minor: 1},
+ {Root: "/", MountPoint: "/c", Source: "/dev/sdc", FsType: "btrfs", Major: 253, Minor: 2},
+ {Root: "/", MountPoint: "/d", Source: "/dev/sdd", FsType: "xfs", Major: 253, Minor: 3},
+- {Root: "/", MountPoint: "/e", Source: "/dev/sde", FsType: "zfs", Major: 253, Minor: 4},
+ {Root: "/", MountPoint: "/f", Source: "overlay", FsType: "overlay", Major: 253, Minor: 5},
+ {Root: "/", MountPoint: "/test1", Source: "tmpfs", FsType: "tmpfs", Major: 253, Minor: 4},
+ {Root: "/", MountPoint: "/test2", Source: "tmpfs", FsType: "tmpfs", Major: 253, Minor: 4},
+@@ -526,7 +525,6 @@
+ "/dev/sdb": {fsType: "ext4", mountpoint: "/b", major: 253, minor: 1},
+ "/dev/sdc": {fsType: "btrfs", mountpoint: "/c", major: 253, minor: 2},
+ "/dev/sdd": {fsType: "xfs", mountpoint: "/d", major: 253, minor: 3},
+- "/dev/sde": {fsType: "zfs", mountpoint: "/e", major: 253, minor: 4},
+ "overlay_253-5": {fsType: "overlay", mountpoint: "/f", major: 253, minor: 5},
+ "/test1": {fsType: "tmpfs", mountpoint: "/test1", major: 253, minor: 4},
+ "/test2": {fsType: "tmpfs", mountpoint: "/test2", major: 253, minor: 4},
+diff -ur com_github_google_cadvisor.orig/fs/types.go com_github_google_cadvisor/fs/types.go
+--- com_github_google_cadvisor.orig/fs/types.go 2020-04-15 17:54:23.086143468 +0200
++++ com_github_google_cadvisor/fs/types.go 2020-04-16 15:10:48.264872763 +0200
+@@ -47,7 +47,6 @@
+ }
+
+ const (
+- ZFS FsType = "zfs"
+ DeviceMapper FsType = "devicemapper"
+ VFS FsType = "vfs"
+ )
diff --git a/third_party/go/patches/cni-plugins-build.patch b/third_party/go/patches/cni-plugins-build.patch
new file mode 100644
index 0000000..e9cef5b
--- /dev/null
+++ b/third_party/go/patches/cni-plugins-build.patch
@@ -0,0 +1,64 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 16e16b82c44cf48f0752a74de83845263fd04fda Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Wed, 5 Feb 2020 16:37:42 +0100
+Subject: [PATCH] Build the plugins we need in pure mode
+
+---
+ plugins/ipam/host-local/BUILD.bazel | 1 +
+ plugins/main/loopback/BUILD.bazel | 1 +
+ plugins/main/ptp/BUILD.bazel | 1 +
+ 3 files changed, 3 insertions(+)
+
+diff --git a/plugins/ipam/host-local/BUILD.bazel b/plugins/ipam/host-local/BUILD.bazel
+index ce1f2d6..72817a6 100644
+--- a/plugins/ipam/host-local/BUILD.bazel
++++ b/plugins/ipam/host-local/BUILD.bazel
+@@ -22,6 +22,7 @@ go_library(
+ go_binary(
+ name = "host-local",
+ embed = [":go_default_library"],
++ pure = "on",
+ visibility = ["//visibility:public"],
+ )
+
+diff --git a/plugins/main/loopback/BUILD.bazel b/plugins/main/loopback/BUILD.bazel
+index c4d6e00..952bfb1 100644
+--- a/plugins/main/loopback/BUILD.bazel
++++ b/plugins/main/loopback/BUILD.bazel
+@@ -19,6 +19,7 @@ go_library(
+ go_binary(
+ name = "loopback",
+ embed = [":go_default_library"],
++ pure = "on",
+ visibility = ["//visibility:public"],
+ )
+
+diff --git a/plugins/main/ptp/BUILD.bazel b/plugins/main/ptp/BUILD.bazel
+index 512de12..2eb5d50 100644
+--- a/plugins/main/ptp/BUILD.bazel
++++ b/plugins/main/ptp/BUILD.bazel
+@@ -23,6 +23,7 @@ go_library(
+ go_binary(
+ name = "ptp",
+ embed = [":go_default_library"],
++ pure = "on",
+ visibility = ["//visibility:public"],
+ )
+
+--
+2.20.1
diff --git a/third_party/go/patches/gvisor.patch b/third_party/go/patches/gvisor.patch
new file mode 100644
index 0000000..143ee59
--- /dev/null
+++ b/third_party/go/patches/gvisor.patch
@@ -0,0 +1,97 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 69b2c3b6e2594a3f28e4ea1141bef542456b3eb2 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@brun.one>
+Date: Wed, 5 Feb 2020 17:14:19 +0100
+Subject: [PATCH] Fix vdso include that breaks in an external
+
+---
+ vdso/cycle_clock.h | 2 +-
+ vdso/seqlock.h | 4 ++--
+ vdso/vdso.cc | 4 ++--
+ vdso/vdso_time.cc | 8 ++++----
+ 4 files changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/vdso/cycle_clock.h b/vdso/cycle_clock.h
+index 5d3fbb25..fe15812c 100644
+--- a/vdso/cycle_clock.h
++++ b/vdso/cycle_clock.h
+@@ -17,7 +17,7 @@
+
+ #include <stdint.h>
+
+-#include "vdso/barrier.h"
++#include "barrier.h"
+
+ namespace vdso {
+
+diff --git a/vdso/seqlock.h b/vdso/seqlock.h
+index 7a173174..fff99748 100644
+--- a/vdso/seqlock.h
++++ b/vdso/seqlock.h
+@@ -18,8 +18,8 @@
+
+ #include <stdint.h>
+
+-#include "vdso/barrier.h"
+-#include "vdso/compiler.h"
++#include "barrier.h"
++#include "compiler.h"
+
+ namespace vdso {
+
+diff --git a/vdso/vdso.cc b/vdso/vdso.cc
+index 8bb80a7a..210d31ff 100644
+--- a/vdso/vdso.cc
++++ b/vdso/vdso.cc
+@@ -19,8 +19,8 @@
+ #include <sys/time.h>
+ #include <time.h>
+
+-#include "vdso/syscalls.h"
+-#include "vdso/vdso_time.h"
++#include "syscalls.h"
++#include "vdso_time.h"
+
+ namespace vdso {
+ namespace {
+diff --git a/vdso/vdso_time.cc b/vdso/vdso_time.cc
+index 1bb4bb86..fb5b281f 100644
+--- a/vdso/vdso_time.cc
++++ b/vdso/vdso_time.cc
+@@ -12,15 +12,15 @@
+ // See the License for the specific language governing permissions and
+ // limitations under the License.
+
+-#include "vdso/vdso_time.h"
++#include "vdso_time.h"
+
+ #include <stdint.h>
+ #include <sys/time.h>
+ #include <time.h>
+
+-#include "vdso/cycle_clock.h"
+-#include "vdso/seqlock.h"
+-#include "vdso/syscalls.h"
++#include "cycle_clock.h"
++#include "seqlock.h"
++#include "syscalls.h"
+
+ // struct params defines the layout of the parameter page maintained by the
+ // kernel (i.e., sentry).
+--
+2.20.1
+
diff --git a/third_party/go/patches/k8s-client-go-build.patch b/third_party/go/patches/k8s-client-go-build.patch
new file mode 100644
index 0000000..d5fad50
--- /dev/null
+++ b/third_party/go/patches/k8s-client-go-build.patch
@@ -0,0 +1,30 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This patch updates BUILD files to reflect changes in Go sources. This only needs to be applied because Gazelle applies patches after BUILDfile generation.
+
+diff -ur io_k8s_client_go.orig/plugin/pkg/client/auth/BUILD.bazel io_k8s_client_go/plugin/pkg/client/auth/BUILD.bazel
+--- io_k8s_client_go.orig/plugin/pkg/client/auth/BUILD.bazel 17:54:23.086143468 +0200
++++ io_k8s_client_go/plugin/pkg/client/auth/BUILD.bazel 17:56:17.098546899 +0200
+@@ -6,9 +6,7 @@
+ importpath = "k8s.io/client-go/plugin/pkg/client/auth",
+ visibility = ["//visibility:public"],
+ deps = [
+- "//plugin/pkg/client/auth/azure:go_default_library",
+ "//plugin/pkg/client/auth/gcp:go_default_library",
+ "//plugin/pkg/client/auth/oidc:go_default_library",
+- "//plugin/pkg/client/auth/openstack:go_default_library",
+ ],
+ )
diff --git a/third_party/go/patches/k8s-client-go.patch b/third_party/go/patches/k8s-client-go.patch
new file mode 100644
index 0000000..ebfae0e
--- /dev/null
+++ b/third_party/go/patches/k8s-client-go.patch
@@ -0,0 +1,29 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This patch rips out azure/openstack auth support for k8s.io/client-go. This should be made configurable upstream.
+
+diff -ur io_k8s_client_go.orig/plugin/pkg/client/auth/plugins.go io_k8s_client_go/plugin/pkg/client/auth/plugins.go
+--- io_k8s_client_go.orig/plugin/pkg/client/auth/plugins.go 2020-04-16 17:46:53.965434780 +0200
++++ io_k8s_client_go/plugin/pkg/client/auth/plugins.go 2020-04-16 17:47:21.720538171 +0200
+@@ -18,8 +18,6 @@
+
+ import (
+ // Initialize all known client auth plugins.
+- _ "k8s.io/client-go/plugin/pkg/client/auth/azure"
+ _ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+ _ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
+- _ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
+ )
diff --git a/third_party/go/patches/k8s-kubernetes-build.patch b/third_party/go/patches/k8s-kubernetes-build.patch
new file mode 100644
index 0000000..797769a
--- /dev/null
+++ b/third_party/go/patches/k8s-kubernetes-build.patch
@@ -0,0 +1,108 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This patch updates BUILD files to reflect changes in Go sources. This only needs to be applied because Gazelle applies patches after BUILDfile generation.
+
+diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go io_k8s_kubernetes/cmd/kubelet/app/plugins.go
+--- io_k8s_kubernetes.orig/cmd/kubelet/app/BUILD 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/app/BUILD 15:12:22.682316924 +0200
+@@ -45,8 +45,6 @@
+ "//pkg/capabilities:go_default_library",
+ "//pkg/cloudprovider/providers:go_default_library",
+ "//pkg/credentialprovider:go_default_library",
+- "//pkg/credentialprovider/aws:go_default_library",
+- "//pkg/credentialprovider/azure:go_default_library",
+ "//pkg/credentialprovider/gcp:go_default_library",
+ "//pkg/features:go_default_library",
+ "//pkg/kubelet:go_default_library",
+@@ -78,25 +76,11 @@
+ "//pkg/util/oom:go_default_library",
+ "//pkg/util/rlimit:go_default_library",
+ "//pkg/volume:go_default_library",
+- "//pkg/volume/cephfs:go_default_library",
+ "//pkg/volume/configmap:go_default_library",
+ "//pkg/volume/csi:go_default_library",
+- "//pkg/volume/downwardapi:go_default_library",
+ "//pkg/volume/emptydir:go_default_library",
+- "//pkg/volume/fc:go_default_library",
+ "//pkg/volume/flexvolume:go_default_library",
+- "//pkg/volume/flocker:go_default_library",
+- "//pkg/volume/git_repo:go_default_library",
+- "//pkg/volume/glusterfs:go_default_library",
+ "//pkg/volume/hostpath:go_default_library",
+- "//pkg/volume/iscsi:go_default_library",
+ "//pkg/volume/local:go_default_library",
+- "//pkg/volume/nfs:go_default_library",
+- "//pkg/volume/portworx:go_default_library",
+- "//pkg/volume/projected:go_default_library",
+- "//pkg/volume/quobyte:go_default_library",
+- "//pkg/volume/rbd:go_default_library",
+- "//pkg/volume/scaleio:go_default_library",
+ "//pkg/volume/secret:go_default_library",
+- "//pkg/volume/storageos:go_default_library",
+ "//pkg/volume/util/hostutil:go_default_library",
+--- io_k8s_kubernetes.orig/cmd/kubelet/app/options/BUILD 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/app/options/BUILD 15:12:22.682316924 +0200
+@@ -20,8 +20,6 @@
+ importpath = "k8s.io/kubernetes/cmd/kubelet/app/options",
+ deps = [
+ "//pkg/apis/core:go_default_library",
+- "//pkg/credentialprovider/azure:go_default_library",
+- "//pkg/credentialprovider/gcp:go_default_library",
+ "//pkg/features:go_default_library",
+ "//pkg/kubelet/apis:go_default_library",
+ "//pkg/kubelet/apis/config:go_default_library",
+--- io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/BUILD 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/pkg/kubelet/cadvisor/BUILD 15:12:22.682316924 +0200
+@@ -37,8 +37,6 @@
+ "@com_github_google_cadvisor//container/systemd/install:go_default_library",
+ "@com_github_google_cadvisor//fs:go_default_library",
+ "@com_github_google_cadvisor//manager:go_default_library",
+- "@com_github_google_cadvisor//utils/cloudinfo/aws:go_default_library",
+- "@com_github_google_cadvisor//utils/cloudinfo/azure:go_default_library",
+ "@com_github_google_cadvisor//utils/cloudinfo/gce:go_default_library",
+ "@com_github_google_cadvisor//utils/sysfs:go_default_library",
+ "@io_k8s_klog//:go_default_library",
+@@ -52,8 +50,6 @@
+ "@com_github_google_cadvisor//container/systemd/install:go_default_library",
+ "@com_github_google_cadvisor//fs:go_default_library",
+ "@com_github_google_cadvisor//manager:go_default_library",
+- "@com_github_google_cadvisor//utils/cloudinfo/aws:go_default_library",
+- "@com_github_google_cadvisor//utils/cloudinfo/azure:go_default_library",
+ "@com_github_google_cadvisor//utils/cloudinfo/gce:go_default_library",
+ "@com_github_google_cadvisor//utils/sysfs:go_default_library",
+ "@io_k8s_klog//:go_default_library",
+--- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/BUILD 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kube-controller-manager/app/BUILD 15:12:22.682316924 +0200
+@@ -90,19 +90,9 @@
+ "//pkg/volume:go_default_library",
+ "//pkg/volume/csi:go_default_library",
+ "//pkg/volume/csimigration:go_default_library",
+- "//pkg/volume/fc:go_default_library",
+ "//pkg/volume/flexvolume:go_default_library",
+- "//pkg/volume/flocker:go_default_library",
+- "//pkg/volume/glusterfs:go_default_library",
+ "//pkg/volume/hostpath:go_default_library",
+- "//pkg/volume/iscsi:go_default_library",
+ "//pkg/volume/local:go_default_library",
+- "//pkg/volume/nfs:go_default_library",
+- "//pkg/volume/portworx:go_default_library",
+- "//pkg/volume/quobyte:go_default_library",
+- "//pkg/volume/rbd:go_default_library",
+- "//pkg/volume/scaleio:go_default_library",
+- "//pkg/volume/storageos:go_default_library",
+ "//pkg/volume/util:go_default_library",
+ "@com_github_spf13_cobra//:go_default_library",
+ "@io_k8s_api//core/v1:go_default_library",
diff --git a/third_party/go/patches/k8s-kubernetes.patch b/third_party/go/patches/k8s-kubernetes.patch
new file mode 100644
index 0000000..6d4663e
--- /dev/null
+++ b/third_party/go/patches/k8s-kubernetes.patch
@@ -0,0 +1,386 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+This fixes OpenAPI codegen for when included from the Smalltown workspace. It basically undoes vendorification.
+
+diff -ur io_k8s_kubernetes.orig/build/code_generation.bzl io_k8s_kubernetes/build/code_generation.bzl
+--- io_k8s_kubernetes.orig/build/code_generation.bzl 2020-04-15 13:43:57.785669620 +0200
++++ io_k8s_kubernetes/build/code_generation.bzl 2020-04-16 18:19:44.297531873 +0200
+@@ -27,6 +27,12 @@
+ ...
+ )
+ """
++ if pkg.startswith('staging/src/k8s.io/'):
++ parts = pkg.split('/', 4)
++ project = parts[3]
++ project = project.replace('-', '_')
++ path = parts[4]
++ return "@io_k8s_%s//%s:go_default_library" % (project, path)
+ return "//%s:go_default_library" % pkg
+
+ def go_pkg(pkg):
+@@ -42,6 +48,8 @@
+ ...
+ )
+ """
++ if pkg.startswith('staging/src/'):
++ return pkg[len('staging/src/'):]
+ for prefix in ["staging/src", "vendor"]:
+ if pkg.startswith(prefix):
+ return paths.relativize(pkg, prefix)
+@@ -49,8 +57,8 @@
+
+ def openapi_deps():
+ deps = [
+- "//vendor/github.com/go-openapi/spec:go_default_library",
+- "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
++ "@com_github_go_openapi_spec//:go_default_library",
++ "@io_k8s_kube_openapi//pkg/common:go_default_library",
+ ]
+ deps.extend([bazel_go_library(pkg) for pkg in tags_values_pkgs["openapi-gen"]["true"]])
+ return deps
+@@ -76,7 +84,7 @@
+ # the generator must run from the repo root inside the generated GOPATH.
+ # All of bazel's $(location)s are relative to the original working directory, however.
+ cmd = " ".join([
+- "$(location //vendor/k8s.io/kube-openapi/cmd/openapi-gen)",
++ "$(location @io_k8s_kube_openapi//cmd/openapi-gen)",
+ "--v 1",
+ "--logtostderr",
+ "--go-header-file $(location //" + openapi_vendor_prefix + "hack/boilerplate:boilerplate.generatego.txt)",
+@@ -88,6 +96,6 @@
+ "&& rm tmp_api_violations.report",
+ ]),
+ go_deps = openapi_deps(),
+- tools = ["//vendor/k8s.io/kube-openapi/cmd/openapi-gen"],
++ tools = ["@io_k8s_kube_openapi//cmd/openapi-gen"],
+ message = "GenOpenAPI",
+ )
+
+The rest rips out a bunch of volume providers. We're only interested in CSI and hostpath/local.
+
+diff -ur io_k8s_kubernetes.orig/cmd/kube-apiserver/app/options/globalflags.go io_k8s_kubernetes/cmd/kube-apiserver/app/options/globalflags.go
+--- io_k8s_kubernetes.orig/cmd/kube-apiserver/app/options/globalflags.go 2020-04-15 13:43:57.811669689 +0200
++++ io_k8s_kubernetes/cmd/kube-apiserver/app/options/globalflags.go 2020-04-17 13:29:41.578264893 +0200
+@@ -32,9 +32,6 @@
+ func AddCustomGlobalFlags(fs *pflag.FlagSet) {
+ // Lookup flags in global flag set and re-register the values with our flagset.
+
+- // Adds flags from k8s.io/kubernetes/pkg/cloudprovider/providers.
+- registerLegacyGlobalFlags(fs)
+-
+ // Adds flags from k8s.io/apiserver/pkg/admission.
+ globalflag.Register(fs, "default-not-ready-toleration-seconds")
+ globalflag.Register(fs, "default-unreachable-toleration-seconds")
+diff -ur io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/controllermanager.go io_k8s_kubernetes/cmd/kube-controller-manager/app/controllermanager.go
+--- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/controllermanager.go 2020-04-15 13:43:57.812669692 +0200
++++ io_k8s_kubernetes/cmd/kube-controller-manager/app/controllermanager.go 2020-04-17 13:30:39.975397151 +0200
+@@ -126,7 +126,6 @@
+ namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
+ verflag.AddFlags(namedFlagSets.FlagSet("global"))
+ globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
+- registerLegacyGlobalFlags(namedFlagSets)
+ for _, f := range namedFlagSets.FlagSets {
+ fs.AddFlagSet(f)
+ }
+diff -ur io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/plugins.go io_k8s_kubernetes/cmd/kube-controller-manager/app/plugins.go
+--- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/plugins.go 2020-04-15 13:43:57.813669694 +0200
++++ io_k8s_kubernetes/cmd/kube-controller-manager/app/plugins.go 2020-04-17 13:34:19.393894095 +0200
+@@ -32,19 +32,9 @@
+ // Volume plugins
+ "k8s.io/kubernetes/pkg/volume"
+ "k8s.io/kubernetes/pkg/volume/csi"
+- "k8s.io/kubernetes/pkg/volume/fc"
+ "k8s.io/kubernetes/pkg/volume/flexvolume"
+- "k8s.io/kubernetes/pkg/volume/flocker"
+- "k8s.io/kubernetes/pkg/volume/glusterfs"
+ "k8s.io/kubernetes/pkg/volume/hostpath"
+- "k8s.io/kubernetes/pkg/volume/iscsi"
+ "k8s.io/kubernetes/pkg/volume/local"
+- "k8s.io/kubernetes/pkg/volume/nfs"
+- "k8s.io/kubernetes/pkg/volume/portworx"
+- "k8s.io/kubernetes/pkg/volume/quobyte"
+- "k8s.io/kubernetes/pkg/volume/rbd"
+- "k8s.io/kubernetes/pkg/volume/scaleio"
+- "k8s.io/kubernetes/pkg/volume/storageos"
+ volumeutil "k8s.io/kubernetes/pkg/volume/util"
+
+ utilfeature "k8s.io/apiserver/pkg/util/feature"
+@@ -58,18 +48,7 @@
+ // The list of plugins is manually compiled. This code and the plugin
+ // initialization code for kubelet really, really need a through refactor.
+ func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) {
+- var err error
+ allPlugins := []volume.VolumePlugin{}
+- allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
+- if err != nil {
+- return allPlugins, err
+- }
+- allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
+ return allPlugins, nil
+ }
+@@ -83,18 +62,7 @@
+
+ // ProbeExpandableVolumePlugins returns volume plugins which are expandable
+ func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
+- var err error
+ allPlugins := []volume.VolumePlugin{}
+- allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
+- if err != nil {
+- return allPlugins, err
+- }
+- allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
+ return allPlugins, nil
+ }
+
+@@ -124,30 +92,7 @@
+ }
+ allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(hostPathConfig)...)
+
+- nfsConfig := volume.VolumeConfig{
+- RecyclerMinimumTimeout: int(config.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS),
+- RecyclerTimeoutIncrement: int(config.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS),
+- RecyclerPodTemplate: volume.NewPersistentVolumeRecyclerPodTemplate(),
+- }
+- if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, &nfsConfig); err != nil {
+- klog.Fatalf("Could not create NFS recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err)
+- }
+- allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...)
+- allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
+- // add rbd provisioner
+- allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
+- var err error
+- allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
+- if err != nil {
+- return allPlugins, err
+- }
+-
+- allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
+
+ if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
+ allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
+diff -ur io_k8s_kubernetes.orig/cmd/kubectl/BUILD io_k8s_kubernetes/cmd/kubectl/BUILD
+--- io_k8s_kubernetes.orig/cmd/kubectl/BUILD 2020-04-20 14:58:52.573455879 +0200
++++ io_k8s_kubernetes/cmd/kubectl/BUILD 2020-04-20 14:56:41.199032687 +0200
+@@ -3,7 +3,7 @@
+ "go_binary",
+ "go_library",
+ )
+-load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
++load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
+
+ go_binary(
+ name = "kubectl",
+
+
+We also take the opportunity to remove azure/gcp auth.
+
+diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/options/globalflags.go io_k8s_kubernetes/cmd/kubelet/app/options/globalflags.go
+--- io_k8s_kubernetes.orig/cmd/kubelet/app/options/globalflags.go 2020-04-15 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/app/options/globalflags.go 2020-04-16 15:58:30.964945445 +0200
+@@ -28,10 +28,6 @@
+ "k8s.io/component-base/logs"
+ "k8s.io/component-base/version/verflag"
+ "k8s.io/klog"
+-
+- // ensure libs have a chance to globally register their flags
+- _ "k8s.io/kubernetes/pkg/credentialprovider/azure"
+- _ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
+ )
+
+ // AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
+@@ -80,14 +76,8 @@
+
+ // addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
+ func addCredentialProviderFlags(fs *pflag.FlagSet) {
+- // lookup flags in global flag set and re-register the values with our flagset
+- global := pflag.CommandLine
+ local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
+
+- // TODO(#58034): This is not a static file, so it's not quite as straightforward as --google-json-key.
+- // We need to figure out how ACR users can dynamically provide pull credentials before we can deprecate this.
+- pflagRegister(global, local, "azure-container-registry-config")
+-
+ fs.AddFlagSet(local)
+ }
+
+diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go io_k8s_kubernetes/cmd/kubelet/app/plugins.go
+--- io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go 2020-04-15 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/app/plugins.go 2020-04-16 16:10:13.366081373 +0200
+@@ -19,8 +19,6 @@
+ // This file exists to force the desired plugin implementations to be linked.
+ import (
+ // Credential providers
+- _ "k8s.io/kubernetes/pkg/credentialprovider/aws"
+- _ "k8s.io/kubernetes/pkg/credentialprovider/azure"
+ _ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
+
+ "k8s.io/component-base/featuregate"
+@@ -28,27 +26,13 @@
+
+ // Volume plugins
+ "k8s.io/kubernetes/pkg/volume"
+- "k8s.io/kubernetes/pkg/volume/cephfs"
+ "k8s.io/kubernetes/pkg/volume/configmap"
+ "k8s.io/kubernetes/pkg/volume/csi"
+- "k8s.io/kubernetes/pkg/volume/downwardapi"
+ "k8s.io/kubernetes/pkg/volume/emptydir"
+- "k8s.io/kubernetes/pkg/volume/fc"
+ "k8s.io/kubernetes/pkg/volume/flexvolume"
+- "k8s.io/kubernetes/pkg/volume/flocker"
+- "k8s.io/kubernetes/pkg/volume/git_repo"
+- "k8s.io/kubernetes/pkg/volume/glusterfs"
+ "k8s.io/kubernetes/pkg/volume/hostpath"
+- "k8s.io/kubernetes/pkg/volume/iscsi"
+ "k8s.io/kubernetes/pkg/volume/local"
+- "k8s.io/kubernetes/pkg/volume/nfs"
+- "k8s.io/kubernetes/pkg/volume/portworx"
+- "k8s.io/kubernetes/pkg/volume/projected"
+- "k8s.io/kubernetes/pkg/volume/quobyte"
+- "k8s.io/kubernetes/pkg/volume/rbd"
+- "k8s.io/kubernetes/pkg/volume/scaleio"
+ "k8s.io/kubernetes/pkg/volume/secret"
+- "k8s.io/kubernetes/pkg/volume/storageos"
+
+ // Cloud providers
+ _ "k8s.io/kubernetes/pkg/cloudprovider/providers"
+@@ -64,30 +48,11 @@
+ //
+ // Kubelet does not currently need to configure volume plugins.
+ // If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig
+- var err error
+- allPlugins, err = appendLegacyProviderVolumes(allPlugins, featureGate)
+- if err != nil {
+- return allPlugins, err
+- }
+ allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(volume.VolumeConfig{})...)
+- allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
+ allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
+- allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
+ allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
+ return allPlugins, nil
+ }
+diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/plugins_providers.go io_k8s_kubernetes/cmd/kubelet/app/plugins_providers.go
+--- io_k8s_kubernetes.orig/cmd/kubelet/app/plugins_providers.go 2020-04-15 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/app/plugins_providers.go 2020-04-15 16:29:49.402465237 +0200
+@@ -24,13 +24,7 @@
+ "k8s.io/klog"
+ "k8s.io/kubernetes/pkg/features"
+ "k8s.io/kubernetes/pkg/volume"
+- "k8s.io/kubernetes/pkg/volume/awsebs"
+- "k8s.io/kubernetes/pkg/volume/azure_dd"
+- "k8s.io/kubernetes/pkg/volume/azure_file"
+- "k8s.io/kubernetes/pkg/volume/cinder"
+ "k8s.io/kubernetes/pkg/volume/csimigration"
+- "k8s.io/kubernetes/pkg/volume/gcepd"
+- "k8s.io/kubernetes/pkg/volume/vsphere_volume"
+ )
+
+ type probeFn func() []volume.VolumePlugin
+@@ -60,11 +54,6 @@
+
+ func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
+ pluginMigrationStatus := make(map[string]pluginInfo)
+- pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
+- pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
+- pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
+- pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins}
+- pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins}
+
+ var err error
+ for pluginName, pluginInfo := range pluginMigrationStatus {
+@@ -74,6 +63,5 @@
+ }
+ }
+
+- allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
+ return allPlugins, nil
+ }
+diff -ur io_k8s_kubernetes.orig/cmd/kubelet/BUILD io_k8s_kubernetes/cmd/kubelet/BUILD
+--- io_k8s_kubernetes.orig/cmd/kubelet/BUILD 2020-04-15 13:43:57.827669732 +0200
++++ io_k8s_kubernetes/cmd/kubelet/BUILD 2020-04-20 14:56:20.446965836 +0200
+@@ -5,7 +5,7 @@
+ go_binary = "go_binary_conditional_pure",
+ )
+ load("@io_bazel_rules_go//go:def.bzl", "go_library")
+-load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
++load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
+
+ go_binary(
+ name = "kubelet",
+diff -ur io_k8s_kubernetes.orig/pkg/controller/nodeipam/ipam/cidr_allocator.go io_k8s_kubernetes/pkg/controller/nodeipam/ipam/cidr_allocator.go
+--- io_k8s_kubernetes.orig/pkg/controller/nodeipam/ipam/cidr_allocator.go 2020-04-15 13:43:57.860669820 +0200
++++ io_k8s_kubernetes/pkg/controller/nodeipam/ipam/cidr_allocator.go 2020-04-17 13:18:36.157842990 +0200
+@@ -111,8 +111,6 @@
+ switch allocatorType {
+ case RangeAllocatorType:
+ return NewCIDRRangeAllocator(kubeClient, nodeInformer, allocatorParams, nodeList)
+- case CloudAllocatorType:
+- return NewCloudCIDRAllocator(kubeClient, cloud, nodeInformer)
+ default:
+ return nil, fmt.Errorf("invalid CIDR allocator type: %v", allocatorType)
+ }
+diff -ur io_k8s_kubernetes.orig/pkg/controller/nodeipam/nolegacyprovider.go io_k8s_kubernetes/pkg/controller/nodeipam/nolegacyprovider.go
+--- io_k8s_kubernetes.orig/pkg/controller/nodeipam/nolegacyprovider.go 2020-04-15 13:43:57.860669820 +0200
++++ io_k8s_kubernetes/pkg/controller/nodeipam/nolegacyprovider.go 2020-04-17 13:27:12.440927122 +0200
+@@ -1,5 +1,3 @@
+-// +build providerless
+-
+ /*
+ Copyright 2019 The Kubernetes Authors.
+
+diff -ur io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/cadvisor_linux.go io_k8s_kubernetes/pkg/kubelet/cadvisor/cadvisor_linux.go
+--- io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/cadvisor_linux.go 2020-04-15 13:43:57.875669859 +0200
++++ io_k8s_kubernetes/pkg/kubelet/cadvisor/cadvisor_linux.go 2020-04-15 18:24:28.683551309 +0200
+@@ -34,8 +34,6 @@
+
+ // Register cloud info providers.
+ // TODO(#68522): Remove this in 1.20+ once the cAdvisor endpoints are removed.
+- _ "github.com/google/cadvisor/utils/cloudinfo/aws"
+- _ "github.com/google/cadvisor/utils/cloudinfo/azure"
+ _ "github.com/google/cadvisor/utils/cloudinfo/gce"
+
+ "github.com/google/cadvisor/cache/memory"