diff --git a/third_party/go/patches/cel-fix-antlr.patch b/third_party/go/patches/cel-fix-antlr.patch
new file mode 100644
index 0000000..5c5636a
--- /dev/null
+++ b/third_party/go/patches/cel-fix-antlr.patch
@@ -0,0 +1,83 @@
+From a0f8916f104566a2538a70153c016de7c5d9f304 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 16 Mar 2022 18:54:11 +0100
+Subject: [PATCH] Use conventional import path for antlr
+
+---
+ WORKSPACE              | 9 +++++----
+ checker/BUILD.bazel    | 2 +-
+ parser/BUILD.bazel     | 4 ++--
+ parser/gen/BUILD.bazel | 2 +-
+ 4 files changed, 9 insertions(+), 8 deletions(-)
+
+diff --git a/WORKSPACE b/WORKSPACE
+index 4238e7a..b62cf15 100644
+--- a/WORKSPACE
++++ b/WORKSPACE
+@@ -99,11 +99,12 @@ go_repository(
+     version = "v0.3.2",
+ )
+ 
+-# Antlr deps to pickup golang concurrency fixes 4/30/2020
++# Antlr deps
+ go_repository(
+-    name = "com_github_antlr",
+-    commit = "621b933c7a7f01c67ae9de15103151fa0f9d6d90",
+-    importpath = "github.com/antlr/antlr4",
++    name = "com_github_antlr_antlr4_runtime_go_antlr",
++    importpath = "github.com/antlr/antlr4/runtime/Go/antlr",
++    sum = "h1:zvkJv+9Pxm1nnEMcKnShREt4qtduHKz4iw4AB4ul0Ao=",
++    version = "v0.0.0-20220209173558-ad29539cd2e9",
+ )
+ 
+ # CEL Spec deps
+diff --git a/checker/BUILD.bazel b/checker/BUILD.bazel
+index 2ef326b..bec40b6 100644
+--- a/checker/BUILD.bazel
++++ b/checker/BUILD.bazel
+@@ -54,7 +54,7 @@ go_test(
+         "//test:go_default_library",
+         "//test/proto2pb:go_default_library",
+         "//test/proto3pb:go_default_library",
+-        "@com_github_antlr//runtime/Go/antlr:go_default_library",
++        "@com_github_antlr_antlr4_runtime_go_antlr//:go_default_library",
+         "@org_golang_google_protobuf//proto:go_default_library",
+     ],
+ )
+diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel
+index a1917b4..b76e6e4 100644
+--- a/parser/BUILD.bazel
++++ b/parser/BUILD.bazel
+@@ -23,7 +23,7 @@ go_library(
+         "//common/operators:go_default_library",
+         "//common/runes:go_default_library",
+         "//parser/gen:go_default_library",
+-        "@com_github_antlr//runtime/Go/antlr:go_default_library",
++        "@com_github_antlr_antlr4_runtime_go_antlr//:go_default_library",
+         "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+@@ -45,7 +45,7 @@ go_test(
+         "//common/debug:go_default_library",
+         "//parser/gen:go_default_library",
+         "//test:go_default_library",
+-        "@com_github_antlr//runtime/Go/antlr:go_default_library",
++        "@com_github_antlr_antlr4_runtime_go_antlr//:go_default_library",
+         "@org_golang_google_protobuf//proto:go_default_library",
+     ],
+ )
+diff --git a/parser/gen/BUILD.bazel b/parser/gen/BUILD.bazel
+index 78e0432..2271131 100644
+--- a/parser/gen/BUILD.bazel
++++ b/parser/gen/BUILD.bazel
+@@ -21,6 +21,6 @@ go_library(
+     ],
+     importpath = "github.com/google/cel-go/parser/gen",
+     deps = [
+-        "@com_github_antlr//runtime/Go/antlr:go_default_library",
++        "@com_github_antlr_antlr4_runtime_go_antlr//:go_default_library",
+     ],
+ )
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/cel-fix-googleapis.patch b/third_party/go/patches/cel-fix-googleapis.patch
new file mode 100644
index 0000000..518cdd0
--- /dev/null
+++ b/third_party/go/patches/cel-fix-googleapis.patch
@@ -0,0 +1,256 @@
+From 86ef097e120745353232555f1d87382cac5ce2f0 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Mon, 21 Mar 2022 16:11:14 +0100
+Subject: [PATCH] Fix googleapis import paths
+
+---
+ cel/BUILD.bazel               |  4 ++--
+ checker/BUILD.bazel           |  2 +-
+ checker/decls/BUILD.bazel     |  2 +-
+ common/BUILD.bazel            |  2 +-
+ common/containers/BUILD.bazel |  4 ++--
+ common/debug/BUILD.bazel      |  2 +-
+ common/types/BUILD.bazel      |  4 ++--
+ common/types/pb/BUILD.bazel   |  2 +-
+ common/types/ref/BUILD.bazel  |  2 +-
+ ext/BUILD.bazel               |  2 +-
+ interpreter/BUILD.bazel       |  4 ++--
+ parser/BUILD.bazel            |  2 +-
+ server/BUILD.bazel            | 12 ++++++------
+ test/BUILD.bazel              |  2 +-
+ 14 files changed, 23 insertions(+), 23 deletions(-)
+
+diff --git a/cel/BUILD.bazel b/cel/BUILD.bazel
+index aed8274..6a5b8f2 100644
+--- a/cel/BUILD.bazel
++++ b/cel/BUILD.bazel
+@@ -27,7 +27,7 @@ go_library(
+         "//interpreter:go_default_library",
+         "//interpreter/functions:go_default_library",
+         "//parser:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//reflect/protodesc:go_default_library",
+         "@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
+@@ -59,6 +59,6 @@ go_test(
+         "//test/proto2pb:go_default_library",
+         "//test/proto3pb:go_default_library",
+         "@io_bazel_rules_go//proto/wkt:descriptor_go_proto",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+     ],
+ )
+diff --git a/checker/BUILD.bazel b/checker/BUILD.bazel
+index 0f6e469..0873707 100644
+--- a/checker/BUILD.bazel
++++ b/checker/BUILD.bazel
+@@ -28,7 +28,7 @@ go_library(
+         "//common/types/pb:go_default_library",
+         "//common/types/ref:go_default_library",
+         "//parser:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+diff --git a/checker/decls/BUILD.bazel b/checker/decls/BUILD.bazel
+index 5a24f1d..9c6bfbc 100644
+--- a/checker/decls/BUILD.bazel
++++ b/checker/decls/BUILD.bazel
+@@ -13,7 +13,7 @@ go_library(
+     ],
+     importpath = "github.com/google/cel-go/checker/decls",
+     deps = [
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//types/known/emptypb:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+     ],
+diff --git a/common/BUILD.bazel b/common/BUILD.bazel
+index 9e4ad65..d306934 100644
+--- a/common/BUILD.bazel
++++ b/common/BUILD.bazel
+@@ -16,7 +16,7 @@ go_library(
+     importpath = "github.com/google/cel-go/common",
+     deps = [
+         "//common/runes:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_x_text//width:go_default_library",
+     ],
+ )
+diff --git a/common/containers/BUILD.bazel b/common/containers/BUILD.bazel
+index 18142d9..3c45794 100644
+--- a/common/containers/BUILD.bazel
++++ b/common/containers/BUILD.bazel
+@@ -12,7 +12,7 @@ go_library(
+     ],
+     importpath = "github.com/google/cel-go/common/containers",
+     deps = [
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+     ],
+ )
+ 
+@@ -26,6 +26,6 @@ go_test(
+         ":go_default_library",
+     ],
+     deps = [
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+     ],
+ )
+diff --git a/common/debug/BUILD.bazel b/common/debug/BUILD.bazel
+index cf5c5d2..6db7f55 100644
+--- a/common/debug/BUILD.bazel
++++ b/common/debug/BUILD.bazel
+@@ -13,6 +13,6 @@ go_library(
+     importpath = "github.com/google/cel-go/common/debug",
+     deps = [
+         "//common:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+     ],
+ )
+diff --git a/common/types/BUILD.bazel b/common/types/BUILD.bazel
+index 32789f5..e3970fd 100644
+--- a/common/types/BUILD.bazel
++++ b/common/types/BUILD.bazel
+@@ -37,7 +37,7 @@ go_library(
+         "//common/types/ref:go_default_library",
+         "//common/types/traits:go_default_library",
+         "@com_github_stoewer_go_strcase//:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//encoding/protojson:go_default_library",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
+@@ -75,7 +75,7 @@ go_test(
+         "//common/types/ref:go_default_library",
+         "//test:go_default_library",
+         "//test/proto3pb:test_all_types_go_proto",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//encoding/protojson:go_default_library",
+         "@org_golang_google_protobuf//types/known/anypb:go_default_library",
+         "@org_golang_google_protobuf//types/known/durationpb:go_default_library",
+diff --git a/common/types/pb/BUILD.bazel b/common/types/pb/BUILD.bazel
+index b0c79a7..37cd0a7 100644
+--- a/common/types/pb/BUILD.bazel
++++ b/common/types/pb/BUILD.bazel
+@@ -16,7 +16,7 @@ go_library(
+     ],
+     importpath = "github.com/google/cel-go/common/types/pb",
+     deps = [
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
+         "@org_golang_google_protobuf//reflect/protoregistry:go_default_library",
+diff --git a/common/types/ref/BUILD.bazel b/common/types/ref/BUILD.bazel
+index 1d0f468..9243b47 100644
+--- a/common/types/ref/BUILD.bazel
++++ b/common/types/ref/BUILD.bazel
+@@ -13,7 +13,7 @@ go_library(
+     ],
+     importpath = "github.com/google/cel-go/common/types/ref",
+     deps = [
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//reflect/protoreflect:go_default_library",
+     ],
+diff --git a/ext/BUILD.bazel b/ext/BUILD.bazel
+index 2b98ad3..68a9d72 100644
+--- a/ext/BUILD.bazel
++++ b/ext/BUILD.bazel
+@@ -19,7 +19,7 @@ go_library(
+         "//common/types:go_default_library",
+         "//common/types/ref:go_default_library",
+         "//interpreter/functions:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+     ],
+ )
+ 
+diff --git a/interpreter/BUILD.bazel b/interpreter/BUILD.bazel
+index 2f4079f..6b30db8 100644
+--- a/interpreter/BUILD.bazel
++++ b/interpreter/BUILD.bazel
+@@ -30,7 +30,7 @@ go_library(
+         "//common/types/ref:go_default_library",
+         "//common/types/traits:go_default_library",
+         "//interpreter/functions:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/durationpb:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+@@ -63,7 +63,7 @@ go_test(
+         "//test:go_default_library",
+         "//test/proto2pb:go_default_library",
+         "//test/proto3pb:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/anypb:go_default_library",
+     ],
+diff --git a/parser/BUILD.bazel b/parser/BUILD.bazel
+index b76e6e4..c098fe6 100644
+--- a/parser/BUILD.bazel
++++ b/parser/BUILD.bazel
+@@ -24,7 +24,7 @@ go_library(
+         "//common/runes:go_default_library",
+         "//parser/gen:go_default_library",
+         "@com_github_antlr_antlr4_runtime_go_antlr//:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+     ],
+diff --git a/server/BUILD.bazel b/server/BUILD.bazel
+index 7174aa9..a4d9ff6 100644
+--- a/server/BUILD.bazel
++++ b/server/BUILD.bazel
+@@ -19,9 +19,9 @@ go_library(
+         "//common/types/traits:go_default_library",
+         "@com_google_cel_spec//proto/test/v1/proto2:test_all_types_go_proto",
+         "@com_google_cel_spec//proto/test/v1/proto3:test_all_types_go_proto",
+-        "@org_golang_google_genproto//googleapis/api/expr/conformance/v1alpha1:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
+-        "@org_golang_google_genproto//googleapis/rpc/status:go_default_library",
++        "@go_googleapis//google/api/expr/conformance/v1alpha1:conformance_go_proto",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
++        "@go_googleapis//google/rpc/status:status_go_proto",
+         "@org_golang_google_grpc//codes:go_default_library",
+         "@org_golang_google_grpc//status:go_default_library",
+         "@org_golang_google_protobuf//proto:go_default_library",
+@@ -44,8 +44,8 @@ go_test(
+         "//common/operators:go_default_library",
+         "//test:go_default_library",
+         "@com_google_cel_spec//tools/celrpc:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/conformance/v1alpha1:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
+-        "@org_golang_google_genproto//googleapis/rpc/status:go_default_library",
++        "@go_googleapis//google/api/expr/conformance/v1alpha1:conformance_go_proto",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
++        "@go_googleapis//google/rpc/status:status_go_proto",
+     ],
+ )
+diff --git a/test/BUILD.bazel b/test/BUILD.bazel
+index 0d39c70..3b8c460 100644
+--- a/test/BUILD.bazel
++++ b/test/BUILD.bazel
+@@ -20,7 +20,7 @@ go_library(
+     importpath = "github.com/google/cel-go/test",
+     deps = [
+         "//common/operators:go_default_library",
+-        "@org_golang_google_genproto//googleapis/api/expr/v1alpha1:go_default_library",
++        "@go_googleapis//google/api/expr/v1alpha1:expr_go_proto",
+         "@org_golang_google_protobuf//proto:go_default_library",
+         "@org_golang_google_protobuf//types/known/structpb:go_default_library",
+     ],
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/containerd-netns-statedir.patch b/third_party/go/patches/containerd-netns-statedir.patch
deleted file mode 100644
index a693eb7..0000000
--- a/third_party/go/patches/containerd-netns-statedir.patch
+++ /dev/null
@@ -1,96 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 3e7a8cebf9d40487adc7d4a22b5c628add5e7eac Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@nexantic.com>
-Date: Wed, 27 Jan 2021 13:05:30 +0100
-Subject: [PATCH] Move netns directory into StateDir
-
----
- pkg/netns/netns_unix.go   | 12 +++++-------
- pkg/server/sandbox_run.go |  3 ++-
- 2 files changed, 7 insertions(+), 8 deletions(-)
-
-diff --git a/pkg/netns/netns_unix.go b/pkg/netns/netns_unix.go
-index 7449e235..b31716cb 100644
---- a/pkg/netns/netns_unix.go
-+++ b/pkg/netns/netns_unix.go
-@@ -48,14 +48,12 @@ import (
- 	osinterface "github.com/containerd/cri/pkg/os"
- )
- 
--const nsRunDir = "/var/run/netns"
--
- // Some of the following functions are migrated from
- // https://github.com/containernetworking/plugins/blob/master/pkg/testutils/netns_linux.go
- 
- // newNS creates a new persistent (bind-mounted) network namespace and returns the
- // path to the network namespace.
--func newNS() (nsPath string, err error) {
-+func newNS(baseDir string) (nsPath string, err error) {
- 	b := make([]byte, 16)
- 	if _, err := rand.Reader.Read(b); err != nil {
- 		return "", errors.Wrap(err, "failed to generate random netns name")
-@@ -64,13 +62,13 @@ func newNS() (nsPath string, err error) {
- 	// Create the directory for mounting network namespaces
- 	// This needs to be a shared mountpoint in case it is mounted in to
- 	// other namespaces (containers)
--	if err := os.MkdirAll(nsRunDir, 0755); err != nil {
-+	if err := os.MkdirAll(baseDir, 0755); err != nil {
- 		return "", err
- 	}
- 
- 	// create an empty file at the mount point
- 	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
--	nsPath = path.Join(nsRunDir, nsName)
-+	nsPath = path.Join(baseDir, nsName)
- 	mountPointFd, err := os.Create(nsPath)
- 	if err != nil {
- 		return "", err
-@@ -164,8 +162,8 @@ type NetNS struct {
- }
- 
- // NewNetNS creates a network namespace.
--func NewNetNS() (*NetNS, error) {
--	path, err := newNS()
-+func NewNetNS(baseDir string) (*NetNS, error) {
-+	path, err := newNS(baseDir)
- 	if err != nil {
- 		return nil, errors.Wrap(err, "failed to setup netns")
- 	}
-diff --git a/pkg/server/sandbox_run.go b/pkg/server/sandbox_run.go
-index dd4c51e3..32a2d6e8 100644
---- a/pkg/server/sandbox_run.go
-+++ b/pkg/server/sandbox_run.go
-@@ -19,6 +19,7 @@ package server
- import (
- 	"encoding/json"
- 	"math"
-+	"path/filepath"
- 	goruntime "runtime"
- 	"strings"
- 
-@@ -117,7 +118,7 @@ func (c *criService) RunPodSandbox(ctx context.Context, r *runtime.RunPodSandbox
- 		// handle. NetNSPath in sandbox metadata and NetNS is non empty only for non host network
- 		// namespaces. If the pod is in host network namespace then both are empty and should not
- 		// be used.
--		sandbox.NetNS, err = netns.NewNetNS()
-+		sandbox.NetNS, err = netns.NewNetNS(filepath.Join(c.config.StateDir, "netns"))
- 		if err != nil {
- 			return nil, errors.Wrapf(err, "failed to create network namespace for sandbox %q", id)
- 		}
--- 
-2.25.1
-
diff --git a/third_party/go/patches/containerd-no-tracing.patch b/third_party/go/patches/containerd-no-tracing.patch
new file mode 100644
index 0000000..ee6f8c8
--- /dev/null
+++ b/third_party/go/patches/containerd-no-tracing.patch
@@ -0,0 +1,43 @@
+From 26ac5a008b349b77288d29cc483ea5d6129f298c Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 16 Mar 2022 18:35:44 +0100
+Subject: [PATCH] Disable tracing for otel compatibility
+
+---
+ cmd/containerd/builtins.go     | 1 -
+ cmd/containerd/command/main.go | 2 --
+ 2 files changed, 3 deletions(-)
+
+diff --git a/cmd/containerd/builtins.go b/cmd/containerd/builtins.go
+index 8c6f1fe86..dd8b1d3d9 100644
+--- a/cmd/containerd/builtins.go
++++ b/cmd/containerd/builtins.go
+@@ -36,5 +36,4 @@ import (
+ 	_ "github.com/containerd/containerd/services/snapshots"
+ 	_ "github.com/containerd/containerd/services/tasks"
+ 	_ "github.com/containerd/containerd/services/version"
+-	_ "github.com/containerd/containerd/tracing/plugin"
+ )
+diff --git a/cmd/containerd/command/main.go b/cmd/containerd/command/main.go
+index f549e08a3..dfde3599e 100644
+--- a/cmd/containerd/command/main.go
++++ b/cmd/containerd/command/main.go
+@@ -35,7 +35,6 @@ import (
+ 	"github.com/containerd/containerd/services/server"
+ 	srvconfig "github.com/containerd/containerd/services/server/config"
+ 	"github.com/containerd/containerd/sys"
+-	"github.com/containerd/containerd/tracing"
+ 	"github.com/containerd/containerd/version"
+ 	"github.com/sirupsen/logrus"
+ 	"github.com/urfave/cli"
+@@ -370,7 +369,6 @@ func setLogFormat(config *srvconfig.Config) error {
+ }
+ 
+ func setLogHooks() {
+-	logrus.StandardLogger().AddHook(tracing.NewLogrusHook())
+ }
+ 
+ func dumpStacks(writeToFile bool) {
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/coredns-remove-unused-plugins.patch b/third_party/go/patches/coredns-remove-unused-plugins.patch
index 9180d48..e952585 100644
--- a/third_party/go/patches/coredns-remove-unused-plugins.patch
+++ b/third_party/go/patches/coredns-remove-unused-plugins.patch
@@ -1,202 +1,156 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From b8a216a10775163dac6267127c35c9230ccbc5f7 Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@nexantic.com>
-Date: Mon, 27 Jul 2020 14:25:03 +0200
-Subject: [PATCH] Removed unused plugins
+From 355d38f38cf241b91a862795892bcb87c5729d2e Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 16 Mar 2022 13:39:39 +0100
+Subject: [PATCH] Remove unused plugins
 
 ---
- core/dnsserver/zdirectives.go | 26 --------------------------
- core/plugin/zplugin.go        | 26 --------------------------
- plugin.cfg                    | 26 --------------------------
- plugin/kubernetes/setup.go    |  3 ---
- 4 files changed, 81 deletions(-)
+ core/dnsserver/zdirectives.go | 7 -------
+ core/plugin/zplugin.go        | 7 -------
+ plugin.cfg                    | 7 -------
+ plugin/kubernetes/setup.go    | 3 ---
+ 4 files changed, 24 deletions(-)
 
 diff --git a/core/dnsserver/zdirectives.go b/core/dnsserver/zdirectives.go
-index 1bf449cb..c3619160 100644
+index bca21718..678c02c6 100644
 --- a/core/dnsserver/zdirectives.go
 +++ b/core/dnsserver/zdirectives.go
-@@ -11,48 +11,22 @@ package dnsserver
+@@ -11,7 +11,6 @@ package dnsserver
  // care what plugin above them are doing.
  var Directives = []string{
  	"metadata",
--	"cancel",
--	"tls",
+-	"geoip",
+ 	"cancel",
+ 	"tls",
  	"reload",
- 	"nsid",
- 	"bufsize",
--	"root",
+@@ -20,14 +19,12 @@ var Directives = []string{
+ 	"root",
  	"bind",
  	"debug",
 -	"trace",
--	"ready",
+ 	"ready",
  	"health",
--	"pprof",
--	"prometheus",
+ 	"pprof",
+ 	"prometheus",
  	"errors",
  	"log",
 -	"dnstap",
+ 	"local",
  	"dns64",
  	"acl",
- 	"any",
--	"chaos",
- 	"loadbalance",
- 	"cache",
--	"rewrite",
--	"dnssec",
--	"autopath",
+@@ -43,15 +40,11 @@ var Directives = []string{
  	"template",
--	"transfer",
+ 	"transfer",
  	"hosts",
 -	"route53",
 -	"azure",
 -	"clouddns",
--	"k8s_external",
+ 	"k8s_external",
  	"kubernetes",
--	"file",
--	"auto",
--	"secondary",
+ 	"file",
+ 	"auto",
+ 	"secondary",
 -	"etcd",
--	"loop",
+ 	"loop",
  	"forward",
--	"grpc",
--	"erratic",
- 	"whoami",
--	"on",
--	"sign",
- }
+ 	"grpc",
 diff --git a/core/plugin/zplugin.go b/core/plugin/zplugin.go
-index cf83be70..de026516 100644
+index a9167eea..2a5d0b13 100644
 --- a/core/plugin/zplugin.go
 +++ b/core/plugin/zplugin.go
-@@ -4,49 +4,23 @@ package plugin
-
- import (
- 	// Include all plugins.
--	_ "github.com/caddyserver/caddy/onevent"
- 	_ "github.com/coredns/coredns/plugin/acl"
+@@ -9,23 +9,18 @@ import (
  	_ "github.com/coredns/coredns/plugin/any"
--	_ "github.com/coredns/coredns/plugin/auto"
--	_ "github.com/coredns/coredns/plugin/autopath"
+ 	_ "github.com/coredns/coredns/plugin/auto"
+ 	_ "github.com/coredns/coredns/plugin/autopath"
 -	_ "github.com/coredns/coredns/plugin/azure"
  	_ "github.com/coredns/coredns/plugin/bind"
  	_ "github.com/coredns/coredns/plugin/bufsize"
  	_ "github.com/coredns/coredns/plugin/cache"
--	_ "github.com/coredns/coredns/plugin/cancel"
--	_ "github.com/coredns/coredns/plugin/chaos"
+ 	_ "github.com/coredns/coredns/plugin/cancel"
+ 	_ "github.com/coredns/coredns/plugin/chaos"
 -	_ "github.com/coredns/coredns/plugin/clouddns"
  	_ "github.com/coredns/coredns/plugin/debug"
  	_ "github.com/coredns/coredns/plugin/dns64"
--	_ "github.com/coredns/coredns/plugin/dnssec"
+ 	_ "github.com/coredns/coredns/plugin/dnssec"
 -	_ "github.com/coredns/coredns/plugin/dnstap"
--	_ "github.com/coredns/coredns/plugin/erratic"
+ 	_ "github.com/coredns/coredns/plugin/erratic"
  	_ "github.com/coredns/coredns/plugin/errors"
 -	_ "github.com/coredns/coredns/plugin/etcd"
--	_ "github.com/coredns/coredns/plugin/file"
+ 	_ "github.com/coredns/coredns/plugin/file"
  	_ "github.com/coredns/coredns/plugin/forward"
--	_ "github.com/coredns/coredns/plugin/grpc"
+-	_ "github.com/coredns/coredns/plugin/geoip"
+ 	_ "github.com/coredns/coredns/plugin/grpc"
+ 	_ "github.com/coredns/coredns/plugin/header"
  	_ "github.com/coredns/coredns/plugin/health"
- 	_ "github.com/coredns/coredns/plugin/hosts"
--	_ "github.com/coredns/coredns/plugin/k8s_external"
- 	_ "github.com/coredns/coredns/plugin/kubernetes"
- 	_ "github.com/coredns/coredns/plugin/loadbalance"
- 	_ "github.com/coredns/coredns/plugin/log"
--	_ "github.com/coredns/coredns/plugin/loop"
- 	_ "github.com/coredns/coredns/plugin/metadata"
--	_ "github.com/coredns/coredns/plugin/metrics"
- 	_ "github.com/coredns/coredns/plugin/nsid"
--	_ "github.com/coredns/coredns/plugin/pprof"
--	_ "github.com/coredns/coredns/plugin/ready"
+@@ -45,12 +40,10 @@ import (
  	_ "github.com/coredns/coredns/plugin/reload"
--	_ "github.com/coredns/coredns/plugin/rewrite"
--	_ "github.com/coredns/coredns/plugin/root"
+ 	_ "github.com/coredns/coredns/plugin/rewrite"
+ 	_ "github.com/coredns/coredns/plugin/root"
 -	_ "github.com/coredns/coredns/plugin/route53"
--	_ "github.com/coredns/coredns/plugin/secondary"
--	_ "github.com/coredns/coredns/plugin/sign"
+ 	_ "github.com/coredns/coredns/plugin/secondary"
+ 	_ "github.com/coredns/coredns/plugin/sign"
  	_ "github.com/coredns/coredns/plugin/template"
--	_ "github.com/coredns/coredns/plugin/tls"
+ 	_ "github.com/coredns/coredns/plugin/tls"
 -	_ "github.com/coredns/coredns/plugin/trace"
--	_ "github.com/coredns/coredns/plugin/transfer"
+ 	_ "github.com/coredns/coredns/plugin/transfer"
  	_ "github.com/coredns/coredns/plugin/whoami"
  )
 diff --git a/plugin.cfg b/plugin.cfg
-index c389d74e..80a8bc51 100644
+index 628e7141..d4ce1c89 100644
 --- a/plugin.cfg
 +++ b/plugin.cfg
-@@ -20,47 +20,21 @@
+@@ -20,7 +20,6 @@
  # log:log
-
+ 
  metadata:metadata
--cancel:cancel
--tls:tls
+-geoip:geoip
+ cancel:cancel
+ tls:tls
  reload:reload
- nsid:nsid
- bufsize:bufsize
--root:root
+@@ -29,14 +28,12 @@ bufsize:bufsize
+ root:root
  bind:bind
  debug:debug
 -trace:trace
--ready:ready
+ ready:ready
  health:health
--pprof:pprof
--prometheus:metrics
+ pprof:pprof
+ prometheus:metrics
  errors:errors
  log:log
 -dnstap:dnstap
+ local:local
  dns64:dns64
  acl:acl
- any:any
--chaos:chaos
- loadbalance:loadbalance
- cache:cache
--rewrite:rewrite
--dnssec:dnssec
--autopath:autopath
+@@ -52,15 +49,11 @@ minimal:minimal
  template:template
--transfer:transfer
+ transfer:transfer
  hosts:hosts
 -route53:route53
 -azure:azure
 -clouddns:clouddns
--k8s_external:k8s_external
+ k8s_external:k8s_external
  kubernetes:kubernetes
--file:file
--auto:auto
--secondary:secondary
+ file:file
+ auto:auto
+ secondary:secondary
 -etcd:etcd
--loop:loop
+ loop:loop
  forward:forward
--grpc:grpc
--erratic:erratic
- whoami:whoami
--on:github.com/caddyserver/caddy/onevent
--sign:sign
+ grpc:grpc
 diff --git a/plugin/kubernetes/setup.go b/plugin/kubernetes/setup.go
-index 0c46a3ab..827fc141 100644
+index 5f5ab058..ae1a7cf8 100644
 --- a/plugin/kubernetes/setup.go
 +++ b/plugin/kubernetes/setup.go
-@@ -20,9 +20,6 @@ import (
- 	"github.com/caddyserver/caddy"
+@@ -17,9 +17,6 @@ import (
+ 
  	"github.com/miekg/dns"
  	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
 -	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"       // pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
 -	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"      // pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
 -	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack" // pull this in here, because we want it excluded if plugin.cfg doesn't have k8s
  	"k8s.io/client-go/tools/clientcmd"
- 	"k8s.io/klog"
+ 	"k8s.io/klog/v2"
  )
---
+-- 
 2.25.1
+
diff --git a/third_party/go/patches/delve-fix-cgo.patch b/third_party/go/patches/delve-fix-cgo.patch
new file mode 100644
index 0000000..d8a5254
--- /dev/null
+++ b/third_party/go/patches/delve-fix-cgo.patch
@@ -0,0 +1,11 @@
+diff -urN b/pkg/proc/internal/ebpf/BUILD.bazel c/pkg/proc/internal/ebpf/BUILD.bazel
+--- b/pkg/proc/internal/ebpf/BUILD.bazel  2022-01-14 23:29:02.948089761 +0000
++++ c/pkg/proc/internal/ebpf/BUILD.bazel  2022-01-14 23:13:52.964810803 +0000
+@@ -7,6 +7,7 @@
+         "helpers.go",
+         "helpers_disabled.go",
+         "trace_bpfel_x86.go",
++        "bpf/include/function_vals.bpf.h",
+     ],
+     cgo = True,
+     embedsrcs = select({
\ No newline at end of file
diff --git a/third_party/go/patches/etcd-fix-stub.patch b/third_party/go/patches/etcd-fix-stub.patch
new file mode 100644
index 0000000..bb47ac4
--- /dev/null
+++ b/third_party/go/patches/etcd-fix-stub.patch
@@ -0,0 +1,9 @@
+--- a/grpc_testing/stub_server.go
++++ b/grpc_testing/stub_server.go
+@@ -86,5 +86,5 @@ func (d dummyStubServer) UnaryCall(context.Context, *testpb.SimpleRequest) (*tes
+ // NewDummyStubServer creates a simple test server that serves Unary calls with
+ // responses with the given payload.
+ func NewDummyStubServer(body []byte) *StubServer {
+-	return New(dummyStubServer{body: body})
++	return New(&dummyStubServer{body: body})
+ }
diff --git a/third_party/go/patches/go-proto-validators-default-alias.patch b/third_party/go/patches/go-proto-validators-default-alias.patch
deleted file mode 100644
index 46f21a9..0000000
--- a/third_party/go/patches/go-proto-validators-default-alias.patch
+++ /dev/null
@@ -1,29 +0,0 @@
-From 49b5f957d01cd50e5303cfc0e6c7f350d7e1c24e Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@monogon.tech>
-Date: Tue, 30 Nov 2021 03:19:37 +0100
-Subject: [PATCH] Alias go_default_library in BUILD to make Gazelle work
-
----
- BUILD.bazel | 6 ++++++
- 1 file changed, 6 insertions(+)
-
-diff --git a/BUILD.bazel b/BUILD.bazel
-index 9a1edd4..e5b8f96 100644
---- a/BUILD.bazel
-+++ b/BUILD.bazel
-@@ -14,6 +14,12 @@ gazelle(
- # By default resolve the top-level package to the gogo variant as it's required for the plugin compilation.
- # gazelle:resolve go github.com/mwitkow/go-proto-validators //:validators_gogo
- 
-+alias(
-+    name = "go_default_library",
-+    actual = "validators_golang",
-+    visibility = ["//visibility:public"],
-+)
-+
- go_proto_compiler(
-     name = "go_proto_validators",
-     options = ["gogoimport=false"],
--- 
-2.25.1
-
diff --git a/third_party/go/patches/go-tpm-tools-fix-proto-paths.patch b/third_party/go/patches/go-tpm-tools-fix-proto-paths.patch
new file mode 100644
index 0000000..ae7fcf3
--- /dev/null
+++ b/third_party/go/patches/go-tpm-tools-fix-proto-paths.patch
@@ -0,0 +1,66 @@
+From a799a94b7b67b21d1eecfe66dbe264ef3e276cd3 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 30 Mar 2022 15:48:23 +0200
+Subject: [PATCH] Fix proto file paths
+
+---
+ proto/0001-Fix-proto-file-paths.patch | 23 +++++++++++++++++++++++
+ proto/{ => attest}/attest.proto       |  2 +-
+ proto/{ => tpm}/tpm.proto             |  0
+ 3 files changed, 24 insertions(+), 1 deletion(-)
+ create mode 100644 proto/0001-Fix-proto-file-paths.patch
+ rename proto/{ => attest}/attest.proto (99%)
+ rename proto/{ => tpm}/tpm.proto (100%)
+
+diff --git a/proto/0001-Fix-proto-file-paths.patch b/proto/0001-Fix-proto-file-paths.patch
+new file mode 100644
+index 0000000..22e3b45
+--- /dev/null
++++ b/proto/0001-Fix-proto-file-paths.patch
+@@ -0,0 +1,23 @@
++From ba5bf5ac1e0b3d92e8f19a0feb2307515c0e2baf Mon Sep 17 00:00:00 2001
++From: Lorenz Brun <lorenz@monogon.tech>
++Date: Wed, 30 Mar 2022 15:48:23 +0200
++Subject: [PATCH] Fix proto file paths
++
++---
++ proto/{ => attest}/attest.proto | 0
++ proto/{ => tpm}/tpm.proto       | 0
++ 2 files changed, 0 insertions(+), 0 deletions(-)
++ rename proto/{ => attest}/attest.proto (100%)
++ rename proto/{ => tpm}/tpm.proto (100%)
++
++diff --git a/proto/attest.proto b/proto/attest/attest.proto
++similarity index 100%
++rename from proto/attest.proto
++rename to proto/attest/attest.proto
++diff --git a/proto/tpm.proto b/proto/tpm/tpm.proto
++similarity index 100%
++rename from proto/tpm.proto
++rename to proto/tpm/tpm.proto
++-- 
++2.25.1
++
+diff --git a/proto/attest.proto b/proto/attest/attest.proto
+similarity index 99%
+rename from proto/attest.proto
+rename to proto/attest/attest.proto
+index fbaf5bd..7960297 100644
+--- a/proto/attest.proto
++++ b/proto/attest/attest.proto
+@@ -3,7 +3,7 @@ syntax = "proto3";
+ package attest;
+ option go_package = "github.com/google/go-tpm-tools/proto/attest";
+ 
+-import "tpm.proto";
++import "proto/tpm/tpm.proto";
+ 
+ // Information uniquely identifying a GCE instance. Can be used to create an
+ // instance URL, which can then be used with GCE APIs. Formatted like:
+diff --git a/proto/tpm.proto b/proto/tpm/tpm.proto
+similarity index 100%
+rename from proto/tpm.proto
+rename to proto/tpm/tpm.proto
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/gvisor-build-against-newer-runtime-specs.patch b/third_party/go/patches/gvisor-build-against-newer-runtime-specs.patch
deleted file mode 100644
index f653492..0000000
--- a/third_party/go/patches/gvisor-build-against-newer-runtime-specs.patch
+++ /dev/null
@@ -1,41 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 9fc5e36f4e0bd691fc502372ebf837d24fbe5297 Mon Sep 17 00:00:00 2001
-From: Serge Bazanski <serge@nexantic.com>
-Date: Fri, 15 Jan 2021 16:22:33 +0100
-Subject: [PATCH] Build against new opencontainers runtime-specs
-Company: nexantic GmbH
-
----
- runsc/container/container.go | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/runsc/container/container.go b/runsc/container/container.go
-index 418a27beb..8508ded5a 100644
---- a/runsc/container/container.go
-+++ b/runsc/container/container.go
-@@ -747,7 +747,7 @@ func (c *Container) State() specs.State {
- 	return specs.State{
- 		Version: specs.Version,
- 		ID:      c.ID,
--		Status:  c.Status.String(),
-+		Status:  specs.ContainerState(c.Status.String()),
- 		Pid:     c.SandboxPid(),
- 		Bundle:  c.BundleDir,
- 	}
--- 
-2.26.2
-
diff --git a/third_party/go/patches/gvisor-cgroup-fix.patch b/third_party/go/patches/gvisor-cgroup-fix.patch
deleted file mode 100644
index aba0b4a..0000000
--- a/third_party/go/patches/gvisor-cgroup-fix.patch
+++ /dev/null
@@ -1,27 +0,0 @@
-From fcd48e672489f41c9977e092937ff806a7e772bd Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@monogon.tech>
-Date: Thu, 11 Nov 2021 13:58:50 +0100
-Subject: [PATCH] Make systemd cgroup optional
-
-This breaks gVisor on systems that use custom inits/cgroup setups which
-don't have this cgroup and don't need it.
----
- runsc/cgroup/cgroup.go | 2 +-
- 1 file changed, 1 insertion(+), 1 deletion(-)
-
-diff --git a/runsc/cgroup/cgroup.go b/runsc/cgroup/cgroup.go
-index 0eb5821a9..2116bddca 100644
---- a/runsc/cgroup/cgroup.go
-+++ b/runsc/cgroup/cgroup.go
-@@ -58,7 +58,7 @@ var controllers = map[string]controller{
- 	"freezer":    &noop{},
- 	"perf_event": &noop{},
- 	"rdma":       &noop{isOptional: true},
--	"systemd":    &noop{},
-+	"systemd":    &noop{isOptional: true},
- }
- 
- // IsOnlyV2 checks whether cgroups V2 is enabled and V1 is not.
--- 
-2.25.1
-
diff --git a/third_party/go/patches/gvisor-containerd-compat.patch b/third_party/go/patches/gvisor-containerd-compat.patch
new file mode 100644
index 0000000..89eb40f
--- /dev/null
+++ b/third_party/go/patches/gvisor-containerd-compat.patch
@@ -0,0 +1,114 @@
+From bf861ce45721791336b617a44844613bb74c677a Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Tue, 22 Mar 2022 00:52:48 +0100
+Subject: [PATCH] containerd 1.6 compatibility
+
+---
+ pkg/shim/proc/exec.go     |  2 +-
+ pkg/shim/proc/init.go     |  2 +-
+ pkg/shim/service.go       | 17 +++++++++--------
+ pkg/shim/service_linux.go |  2 +-
+ 4 files changed, 12 insertions(+), 11 deletions(-)
+
+diff --git a/pkg/shim/proc/exec.go b/pkg/shim/proc/exec.go
+index da2e21598..d0d14dd24 100644
+--- a/pkg/shim/proc/exec.go
++++ b/pkg/shim/proc/exec.go
+@@ -238,7 +238,7 @@ func (e *execProcess) start(ctx context.Context) error {
+ 		if err != nil {
+ 			return fmt.Errorf("failed to retrieve console master: %w", err)
+ 		}
+-		if e.console, err = e.parent.Platform.CopyConsole(ctx, console, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg); err != nil {
++		if e.console, err = e.parent.Platform.CopyConsole(ctx, console, e.id, e.stdio.Stdin, e.stdio.Stdout, e.stdio.Stderr, &e.wg); err != nil {
+ 			return fmt.Errorf("failed to start console copy: %w", err)
+ 		}
+ 	} else if !e.stdio.IsNull() {
+diff --git a/pkg/shim/proc/init.go b/pkg/shim/proc/init.go
+index 6bf090813..76df1101a 100644
+--- a/pkg/shim/proc/init.go
++++ b/pkg/shim/proc/init.go
+@@ -152,7 +152,7 @@ func (p *Init) Create(ctx context.Context, r *CreateConfig) (err error) {
+ 		if err != nil {
+ 			return fmt.Errorf("failed to retrieve console master: %w", err)
+ 		}
+-		console, err = p.Platform.CopyConsole(ctx, console, r.Stdin, r.Stdout, r.Stderr, &p.wg)
++		console, err = p.Platform.CopyConsole(ctx, console, r.ID, r.Stdin, r.Stdout, r.Stderr, &p.wg)
+ 		if err != nil {
+ 			return fmt.Errorf("failed to start console copy: %w", err)
+ 		}
+diff --git a/pkg/shim/service.go b/pkg/shim/service.go
+index 68966afdf..772168052 100644
+--- a/pkg/shim/service.go
++++ b/pkg/shim/service.go
+@@ -50,7 +50,7 @@ import (
+ 	"github.com/sirupsen/logrus"
+ 	"golang.org/x/sys/unix"
+ 	"gvisor.dev/gvisor/pkg/cleanup"
+-	"gvisor.dev/gvisor/pkg/shim/runtimeoptions/v14"
++	v14 "gvisor.dev/gvisor/pkg/shim/runtimeoptions/v14"
+ 
+ 	"gvisor.dev/gvisor/pkg/shim/proc"
+ 	"gvisor.dev/gvisor/pkg/shim/runsc"
+@@ -189,7 +189,7 @@ type service struct {
+ 	shimAddress string
+ }
+ 
+-func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
++func (s *service) newCommand(ctx context.Context, id, containerdBinary, containerdAddress string) (*exec.Cmd, error) {
+ 	ns, err := namespaces.NamespaceRequired(ctx)
+ 	if err != nil {
+ 		return nil, err
+@@ -204,6 +204,7 @@ func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAd
+ 	}
+ 	args := []string{
+ 		"-namespace", ns,
++		"-id", id,
+ 		"-address", containerdAddress,
+ 		"-publish-binary", containerdBinary,
+ 	}
+@@ -219,14 +220,14 @@ func (s *service) newCommand(ctx context.Context, containerdBinary, containerdAd
+ 	return cmd, nil
+ }
+ 
+-func (s *service) StartShim(ctx context.Context, id, containerdBinary, containerdAddress, containerdTTRPCAddress string) (string, error) {
+-	log.L.Debugf("StartShim, id: %s, binary: %q, address: %q", id, containerdBinary, containerdAddress)
++func (s *service) StartShim(ctx context.Context, opts shim.StartOpts) (string, error) {
++	log.L.Debugf("StartShim, id: %s, binary: %q, address: %q", opts.ID, opts.ContainerdBinary, opts.Address)
+ 
+-	cmd, err := s.newCommand(ctx, containerdBinary, containerdAddress)
++	cmd, err := s.newCommand(ctx, opts.ID, opts.ContainerdBinary, opts.Address)
+ 	if err != nil {
+ 		return "", err
+ 	}
+-	address, err := shim.SocketAddress(ctx, containerdAddress, id)
++	address, err := shim.SocketAddress(ctx, opts.Address, opts.ID)
+ 	if err != nil {
+ 		return "", err
+ 	}
+@@ -280,8 +281,8 @@ func (s *service) StartShim(ctx context.Context, id, containerdBinary, container
+ 	if err := shim.WriteAddress(shimAddressPath, address); err != nil {
+ 		return "", err
+ 	}
+-	if err := shim.SetScore(cmd.Process.Pid); err != nil {
+-		return "", fmt.Errorf("failed to set OOM Score on shim: %w", err)
++	if err := shim.AdjustOOMScore(cmd.Process.Pid); err != nil {
++		return "", fmt.Errorf("failed to adjust OOM score for shim: %w", err)
+ 	}
+ 	cu.Release()
+ 	return address, nil
+diff --git a/pkg/shim/service_linux.go b/pkg/shim/service_linux.go
+index fb2f8b062..52c82ca90 100644
+--- a/pkg/shim/service_linux.go
++++ b/pkg/shim/service_linux.go
+@@ -33,7 +33,7 @@ type linuxPlatform struct {
+ 	epoller *console.Epoller
+ }
+ 
+-func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console, stdin, stdout, stderr string, wg *sync.WaitGroup) (console.Console, error) {
++func (p *linuxPlatform) CopyConsole(ctx context.Context, console console.Console, id, stdin, stdout, stderr string, wg *sync.WaitGroup) (console.Console, error) {
+ 	if p.epoller == nil {
+ 		return nil, fmt.Errorf("uninitialized epoller")
+ 	}
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/gvisor-fix-resolution.patch b/third_party/go/patches/gvisor-fix-resolution.patch
new file mode 100644
index 0000000..71e0af5
--- /dev/null
+++ b/third_party/go/patches/gvisor-fix-resolution.patch
@@ -0,0 +1,10 @@
+--- a/pkg/coverage/BUILD.bazel
++++ b/pkg/coverage/BUILD.bazel
+@@ -11,7 +11,7 @@ go_library(
+     deps = [
+         "//pkg/hostarch",
+         "//pkg/sync",
+-        "@io_bazel_rules_go//go/tools/coverdata:go_default_library",
++        "@io_bazel_rules_go//go/tools/coverdata",
+     ],
+ )
diff --git a/third_party/go/patches/gvisor-shim-root.patch b/third_party/go/patches/gvisor-shim-root.patch
deleted file mode 100644
index 6b10797..0000000
--- a/third_party/go/patches/gvisor-shim-root.patch
+++ /dev/null
@@ -1,67 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 6cbcb8e61b60046e51ff79674b78031707739401 Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@brun.one>
-Date: Wed, 6 May 2020 18:52:14 +0200
-Subject: [PATCH] Fix externally-configured non-standard root paths
-
-Going upstream as https://github.com/google/gvisor-containerd-shim/pull/60
-
----
- pkg/v2/service.go | 6 ++++--
- 1 file changed, 4 insertions(+), 2 deletions(-)
-
-diff --git a/pkg/v2/service.go b/pkg/v2/service.go
-index c1df4b8..f7f5847 100644
---- a/pkg/v2/service.go
-+++ b/pkg/v2/service.go
-@@ -103,6 +103,7 @@ type service struct {
- 	processes map[string]process.Process
- 	events    chan interface{}
- 	platform  stdio.Platform
-+	opts      options.Options
- 	ec        chan proc.Exit
- 
- 	id     string
-@@ -194,7 +195,7 @@ func (s *service) Cleanup(ctx context.Context) (*taskAPI.DeleteResponse, error)
- 	if err != nil {
- 		return nil, err
- 	}
--	r := proc.NewRunsc(proc.RunscRoot, path, ns, runtime, nil)
-+	r := proc.NewRunsc(s.opts.Root, path, ns, runtime, nil)
- 	if err := r.Delete(ctx, s.id, &runsc.DeleteOpts{
- 		Force: true,
- 	}); err != nil {
-@@ -344,6 +345,7 @@ func (s *service) Create(ctx context.Context, r *taskAPI.CreateTaskRequest) (_ *
- 	s.id = r.ID
- 	s.bundle = r.Bundle
- 	s.task = process
-+	s.opts = opts
- 	return &taskAPI.CreateTaskResponse{
- 		Pid: uint32(process.Pid()),
- 	}, nil
-@@ -577,7 +579,7 @@ func (s *service) Stats(ctx context.Context, r *taskAPI.StatsRequest) (*taskAPI.
- 	if err != nil {
- 		return nil, err
- 	}
--	rs := proc.NewRunsc(proc.RunscRoot, path, ns, runtime, nil)
-+	rs := proc.NewRunsc(s.opts.Root, path, ns, runtime, nil)
- 	stats, err := rs.Stats(ctx, s.id)
- 	if err != nil {
- 		return nil, err
--- 
-2.25.1
-
diff --git a/third_party/go/patches/gvisor.patch b/third_party/go/patches/gvisor.patch
deleted file mode 100644
index 143ee59..0000000
--- a/third_party/go/patches/gvisor.patch
+++ /dev/null
@@ -1,97 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 69b2c3b6e2594a3f28e4ea1141bef542456b3eb2 Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@brun.one>
-Date: Wed, 5 Feb 2020 17:14:19 +0100
-Subject: [PATCH] Fix vdso include that breaks in an external
-
----
- vdso/cycle_clock.h | 2 +-
- vdso/seqlock.h     | 4 ++--
- vdso/vdso.cc       | 4 ++--
- vdso/vdso_time.cc  | 8 ++++----
- 4 files changed, 9 insertions(+), 9 deletions(-)
-
-diff --git a/vdso/cycle_clock.h b/vdso/cycle_clock.h
-index 5d3fbb25..fe15812c 100644
---- a/vdso/cycle_clock.h
-+++ b/vdso/cycle_clock.h
-@@ -17,7 +17,7 @@
- 
- #include <stdint.h>
- 
--#include "vdso/barrier.h"
-+#include "barrier.h"
- 
- namespace vdso {
- 
-diff --git a/vdso/seqlock.h b/vdso/seqlock.h
-index 7a173174..fff99748 100644
---- a/vdso/seqlock.h
-+++ b/vdso/seqlock.h
-@@ -18,8 +18,8 @@
- 
- #include <stdint.h>
- 
--#include "vdso/barrier.h"
--#include "vdso/compiler.h"
-+#include "barrier.h"
-+#include "compiler.h"
- 
- namespace vdso {
- 
-diff --git a/vdso/vdso.cc b/vdso/vdso.cc
-index 8bb80a7a..210d31ff 100644
---- a/vdso/vdso.cc
-+++ b/vdso/vdso.cc
-@@ -19,8 +19,8 @@
- #include <sys/time.h>
- #include <time.h>
- 
--#include "vdso/syscalls.h"
--#include "vdso/vdso_time.h"
-+#include "syscalls.h"
-+#include "vdso_time.h"
- 
- namespace vdso {
- namespace {
-diff --git a/vdso/vdso_time.cc b/vdso/vdso_time.cc
-index 1bb4bb86..fb5b281f 100644
---- a/vdso/vdso_time.cc
-+++ b/vdso/vdso_time.cc
-@@ -12,15 +12,15 @@
- // See the License for the specific language governing permissions and
- // limitations under the License.
- 
--#include "vdso/vdso_time.h"
-+#include "vdso_time.h"
- 
- #include <stdint.h>
- #include <sys/time.h>
- #include <time.h>
- 
--#include "vdso/cycle_clock.h"
--#include "vdso/seqlock.h"
--#include "vdso/syscalls.h"
-+#include "cycle_clock.h"
-+#include "seqlock.h"
-+#include "syscalls.h"
- 
- // struct params defines the layout of the parameter page maintained by the
- // kernel (i.e., sentry).
--- 
-2.20.1
-
diff --git a/third_party/go/patches/k8s-adopt-to-go-jose-2.3.patch b/third_party/go/patches/k8s-adopt-to-go-jose-2.3.patch
new file mode 100644
index 0000000..9efd08a
--- /dev/null
+++ b/third_party/go/patches/k8s-adopt-to-go-jose-2.3.patch
@@ -0,0 +1,136 @@
+From a1411288423dfc4062844b9f699a30fd7cbe090d Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Mon, 21 Mar 2022 15:20:19 +0100
+Subject: [PATCH 1/2] Adopt to API breakage in go-jose 2.3.0
+
+---
+ pkg/serviceaccount/claims_test.go          | 40 +++++++++++-----------
+ test/integration/auth/svcaccttoken_test.go |  6 ++--
+ 2 files changed, 23 insertions(+), 23 deletions(-)
+
+diff --git a/pkg/serviceaccount/claims_test.go b/pkg/serviceaccount/claims_test.go
+index 2e968f60335..a0b5a595c2f 100644
+--- a/pkg/serviceaccount/claims_test.go
++++ b/pkg/serviceaccount/claims_test.go
+@@ -85,9 +85,9 @@ func TestClaims(t *testing.T) {
+ 
+ 			sc: &jwt.Claims{
+ 				Subject:   "system:serviceaccount:myns:mysvcacct",
+-				IssuedAt:  jwt.NumericDate(1514764800),
+-				NotBefore: jwt.NumericDate(1514764800),
+-				Expiry:    jwt.NumericDate(1514764800),
++				IssuedAt:  jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				NotBefore: jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				Expiry:    jwt.NewNumericDate(time.Unix(1514764800, 0)),
+ 			},
+ 			pc: &privateClaims{
+ 				Kubernetes: kubernetes{
+@@ -107,9 +107,9 @@ func TestClaims(t *testing.T) {
+ 
+ 			sc: &jwt.Claims{
+ 				Subject:   "system:serviceaccount:myns:mysvcacct",
+-				IssuedAt:  jwt.NumericDate(1514764800),
+-				NotBefore: jwt.NumericDate(1514764800),
+-				Expiry:    jwt.NumericDate(1514764800 + 100),
++				IssuedAt:  jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				NotBefore: jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				Expiry:    jwt.NewNumericDate(time.Unix(1514764800+100, 0)),
+ 			},
+ 			pc: &privateClaims{
+ 				Kubernetes: kubernetes{
+@@ -130,9 +130,9 @@ func TestClaims(t *testing.T) {
+ 			sc: &jwt.Claims{
+ 				Subject:   "system:serviceaccount:myns:mysvcacct",
+ 				Audience:  []string{"1"},
+-				IssuedAt:  jwt.NumericDate(1514764800),
+-				NotBefore: jwt.NumericDate(1514764800),
+-				Expiry:    jwt.NumericDate(1514764800 + 100),
++				IssuedAt:  jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				NotBefore: jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				Expiry:    jwt.NewNumericDate(time.Unix(1514764800+100, 0)),
+ 			},
+ 			pc: &privateClaims{
+ 				Kubernetes: kubernetes{
+@@ -152,9 +152,9 @@ func TestClaims(t *testing.T) {
+ 			sc: &jwt.Claims{
+ 				Subject:   "system:serviceaccount:myns:mysvcacct",
+ 				Audience:  []string{"1", "2"},
+-				IssuedAt:  jwt.NumericDate(1514764800),
+-				NotBefore: jwt.NumericDate(1514764800),
+-				Expiry:    jwt.NumericDate(1514764800 + 100),
++				IssuedAt:  jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				NotBefore: jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				Expiry:    jwt.NewNumericDate(time.Unix(1514764800+100, 0)),
+ 			},
+ 			pc: &privateClaims{
+ 				Kubernetes: kubernetes{
+@@ -175,16 +175,16 @@ func TestClaims(t *testing.T) {
+ 
+ 			sc: &jwt.Claims{
+ 				Subject:   "system:serviceaccount:myns:mysvcacct",
+-				IssuedAt:  jwt.NumericDate(1514764800),
+-				NotBefore: jwt.NumericDate(1514764800),
+-				Expiry:    jwt.NumericDate(1514764800 + 60*60*24),
++				IssuedAt:  jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				NotBefore: jwt.NewNumericDate(time.Unix(1514764800, 0)),
++				Expiry:    jwt.NewNumericDate(time.Unix(1514764800+60*60*24, 0)),
+ 			},
+ 			pc: &privateClaims{
+ 				Kubernetes: kubernetes{
+ 					Namespace: "myns",
+ 					Svcacct:   ref{Name: "mysvcacct", UID: "mysvcacct-uid"},
+ 					Pod:       &ref{Name: "mypod", UID: "mypod-uid"},
+-					WarnAfter: jwt.NumericDate(1514764800 + 60*60),
++					WarnAfter: jwt.NewNumericDate(time.Unix(1514764800+60*60, 0)),
+ 				},
+ 			},
+ 		},
+@@ -223,8 +223,8 @@ type claimTestCase struct {
+ 	name      string
+ 	getter    ServiceAccountTokenGetter
+ 	private   *privateClaims
+-	expiry    jwt.NumericDate
+-	notBefore jwt.NumericDate
++	expiry    *jwt.NumericDate
++	notBefore *jwt.NumericDate
+ 	expectErr string
+ }
+ 
+@@ -365,8 +365,8 @@ func TestValidatePrivateClaims(t *testing.T) {
+ 	for _, tc := range testcases {
+ 		t.Run(tc.name, func(t *testing.T) {
+ 			v := &validator{tc.getter}
+-			expiry := jwt.NumericDate(nowUnix)
+-			if tc.expiry != 0 {
++			expiry := jwt.NewNumericDate(time.Unix(nowUnix, 0))
++			if tc.expiry != nil {
+ 				expiry = tc.expiry
+ 			}
+ 			_, err := v.Validate(context.Background(), "", &jwt.Claims{Expiry: expiry, NotBefore: tc.notBefore}, tc.private)
+diff --git a/test/integration/auth/svcaccttoken_test.go b/test/integration/auth/svcaccttoken_test.go
+index da50bf4736e..5311b6c90c3 100644
+--- a/test/integration/auth/svcaccttoken_test.go
++++ b/test/integration/auth/svcaccttoken_test.go
+@@ -421,16 +421,16 @@ func TestServiceAccountTokenCreate(t *testing.T) {
+ 			t.Fatalf("error parsing warnafter: %v", err)
+ 		}
+ 
+-		if exp < int64(actualExpiry)-leeway || exp > int64(actualExpiry)+leeway {
++		if exp < int64(*actualExpiry)-leeway || exp > int64(*actualExpiry)+leeway {
+ 			t.Errorf("unexpected token exp %d, should within range of %d +- %d seconds", exp, actualExpiry, leeway)
+ 		}
+-		if warnafter < int64(assumedExpiry)-leeway || warnafter > int64(assumedExpiry)+leeway {
++		if warnafter < int64(*assumedExpiry)-leeway || warnafter > int64(*assumedExpiry)+leeway {
+ 			t.Errorf("unexpected token warnafter %d, should within range of %d +- %d seconds", warnafter, assumedExpiry, leeway)
+ 		}
+ 
+ 		checkExpiration(t, treq, requestExp)
+ 		expStatus := treq.Status.ExpirationTimestamp.Time.Unix()
+-		if expStatus < int64(assumedExpiry)-leeway || warnafter > int64(assumedExpiry)+leeway {
++		if expStatus < int64(*assumedExpiry)-leeway || warnafter > int64(*assumedExpiry)+leeway {
+ 			t.Errorf("unexpected expiration returned in tokenrequest status %d, should within range of %d +- %d seconds", expStatus, assumedExpiry, leeway)
+ 		}
+ 	})
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/k8s-adopt-to-runc-1.1.patch b/third_party/go/patches/k8s-adopt-to-runc-1.1.patch
new file mode 100644
index 0000000..8bfdcf2
--- /dev/null
+++ b/third_party/go/patches/k8s-adopt-to-runc-1.1.patch
@@ -0,0 +1,168 @@
+From 1564b39d0fbeac776a0d92236a0ca0c7cbdc6c5c Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Mon, 21 Mar 2022 15:21:25 +0100
+Subject: [PATCH 2/2] Adopt to API breakage in runc 1.1.0
+
+---
+ pkg/kubelet/cm/cgroup_manager_linux.go                | 11 ++++++-----
+ pkg/kubelet/cm/container_manager_linux.go             |  4 ++--
+ pkg/kubelet/cm/qos_container_manager_linux.go         |  4 ++--
+ pkg/kubelet/dockershim/cm/container_manager_linux.go  |  2 +-
+ .../kuberuntime/kuberuntime_container_linux.go        |  4 ++--
+ .../kuberuntime/kuberuntime_container_linux_test.go   |  6 +++---
+ 6 files changed, 16 insertions(+), 15 deletions(-)
+
+diff --git a/pkg/kubelet/cm/cgroup_manager_linux.go b/pkg/kubelet/cm/cgroup_manager_linux.go
+index 230173690d5..4bd50db5050 100644
+--- a/pkg/kubelet/cm/cgroup_manager_linux.go
++++ b/pkg/kubelet/cm/cgroup_manager_linux.go
+@@ -27,6 +27,7 @@ import (
+ 	"sync"
+ 	"time"
+ 
++	"github.com/opencontainers/runc/libcontainer/cgroups"
+ 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
+ 	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+ 	cgroupfs2 "github.com/opencontainers/runc/libcontainer/cgroups/fs2"
+@@ -150,18 +151,18 @@ func (l *libcontainerAdapter) newManager(cgroups *libcontainerconfigs.Cgroup, pa
+ 	switch l.cgroupManagerType {
+ 	case libcontainerCgroupfs:
+ 		if libcontainercgroups.IsCgroup2UnifiedMode() {
+-			return cgroupfs2.NewManager(cgroups, paths["memory"], false)
++			return cgroupfs2.NewManager(cgroups, paths["memory"])
+ 		}
+-		return cgroupfs.NewManager(cgroups, paths, false), nil
++		return cgroupfs.NewManager(cgroups, paths)
+ 	case libcontainerSystemd:
+ 		// this means you asked systemd to manage cgroups, but systemd was not on the host, so all you can do is panic...
+ 		if !cgroupsystemd.IsRunningSystemd() {
+ 			panic("systemd cgroup manager not available")
+ 		}
+ 		if libcontainercgroups.IsCgroup2UnifiedMode() {
+-			return cgroupsystemd.NewUnifiedManager(cgroups, paths["memory"], false), nil
++			return cgroupsystemd.NewUnifiedManager(cgroups, paths["memory"])
+ 		}
+-		return cgroupsystemd.NewLegacyManager(cgroups, paths), nil
++		return cgroupsystemd.NewLegacyManager(cgroups, paths)
+ 	}
+ 	return nil, fmt.Errorf("invalid cgroup manager configuration")
+ }
+@@ -420,7 +421,7 @@ func (m *cgroupManagerImpl) toResources(resourceConfig *ResourceConfig) *libcont
+ 		pageSizes.Insert(sizeString)
+ 	}
+ 	// for each page size omitted, limit to 0
+-	for _, pageSize := range cgroupfs.HugePageSizes {
++	for _, pageSize := range cgroups.HugePageSizes() {
+ 		if pageSizes.Has(pageSize) {
+ 			continue
+ 		}
+diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
+index 3892bae081d..4c79f212ab5 100644
+--- a/pkg/kubelet/cm/container_manager_linux.go
++++ b/pkg/kubelet/cm/container_manager_linux.go
+@@ -401,10 +401,10 @@ func createManager(containerName string) (cgroups.Manager, error) {
+ 	}
+ 
+ 	if cgroups.IsCgroup2UnifiedMode() {
+-		return cgroupfs2.NewManager(cg, "", false)
++		return cgroupfs2.NewManager(cg, "")
+ 
+ 	}
+-	return cgroupfs.NewManager(cg, nil, false), nil
++	return cgroupfs.NewManager(cg, nil)
+ }
+ 
+ type KernelTunableBehavior string
+diff --git a/pkg/kubelet/cm/qos_container_manager_linux.go b/pkg/kubelet/cm/qos_container_manager_linux.go
+index bb79109b141..d0a78db2483 100644
+--- a/pkg/kubelet/cm/qos_container_manager_linux.go
++++ b/pkg/kubelet/cm/qos_container_manager_linux.go
+@@ -28,8 +28,8 @@ import (
+ 	"k8s.io/apimachinery/pkg/util/wait"
+ 
+ 	units "github.com/docker/go-units"
++	"github.com/opencontainers/runc/libcontainer/cgroups"
+ 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
+-	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+ 	v1 "k8s.io/api/core/v1"
+ 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+ 	"k8s.io/kubernetes/pkg/api/v1/resource"
+@@ -147,7 +147,7 @@ func (m *qosContainerManagerImpl) Start(getNodeAllocatable func() v1.ResourceLis
+ // setHugePagesUnbounded ensures hugetlb is effectively unbounded
+ func (m *qosContainerManagerImpl) setHugePagesUnbounded(cgroupConfig *CgroupConfig) error {
+ 	hugePageLimit := map[int64]int64{}
+-	for _, pageSize := range cgroupfs.HugePageSizes {
++	for _, pageSize := range cgroups.HugePageSizes() {
+ 		pageSizeBytes, err := units.RAMInBytes(pageSize)
+ 		if err != nil {
+ 			return err
+diff --git a/pkg/kubelet/dockershim/cm/container_manager_linux.go b/pkg/kubelet/dockershim/cm/container_manager_linux.go
+index 759e27f26c5..93d6c51ac00 100644
+--- a/pkg/kubelet/dockershim/cm/container_manager_linux.go
++++ b/pkg/kubelet/dockershim/cm/container_manager_linux.go
+@@ -129,7 +129,7 @@ func createCgroupManager(name string) (cgroups.Manager, error) {
+ 			SkipDevices: true,
+ 		},
+ 	}
+-	return cgroupfs.NewManager(cg, nil, false), nil
++	return cgroupfs.NewManager(cg, nil)
+ }
+ 
+ // getMemoryCapacity returns the memory capacity on the machine in bytes.
+diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go
+index 6cb9e54729e..fd922f07c7c 100644
+--- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go
++++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux.go
+@@ -23,8 +23,8 @@ import (
+ 	"strconv"
+ 	"time"
+ 
++	"github.com/opencontainers/runc/libcontainer/cgroups"
+ 	libcontainercgroups "github.com/opencontainers/runc/libcontainer/cgroups"
+-	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
+ 	v1 "k8s.io/api/core/v1"
+ 	"k8s.io/apimachinery/pkg/api/resource"
+ 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+@@ -170,7 +170,7 @@ func GetHugepageLimitsFromResources(resources v1.ResourceRequirements) []*runtim
+ 	var hugepageLimits []*runtimeapi.HugepageLimit
+ 
+ 	// For each page size, limit to 0.
+-	for _, pageSize := range cgroupfs.HugePageSizes {
++	for _, pageSize := range cgroups.HugePageSizes() {
+ 		hugepageLimits = append(hugepageLimits, &runtimeapi.HugepageLimit{
+ 			PageSize: pageSize,
+ 			Limit:    uint64(0),
+diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go
+index 46817e00fb0..f166adc6fe1 100644
+--- a/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go
++++ b/pkg/kubelet/kuberuntime/kuberuntime_container_linux_test.go
+@@ -25,7 +25,7 @@ import (
+ 	"testing"
+ 
+ 	"github.com/google/go-cmp/cmp"
+-	cgroupfs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
++	"github.com/opencontainers/runc/libcontainer/cgroups"
+ 	"github.com/stretchr/testify/assert"
+ 	v1 "k8s.io/api/core/v1"
+ 	"k8s.io/apimachinery/pkg/api/resource"
+@@ -366,7 +366,7 @@ func TestGetHugepageLimitsFromResources(t *testing.T) {
+ 	var baseHugepage []*runtimeapi.HugepageLimit
+ 
+ 	// For each page size, limit to 0.
+-	for _, pageSize := range cgroupfs.HugePageSizes {
++	for _, pageSize := range cgroups.HugePageSizes() {
+ 		baseHugepage = append(baseHugepage, &runtimeapi.HugepageLimit{
+ 			PageSize: pageSize,
+ 			Limit:    uint64(0),
+@@ -481,7 +481,7 @@ func TestGetHugepageLimitsFromResources(t *testing.T) {
+ 		machineHugepageSupport := true
+ 		for _, hugepageLimit := range test.expected {
+ 			hugepageSupport := false
+-			for _, pageSize := range cgroupfs.HugePageSizes {
++			for _, pageSize := range cgroups.HugePageSizes() {
+ 				if pageSize == hugepageLimit.PageSize {
+ 					hugepageSupport = true
+ 					break
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/k8s-backport-no-dockershim.patch b/third_party/go/patches/k8s-backport-no-dockershim.patch
new file mode 100644
index 0000000..452c7e1
--- /dev/null
+++ b/third_party/go/patches/k8s-backport-no-dockershim.patch
@@ -0,0 +1,191 @@
+From bc78dff42ec6be929648e91f3ef2dd6dae5169fb Mon Sep 17 00:00:00 2001
+From: Davanum Srinivas <davanum@gmail.com>
+Date: Tue, 7 Dec 2021 14:48:57 -0500
+Subject: [PATCH] update files to drop dockershim
+
+Signed-off-by: Davanum Srinivas <davanum@gmail.com>
+---
+ build/dependencies.yaml                       |  2 --
+ cmd/kubelet/app/options/globalflags_linux.go  | 11 --------
+ go.mod                                        |  5 ----
+ go.sum                                        |  3 ---
+ pkg/kubelet/kubelet.go                        | 26 +------------------
+ .../legacy-cloud-providers/aws/aws_fakes.go   |  1 +
+ test/e2e/framework/.import-restrictions       | 10 -------
+ 7 files changed, 2 insertions(+), 56 deletions(-)
+
+diff --git a/build/dependencies.yaml b/build/dependencies.yaml
+index ff296e255b961..b0b1b5a6d41b8 100644
+--- a/build/dependencies.yaml
++++ b/build/dependencies.yaml
+@@ -172,8 +172,6 @@ dependencies:
+       match: defaultPodSandboxImageVersion\s+=
+     - path: hack/testdata/pod-with-precision.json
+       match: k8s.gcr.io\/pause:\d+\.\d+
+-    - path: pkg/kubelet/dockershim/docker_sandbox.go
+-      match: k8s.gcr.io\/pause:\d+\.\d+
+     - path: staging/src/k8s.io/kubectl/testdata/set/multi-resource-yaml.yaml
+       match: k8s.gcr.io\/pause:\d+\.\d+
+     - path: staging/src/k8s.io/kubectl/testdata/set/namespaced-resource.yaml
+diff --git a/cmd/kubelet/app/options/globalflags_linux.go b/cmd/kubelet/app/options/globalflags_linux.go
+index ad3b68628f661..e75e65ec37cd0 100644
+--- a/cmd/kubelet/app/options/globalflags_linux.go
++++ b/cmd/kubelet/app/options/globalflags_linux.go
+@@ -28,7 +28,6 @@ import (
+ 	// ensure libs have a chance to globally register their flags
+ 	_ "github.com/google/cadvisor/container/common"
+ 	_ "github.com/google/cadvisor/container/containerd"
+-	_ "github.com/google/cadvisor/container/docker"
+ 	_ "github.com/google/cadvisor/container/raw"
+ 	_ "github.com/google/cadvisor/machine"
+ 	_ "github.com/google/cadvisor/manager"
+@@ -41,9 +40,6 @@ func addCadvisorFlags(fs *pflag.FlagSet) {
+ 	global := flag.CommandLine
+ 	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
+ 
+-	// These flags were also implicit from cadvisor, but are actually used by something in the core repo:
+-	// TODO(mtaufen): This one is stil used by our salt, but for heaven's sake it's even deprecated in cadvisor
+-	register(global, local, "docker_root")
+ 	// e2e node tests rely on this
+ 	register(global, local, "housekeeping_interval")
+ 
+@@ -54,13 +50,6 @@ func addCadvisorFlags(fs *pflag.FlagSet) {
+ 	registerDeprecated(global, local, "boot_id_file", deprecated)
+ 	registerDeprecated(global, local, "container_hints", deprecated)
+ 	registerDeprecated(global, local, "containerd", deprecated)
+-	registerDeprecated(global, local, "docker", deprecated)
+-	registerDeprecated(global, local, "docker_env_metadata_whitelist", deprecated)
+-	registerDeprecated(global, local, "docker_only", deprecated)
+-	registerDeprecated(global, local, "docker-tls", deprecated)
+-	registerDeprecated(global, local, "docker-tls-ca", deprecated)
+-	registerDeprecated(global, local, "docker-tls-cert", deprecated)
+-	registerDeprecated(global, local, "docker-tls-key", deprecated)
+ 	registerDeprecated(global, local, "enable_load_reader", deprecated)
+ 	registerDeprecated(global, local, "event_storage_age_limit", deprecated)
+ 	registerDeprecated(global, local, "event_storage_event_limit", deprecated)
+diff --git a/go.mod b/go.mod
+index a17878d68d030..7dccd35bb6b91 100644
+--- a/go.mod
++++ b/go.mod
+@@ -25,15 +25,12 @@ require (
+ 	github.com/boltdb/bolt v1.3.1 // indirect
+ 	github.com/clusterhq/flocker-go v0.0.0-20160920122132-2b8b7259d313
+ 	github.com/container-storage-interface/spec v1.5.0
+-	github.com/containernetworking/cni v0.8.1
+ 	github.com/coredns/corefile-migration v1.0.14
+ 	github.com/coreos/go-oidc v2.1.0+incompatible
+ 	github.com/coreos/go-systemd/v22 v22.3.2
+ 	github.com/cpuguy83/go-md2man/v2 v2.0.0
+ 	github.com/davecgh/go-spew v1.1.1
+ 	github.com/docker/distribution v2.7.1+incompatible
+-	github.com/docker/docker v20.10.7+incompatible
+-	github.com/docker/go-connections v0.4.0
+ 	github.com/docker/go-units v0.4.0
+ 	github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153
+ 	github.com/emicklei/go-restful v2.9.5+incompatible
+@@ -63,7 +60,6 @@ require (
+ 	github.com/mvdan/xurls v1.1.0
+ 	github.com/onsi/ginkgo v1.14.0
+ 	github.com/onsi/gomega v1.10.1
+-	github.com/opencontainers/go-digest v1.0.0
+ 	github.com/opencontainers/runc v1.0.2
+ 	github.com/opencontainers/selinux v1.8.2
+ 	github.com/pkg/errors v0.9.1
+@@ -209,7 +205,6 @@ replace (
+ 	github.com/containerd/go-runc => github.com/containerd/go-runc v1.0.0
+ 	github.com/containerd/ttrpc => github.com/containerd/ttrpc v1.0.2
+ 	github.com/containerd/typeurl => github.com/containerd/typeurl v1.0.2
+-	github.com/containernetworking/cni => github.com/containernetworking/cni v0.8.1
+ 	github.com/coredns/caddy => github.com/coredns/caddy v1.1.0
+ 	github.com/coredns/corefile-migration => github.com/coredns/corefile-migration v1.0.14
+ 	github.com/coreos/go-oidc => github.com/coreos/go-oidc v2.1.0+incompatible
+diff --git a/go.sum b/go.sum
+index b458fb06802b3..9121b2f4ac81a 100644
+--- a/go.sum
++++ b/go.sum
+@@ -116,8 +116,6 @@ github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9
+ github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y=
+ github.com/containerd/typeurl v1.0.2 h1:Chlt8zIieDbzQFzXzAeBEF92KhExuE4p9p92/QmY7aY=
+ github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s=
+-github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI=
+-github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+ github.com/coredns/caddy v1.1.0 h1:ezvsPrT/tA/7pYDBZxu0cT0VmWk75AfIaf6GSYCNMf0=
+ github.com/coredns/caddy v1.1.0/go.mod h1:A6ntJQlAWuQfFlsd9hvigKbo2WS0VUs2l1e2F+BawD4=
+ github.com/coredns/corefile-migration v1.0.14 h1:Tz3WZhoj2NdP8drrQH86NgnCng+VrPjNeg2Oe1ALKag=
+@@ -353,7 +351,6 @@ github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb h1:e+l77LJOEqXTIQih
+ github.com/mohae/deepcopy v0.0.0-20170603005431-491d3605edfb/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8=
+ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/2gBQ3RWajuToeY6ZtZTIKv2v7ThUy5KKusIT0yc0=
+ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4=
+-github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+ github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+ github.com/mrunalp/fileutils v0.5.0 h1:NKzVxiH7eSk+OQ4M+ZYW1K6h27RUV3MI6NUTsHhU6Z4=
+ github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
+index 2013c871a608a..0e6f5f946dea0 100644
+--- a/pkg/kubelet/kubelet.go
++++ b/pkg/kubelet/kubelet.go
+@@ -73,7 +73,6 @@ import (
+ 	"k8s.io/kubernetes/pkg/kubelet/configmap"
+ 	kubecontainer "k8s.io/kubernetes/pkg/kubelet/container"
+ 	"k8s.io/kubernetes/pkg/kubelet/cri/remote"
+-	"k8s.io/kubernetes/pkg/kubelet/cri/streaming"
+ 	"k8s.io/kubernetes/pkg/kubelet/events"
+ 	"k8s.io/kubernetes/pkg/kubelet/eviction"
+ 	"k8s.io/kubernetes/pkg/kubelet/images"
+@@ -310,18 +309,7 @@ func PreInitRuntimeService(kubeCfg *kubeletconfiginternal.KubeletConfiguration,
+ 
+ 	switch containerRuntime {
+ 	case kubetypes.DockerContainerRuntime:
+-		klog.InfoS("Using dockershim is deprecated, please consider using a full-fledged CRI implementation")
+-		if err := runDockershim(
+-			kubeCfg,
+-			kubeDeps,
+-			crOptions,
+-			runtimeCgroups,
+-			remoteRuntimeEndpoint,
+-			remoteImageEndpoint,
+-			nonMasqueradeCIDR,
+-		); err != nil {
+-			return err
+-		}
++		return fmt.Errorf("using dockershim is not supported, please consider using a full-fledged CRI implementation")
+ 	case kubetypes.RemoteContainerRuntime:
+ 		// No-op.
+ 		break
+@@ -2440,15 +2428,3 @@ func isSyncPodWorthy(event *pleg.PodLifecycleEvent) bool {
+ 	// ContainerRemoved doesn't affect pod state
+ 	return event.Type != pleg.ContainerRemoved
+ }
+-
+-// Gets the streaming server configuration to use with in-process CRI shims.
+-func getStreamingConfig(kubeCfg *kubeletconfiginternal.KubeletConfiguration, kubeDeps *Dependencies, crOptions *config.ContainerRuntimeOptions) *streaming.Config {
+-	config := &streaming.Config{
+-		StreamIdleTimeout:               kubeCfg.StreamingConnectionIdleTimeout.Duration,
+-		StreamCreationTimeout:           streaming.DefaultConfig.StreamCreationTimeout,
+-		SupportedRemoteCommandProtocols: streaming.DefaultConfig.SupportedRemoteCommandProtocols,
+-		SupportedPortForwardProtocols:   streaming.DefaultConfig.SupportedPortForwardProtocols,
+-	}
+-	config.Addr = net.JoinHostPort("localhost", "0")
+-	return config
+-}
+diff --git a/test/e2e/framework/.import-restrictions b/test/e2e/framework/.import-restrictions
+index a60fb9d790847..1353f40df9ddf 100644
+--- a/test/e2e/framework/.import-restrictions
++++ b/test/e2e/framework/.import-restrictions
+@@ -86,16 +86,6 @@ rules:
+       - k8s.io/kubernetes/pkg/kubelet/config
+       - k8s.io/kubernetes/pkg/kubelet/configmap
+       - k8s.io/kubernetes/pkg/kubelet/container
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/cm
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/libdocker
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/metrics
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/network
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/network/cni
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/network/hostport
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/network/kubenet
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/network/metrics
+-      - k8s.io/kubernetes/pkg/kubelet/dockershim/remote
+       - k8s.io/kubernetes/pkg/kubelet/envvars
+       - k8s.io/kubernetes/pkg/kubelet/eviction
+       - k8s.io/kubernetes/pkg/kubelet/eviction/api
diff --git a/third_party/go/patches/k8s-client-go.patch b/third_party/go/patches/k8s-client-go.patch
deleted file mode 100644
index ebfae0e..0000000
--- a/third_party/go/patches/k8s-client-go.patch
+++ /dev/null
@@ -1,29 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-This patch rips out azure/openstack auth support for k8s.io/client-go. This should be made configurable upstream.
-
-diff -ur io_k8s_client_go.orig/plugin/pkg/client/auth/plugins.go io_k8s_client_go/plugin/pkg/client/auth/plugins.go
---- io_k8s_client_go.orig/plugin/pkg/client/auth/plugins.go	2020-04-16 17:46:53.965434780 +0200
-+++ io_k8s_client_go/plugin/pkg/client/auth/plugins.go	2020-04-16 17:47:21.720538171 +0200
-@@ -18,8 +18,6 @@
- 
- import (
- 	// Initialize all known client auth plugins.
--	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
- 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
- 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
--	_ "k8s.io/client-go/plugin/pkg/client/auth/openstack"
- )
diff --git a/third_party/go/patches/k8s-drop-legacy-log-path.patch b/third_party/go/patches/k8s-drop-legacy-log-path.patch
index af7b007..356a681 100644
--- a/third_party/go/patches/k8s-drop-legacy-log-path.patch
+++ b/third_party/go/patches/k8s-drop-legacy-log-path.patch
@@ -1,21 +1,6 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 227ccd88e378a002b7c23703eec96aa1d25949eb Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@brun.one>
-Date: Wed, 3 Feb 2021 16:47:38 +0100
+From b2d875981cec8eda9c041f858004f613ea928895 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 16 Mar 2022 18:10:09 +0100
 Subject: [PATCH] Drop legacy log path
 
 ---
@@ -23,14 +8,14 @@
  .../kuberuntime/kuberuntime_container.go      | 32 ---------------
  pkg/kubelet/kuberuntime/kuberuntime_gc.go     | 39 -------------------
  pkg/kubelet/runonce.go                        |  8 ----
- test/e2e_node/log_path_test.go                | 19 +--------
- 5 files changed, 1 insertion(+), 104 deletions(-)
+ test/e2e_node/log_path_test.go                | 18 ---------
+ 5 files changed, 104 deletions(-)
 
 diff --git a/pkg/kubelet/kubelet.go b/pkg/kubelet/kubelet.go
-index 902dc7532e1..2d582f65b19 100644
+index 2013c871a60..c3fd4221544 100644
 --- a/pkg/kubelet/kubelet.go
 +++ b/pkg/kubelet/kubelet.go
-@@ -1250,13 +1250,6 @@ func (kl *Kubelet) initializeModules() error {
+@@ -1379,13 +1379,6 @@ func (kl *Kubelet) initializeModules() error {
  		return err
  	}
  
@@ -45,10 +30,10 @@
  	kl.imageManager.Start()
  
 diff --git a/pkg/kubelet/kuberuntime/kuberuntime_container.go b/pkg/kubelet/kuberuntime/kuberuntime_container.go
-index af361122c35..d5b2d245219 100644
+index 9c762ac309e..d1148cc7de5 100644
 --- a/pkg/kubelet/kuberuntime/kuberuntime_container.go
 +++ b/pkg/kubelet/kuberuntime/kuberuntime_container.go
-@@ -190,25 +190,6 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
+@@ -254,25 +254,6 @@ func (m *kubeGenericRuntimeManager) startContainer(podSandboxID string, podSandb
  	}
  	m.recordContainerEvent(pod, container, containerID, v1.EventTypeNormal, events.StartedContainer, fmt.Sprintf("Started container %s", container.Name))
  
@@ -66,15 +51,15 @@
 -	// to create it in the first place. it happens when journald logging driver is used with docker.
 -	if _, err := m.osInterface.Stat(containerLog); !os.IsNotExist(err) {
 -		if err := m.osInterface.Symlink(containerLog, legacySymlink); err != nil {
--			klog.Errorf("Failed to create legacy symbolic link %q to container %q log %q: %v",
--				legacySymlink, containerID, containerLog, err)
+-			klog.ErrorS(err, "Failed to create legacy symbolic link", "path", legacySymlink,
+-				"containerID", containerID, "containerLogPath", containerLog)
 -		}
 -	}
 -
  	// Step 4: execute the post start hook.
  	if container.Lifecycle != nil && container.Lifecycle.PostStart != nil {
  		kubeContainerID := kubecontainer.ContainerID{
-@@ -861,19 +842,6 @@ func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error
+@@ -967,19 +948,6 @@ func (m *kubeGenericRuntimeManager) removeContainerLog(containerID string) error
  		return err
  	}
  
@@ -95,7 +80,7 @@
  }
  
 diff --git a/pkg/kubelet/kuberuntime/kuberuntime_gc.go b/pkg/kubelet/kuberuntime/kuberuntime_gc.go
-index 8c4f786db9b..b5b104ee6a6 100644
+index 610026661b7..a09dec91fbb 100644
 --- a/pkg/kubelet/kuberuntime/kuberuntime_gc.go
 +++ b/pkg/kubelet/kuberuntime/kuberuntime_gc.go
 @@ -18,7 +18,6 @@ package kuberuntime
@@ -106,7 +91,7 @@
  	"path/filepath"
  	"sort"
  	"time"
-@@ -346,44 +345,6 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {
+@@ -347,44 +346,6 @@ func (cgc *containerGC) evictPodLogsDirectories(allSourcesReady bool) error {
  			}
  		}
  	}
@@ -121,7 +106,7 @@
 -				if err != nil {
 -					// TODO: we should handle container not found (i.e. container was deleted) case differently
 -					// once https://github.com/kubernetes/kubernetes/issues/63336 is resolved
--					klog.Infof("Error getting ContainerStatus for containerID %q: %v", containerID, err)
+-					klog.InfoS("Error getting ContainerStatus for containerID", "containerID", containerID, "err", err)
 -				} else if status.State != runtimeapi.ContainerState_CONTAINER_EXITED {
 -					// Here is how container log rotation works (see containerLogManager#rotateLatestLog):
 -					//
@@ -134,17 +119,17 @@
 -					// See https://github.com/kubernetes/kubernetes/issues/52172
 -					//
 -					// We only remove unhealthy symlink for dead containers
--					klog.V(5).Infof("Container %q is still running, not removing symlink %q.", containerID, logSymlink)
+-					klog.V(5).InfoS("Container is still running, not removing symlink", "containerID", containerID, "path", logSymlink)
 -					continue
 -				}
 -			} else {
--				klog.V(4).Infof("unable to obtain container Id: %v", err)
+-				klog.V(4).InfoS("Unable to obtain container ID", "err", err)
 -			}
 -			err := osInterface.Remove(logSymlink)
 -			if err != nil {
--				klog.Errorf("Failed to remove container log dead symlink %q: %v", logSymlink, err)
+-				klog.ErrorS(err, "Failed to remove container log dead symlink", "path", logSymlink)
 -			} else {
--				klog.V(4).Infof("removed symlink %s", logSymlink)
+-				klog.V(4).InfoS("Removed symlink", "path", logSymlink)
 -			}
 -		}
 -	}
@@ -152,40 +137,36 @@
  }
  
 diff --git a/pkg/kubelet/runonce.go b/pkg/kubelet/runonce.go
-index 1da9c225186..d6a5a63e92d 100644
+index 19b8a4f6a7b..2f0aad713e3 100644
 --- a/pkg/kubelet/runonce.go
 +++ b/pkg/kubelet/runonce.go
-@@ -18,7 +18,6 @@ package kubelet
- 
+@@ -19,7 +19,6 @@ package kubelet
  import (
+ 	"context"
  	"fmt"
 -	"os"
  	"time"
  
- 	"k8s.io/api/core/v1"
-@@ -48,13 +47,6 @@ func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult,
+ 	v1 "k8s.io/api/core/v1"
+@@ -49,13 +48,6 @@ func (kl *Kubelet) RunOnce(updates <-chan kubetypes.PodUpdate) ([]RunPodResult,
  		return nil, err
  	}
  
 -	// If the container logs directory does not exist, create it.
 -	if _, err := os.Stat(ContainerLogsDir); err != nil {
 -		if err := kl.os.MkdirAll(ContainerLogsDir, 0755); err != nil {
--			klog.Errorf("Failed to create directory %q: %v", ContainerLogsDir, err)
+-			klog.ErrorS(err, "Failed to create directory", "path", ContainerLogsDir)
 -		}
 -	}
 -
  	select {
  	case u := <-updates:
- 		klog.Infof("processing manifest with %d pods", len(u.Pods))
+ 		klog.InfoS("Processing manifest with pods", "numPods", len(u.Pods))
 diff --git a/test/e2e_node/log_path_test.go b/test/e2e_node/log_path_test.go
-index 41646f326a5..6568d31e242 100644
+index cfdd9823cb1..35f3b7be465 100644
 --- a/test/e2e_node/log_path_test.go
 +++ b/test/e2e_node/log_path_test.go
-@@ -18,11 +18,10 @@ package e2enode
- 
- import (
- 	"context"
-+
+@@ -22,8 +22,6 @@ import (
  	v1 "k8s.io/api/core/v1"
  	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
  	"k8s.io/apimachinery/pkg/util/uuid"
@@ -194,7 +175,7 @@
  	"k8s.io/kubernetes/test/e2e/framework"
  	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
  	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-@@ -138,22 +137,6 @@ var _ = framework.KubeDescribe("ContainerLogPath [NodeConformance]", func() {
+@@ -144,22 +142,6 @@ var _ = SIGDescribe("ContainerLogPath [NodeConformance]", func() {
  				err := createAndWaitPod(makeLogPod(logPodName, logString))
  				framework.ExpectNoError(err, "Failed waiting for pod: %s to enter success state", logPodName)
  			})
diff --git a/third_party/go/patches/k8s-e2e-tests-providerless.patch b/third_party/go/patches/k8s-e2e-tests-providerless.patch
deleted file mode 100644
index b761257..0000000
--- a/third_party/go/patches/k8s-e2e-tests-providerless.patch
+++ /dev/null
@@ -1,5147 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 65e40a970e3f33f44423653767c9ca8ff792bf70 Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@nexantic.com>
-Date: Mon, 20 Jul 2020 16:50:56 +0200
-Subject: [PATCH] POC Make e2e test suite support providerless
-
----
- .../custom_metrics_stackdriver_autoscaling.go |   2 +
- test/e2e/cloud/imports.go                     |   2 +
- test/e2e/e2e.go                               |  10 -
- test/e2e/e2e_providers.go                     |  32 +
- .../framework/providers/gce/firewall_test.go  |   2 +
- test/e2e/instrumentation/logging/imports.go   |   2 +
- .../instrumentation/monitoring/accelerator.go |   2 +
- .../monitoring/custom_metrics_deployments.go  |   2 +
- .../monitoring/custom_metrics_stackdriver.go  |   4 +-
- .../instrumentation/monitoring/stackdriver.go |   2 +
- .../monitoring/stackdriver_metadata_agent.go  |   4 +-
- test/e2e/network/firewall.go                  |   2 +
- test/e2e/network/ingress.go                   |   2 +
- test/e2e/network/ingress_scale.go             |   2 +
- test/e2e/network/network_tiers.go             |   2 +
- test/e2e/network/scale/ingress.go             |   2 +
- .../network/scale/localrun/ingress_scale.go   |   2 +-
- test/e2e/network/service.go                   | 955 -----------------
- test/e2e/network/service_providers.go         | 980 ++++++++++++++++++
- test/e2e/node/recreate_node.go                |   2 +
- test/e2e/scheduling/nvidia-gpus.go            |   2 +
- test/e2e/scheduling/ubernetes_lite_volumes.go |   2 +
- test/e2e/storage/drivers/in_tree.go           | 732 -------------
- test/e2e/storage/drivers/in_tree_providers.go | 751 ++++++++++++++
- test/e2e/storage/in_tree_volumes.go           |   5 -
- test/e2e/storage/in_tree_volumes_providers.go |  46 +
- .../nfs_persistent_volume-disruptive.go       |   2 +-
- test/e2e/storage/pd.go                        |   2 +
- test/e2e/storage/persistent_volumes-gce.go    |   2 +
- test/e2e/storage/regional_pd.go               |   3 +
- test/e2e/storage/utils/BUILD                  |   3 -
- test/e2e/storage/utils/ebs.go                 |   2 +
- test/e2e/storage/volume_provisioning.go       | 527 ----------
- .../storage/volume_provisioning_providers.go  | 577 +++++++++++
- test/e2e/upgrades/nvidia-gpu.go               |   2 +
- 35 files changed, 2435 insertions(+), 2236 deletions(-)
- create mode 100644 test/e2e/e2e_providers.go
- create mode 100644 test/e2e/network/service_providers.go
- create mode 100644 test/e2e/storage/drivers/in_tree_providers.go
- create mode 100644 test/e2e/storage/in_tree_volumes_providers.go
- create mode 100644 test/e2e/storage/volume_provisioning_providers.go
-
-diff --git a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
-index d3a7862d338..8bacec7fe1d 100644
---- a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
-+++ b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/cloud/imports.go b/test/e2e/cloud/imports.go
-index 5aa1def97d1..382cb1a2264 100644
---- a/test/e2e/cloud/imports.go
-+++ b/test/e2e/cloud/imports.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2019 The Kubernetes Authors.
- 
-diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go
-index d1e23325d69..f5717e417e7 100644
---- a/test/e2e/e2e.go
-+++ b/test/e2e/e2e.go
-@@ -53,16 +53,6 @@ import (
- 	utilnet "k8s.io/utils/net"
- 
- 	clientset "k8s.io/client-go/kubernetes"
--	// ensure auth plugins are loaded
--	_ "k8s.io/client-go/plugin/pkg/client/auth"
--
--	// ensure that cloud providers are loaded
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/openstack"
--	_ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere"
- )
- 
- const (
-diff --git a/test/e2e/e2e_providers.go b/test/e2e/e2e_providers.go
-new file mode 100644
-index 00000000000..cf96642b110
---- /dev/null
-+++ b/test/e2e/e2e_providers.go
-@@ -0,0 +1,32 @@
-+// +build !providerless
-+
-+/*
-+Copyright 2020 The Kubernetes Authors.
-+
-+Licensed under the Apache License, Version 2.0 (the "License");
-+you may not use this file except in compliance with the License.
-+You may obtain a copy of the License at
-+
-+    http://www.apache.org/licenses/LICENSE-2.0
-+
-+Unless required by applicable law or agreed to in writing, software
-+distributed under the License is distributed on an "AS IS" BASIS,
-+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+See the License for the specific language governing permissions and
-+limitations under the License.
-+*/
-+
-+package e2e
-+
-+import (
-+	// ensure auth plugins are loaded
-+	_ "k8s.io/client-go/plugin/pkg/client/auth"
-+
-+	// ensure that cloud providers are loaded
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/openstack"
-+	_ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere"
-+)
-diff --git a/test/e2e/framework/providers/gce/firewall_test.go b/test/e2e/framework/providers/gce/firewall_test.go
-index 647441dc962..2a92543a5a7 100644
---- a/test/e2e/framework/providers/gce/firewall_test.go
-+++ b/test/e2e/framework/providers/gce/firewall_test.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2018 The Kubernetes Authors.
- 
-diff --git a/test/e2e/instrumentation/logging/imports.go b/test/e2e/instrumentation/logging/imports.go
-index 5dd66717db1..fc15c04bfef 100644
---- a/test/e2e/instrumentation/logging/imports.go
-+++ b/test/e2e/instrumentation/logging/imports.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/instrumentation/monitoring/accelerator.go b/test/e2e/instrumentation/monitoring/accelerator.go
-index 90047e46ea1..6fa094e6a18 100644
---- a/test/e2e/instrumentation/monitoring/accelerator.go
-+++ b/test/e2e/instrumentation/monitoring/accelerator.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
-index de80b129315..8d96b93bf11 100644
---- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
-+++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
-index 277b5a0ab24..ddbc3f20802 100644
---- a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
-+++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-@@ -21,7 +23,7 @@ import (
- 	"time"
- 
- 	gcm "google.golang.org/api/monitoring/v3"
--	"k8s.io/api/core/v1"
-+	v1 "k8s.io/api/core/v1"
- 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- 	"k8s.io/apimachinery/pkg/labels"
- 	"k8s.io/apimachinery/pkg/runtime/schema"
-diff --git a/test/e2e/instrumentation/monitoring/stackdriver.go b/test/e2e/instrumentation/monitoring/stackdriver.go
-index dbc5e51c20d..3db0120900b 100644
---- a/test/e2e/instrumentation/monitoring/stackdriver.go
-+++ b/test/e2e/instrumentation/monitoring/stackdriver.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
-index 321591344db..bad9be5b5bf 100644
---- a/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
-+++ b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-@@ -24,7 +26,7 @@ import (
- 	"reflect"
- 	"time"
- 
--	"k8s.io/api/core/v1"
-+	v1 "k8s.io/api/core/v1"
- 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- 	clientset "k8s.io/client-go/kubernetes"
- 	"k8s.io/kubernetes/test/e2e/framework"
-diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go
-index f4200f5a30c..f8612ed75a9 100644
---- a/test/e2e/network/firewall.go
-+++ b/test/e2e/network/firewall.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2016 The Kubernetes Authors.
- 
-diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go
-index 6c3b09e41f2..8485f8ce50e 100644
---- a/test/e2e/network/ingress.go
-+++ b/test/e2e/network/ingress.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2015 The Kubernetes Authors.
- 
-diff --git a/test/e2e/network/ingress_scale.go b/test/e2e/network/ingress_scale.go
-index 6cc8585b7b2..867c834868c 100644
---- a/test/e2e/network/ingress_scale.go
-+++ b/test/e2e/network/ingress_scale.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2018 The Kubernetes Authors.
- 
-diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go
-index 5ae68a5a1ee..f3ea1f72a6b 100644
---- a/test/e2e/network/network_tiers.go
-+++ b/test/e2e/network/network_tiers.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/network/scale/ingress.go b/test/e2e/network/scale/ingress.go
-index 954296beb52..43ad9c9b618 100644
---- a/test/e2e/network/scale/ingress.go
-+++ b/test/e2e/network/scale/ingress.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2018 The Kubernetes Authors.
- 
-diff --git a/test/e2e/network/scale/localrun/ingress_scale.go b/test/e2e/network/scale/localrun/ingress_scale.go
-index 2e2c39884da..5a27f5f4cb2 100644
---- a/test/e2e/network/scale/localrun/ingress_scale.go
-+++ b/test/e2e/network/scale/localrun/ingress_scale.go
-@@ -27,7 +27,7 @@ import (
- 
- 	"k8s.io/klog/v2"
- 
--	"k8s.io/api/core/v1"
-+	v1 "k8s.io/api/core/v1"
- 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- 	clientset "k8s.io/client-go/kubernetes"
- 	"k8s.io/client-go/tools/clientcmd"
-diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go
-index 35ac43001d2..b458347a9f9 100644
---- a/test/e2e/network/service.go
-+++ b/test/e2e/network/service.go
-@@ -31,8 +31,6 @@ import (
- 
- 	utilnet "k8s.io/apimachinery/pkg/util/net"
- 
--	compute "google.golang.org/api/compute/v1"
--
- 	"k8s.io/client-go/tools/cache"
- 
- 	appsv1 "k8s.io/api/apps/v1"
-@@ -52,11 +50,9 @@ import (
- 	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
- 	e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints"
- 	e2eendpointslice "k8s.io/kubernetes/test/e2e/framework/endpointslice"
--	e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem"
- 	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
- 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
- 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
--	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
- 	e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
- 	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
- 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-@@ -64,7 +60,6 @@ import (
- 	"k8s.io/kubernetes/test/e2e/storage/utils"
- 	testutils "k8s.io/kubernetes/test/utils"
- 	imageutils "k8s.io/kubernetes/test/utils/image"
--	gcecloud "k8s.io/legacy-cloud-providers/gce"
- 
- 	"github.com/onsi/ginkgo"
- 	"github.com/onsi/gomega"
-@@ -1242,375 +1237,6 @@ var _ = SIGDescribe("Services", func() {
- 		framework.ExpectNoError(err)
- 	})
- 
--	// TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed.
--	ginkgo.It("should be able to change the type and ports of a service [Slow] [DisabledForLargeClusters]", func() {
--		// requires cloud load-balancer support
--		e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
--
--		loadBalancerSupportsUDP := !framework.ProviderIs("aws")
--
--		loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault
--		if framework.ProviderIs("aws") {
--			loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS
--		}
--		loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
--
--		// This test is more monolithic than we'd like because LB turnup can be
--		// very slow, so we lumped all the tests into one LB lifecycle.
--
--		serviceName := "mutability-test"
--		ns1 := f.Namespace.Name // LB1 in ns1 on TCP
--		framework.Logf("namespace for TCP test: %s", ns1)
--
--		ginkgo.By("creating a second namespace")
--		namespacePtr, err := f.CreateNamespace("services", nil)
--		framework.ExpectNoError(err, "failed to create namespace")
--		ns2 := namespacePtr.Name // LB2 in ns2 on UDP
--		framework.Logf("namespace for UDP test: %s", ns2)
--
--		nodeIP, err := e2enode.PickIP(cs) // for later
--		framework.ExpectNoError(err)
--
--		// Test TCP and UDP Services.  Services with the same name in different
--		// namespaces should get different node ports and load balancers.
--
--		ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
--		tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
--		tcpService, err := tcpJig.CreateTCPService(nil)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2)
--		udpJig := e2eservice.NewTestJig(cs, ns2, serviceName)
--		udpService, err := udpJig.CreateUDPService(nil)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("verifying that TCP and UDP use the same port")
--		if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port {
--			framework.Failf("expected to use the same port for TCP and UDP")
--		}
--		svcPort := int(tcpService.Spec.Ports[0].Port)
--		framework.Logf("service port (TCP and UDP): %d", svcPort)
--
--		ginkgo.By("creating a pod to be part of the TCP service " + serviceName)
--		_, err = tcpJig.Run(nil)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("creating a pod to be part of the UDP service " + serviceName)
--		_, err = udpJig.Run(nil)
--		framework.ExpectNoError(err)
--
--		// Change the services to NodePort.
--
--		ginkgo.By("changing the TCP service to type=NodePort")
--		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Type = v1.ServiceTypeNodePort
--		})
--		framework.ExpectNoError(err)
--		tcpNodePort := int(tcpService.Spec.Ports[0].NodePort)
--		framework.Logf("TCP node port: %d", tcpNodePort)
--
--		ginkgo.By("changing the UDP service to type=NodePort")
--		udpService, err = udpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Type = v1.ServiceTypeNodePort
--		})
--		framework.ExpectNoError(err)
--		udpNodePort := int(udpService.Spec.Ports[0].NodePort)
--		framework.Logf("UDP node port: %d", udpNodePort)
--
--		ginkgo.By("hitting the TCP service's NodePort")
--		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the UDP service's NodePort")
--		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		// Change the services to LoadBalancer.
--
--		// Here we test that LoadBalancers can receive static IP addresses.  This isn't
--		// necessary, but is an additional feature this monolithic test checks.
--		requestedIP := ""
--		staticIPName := ""
--		if framework.ProviderIs("gce", "gke") {
--			ginkgo.By("creating a static load balancer IP")
--			staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID)
--			gceCloud, err := gce.GetGCECloud()
--			framework.ExpectNoError(err, "failed to get GCE cloud provider")
--
--			err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region())
--			defer func() {
--				if staticIPName != "" {
--					// Release GCE static IP - this is not kube-managed and will not be automatically released.
--					if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
--						framework.Logf("failed to release static IP %s: %v", staticIPName, err)
--					}
--				}
--			}()
--			framework.ExpectNoError(err, "failed to create region address: %s", staticIPName)
--			reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region())
--			framework.ExpectNoError(err, "failed to get region address: %s", staticIPName)
--
--			requestedIP = reservedAddr.Address
--			framework.Logf("Allocated static load balancer IP: %s", requestedIP)
--		}
--
--		ginkgo.By("changing the TCP service to type=LoadBalancer")
--		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable
--			s.Spec.Type = v1.ServiceTypeLoadBalancer
--		})
--		framework.ExpectNoError(err)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("changing the UDP service to type=LoadBalancer")
--			udpService, err = udpJig.UpdateService(func(s *v1.Service) {
--				s.Spec.Type = v1.ServiceTypeLoadBalancer
--			})
--			framework.ExpectNoError(err)
--		}
--		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService))
--		if loadBalancerSupportsUDP {
--			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService))
--		}
--
--		ginkgo.By("waiting for the TCP service to have a load balancer")
--		// Wait for the load balancer to be created asynchronously
--		tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
--		framework.ExpectNoError(err)
--		if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
--			framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort)
--		}
--		if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP {
--			framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
--		}
--		tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
--		framework.Logf("TCP load balancer: %s", tcpIngressIP)
--
--		if framework.ProviderIs("gce", "gke") {
--			// Do this as early as possible, which overrides the `defer` above.
--			// This is mostly out of fear of leaking the IP in a timeout case
--			// (as of this writing we're not 100% sure where the leaks are
--			// coming from, so this is first-aid rather than surgery).
--			ginkgo.By("demoting the static IP to ephemeral")
--			if staticIPName != "" {
--				gceCloud, err := gce.GetGCECloud()
--				framework.ExpectNoError(err, "failed to get GCE cloud provider")
--				// Deleting it after it is attached "demotes" it to an
--				// ephemeral IP, which can be auto-released.
--				if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
--					framework.Failf("failed to release static IP %s: %v", staticIPName, err)
--				}
--				staticIPName = ""
--			}
--		}
--
--		var udpIngressIP string
--		if loadBalancerSupportsUDP {
--			ginkgo.By("waiting for the UDP service to have a load balancer")
--			// 2nd one should be faster since they ran in parallel.
--			udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--			if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
--				framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort)
--			}
--			udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])
--			framework.Logf("UDP load balancer: %s", udpIngressIP)
--
--			ginkgo.By("verifying that TCP and UDP use different load balancers")
--			if tcpIngressIP == udpIngressIP {
--				framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
--			}
--		}
--
--		ginkgo.By("hitting the TCP service's NodePort")
--		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the UDP service's NodePort")
--		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the TCP service's LoadBalancer")
--		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("hitting the UDP service's LoadBalancer")
--			testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
--		}
--
--		// Change the services' node ports.
--
--		ginkgo.By("changing the TCP service's NodePort")
--		tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort)
--		framework.ExpectNoError(err)
--		tcpNodePortOld := tcpNodePort
--		tcpNodePort = int(tcpService.Spec.Ports[0].NodePort)
--		if tcpNodePort == tcpNodePortOld {
--			framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort)
--		}
--		if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
--			framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
--		}
--		framework.Logf("TCP node port: %d", tcpNodePort)
--
--		ginkgo.By("changing the UDP service's NodePort")
--		udpService, err = udpJig.ChangeServiceNodePort(udpNodePort)
--		framework.ExpectNoError(err)
--		udpNodePortOld := udpNodePort
--		udpNodePort = int(udpService.Spec.Ports[0].NodePort)
--		if udpNodePort == udpNodePortOld {
--			framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort)
--		}
--		if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
--			framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
--		}
--		framework.Logf("UDP node port: %d", udpNodePort)
--
--		ginkgo.By("hitting the TCP service's new NodePort")
--		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the UDP service's new NodePort")
--		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("checking the old TCP NodePort is closed")
--		testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("checking the old UDP NodePort is closed")
--		testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the TCP service's LoadBalancer")
--		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("hitting the UDP service's LoadBalancer")
--			testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
--		}
--
--		// Change the services' main ports.
--
--		ginkgo.By("changing the TCP service's port")
--		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Ports[0].Port++
--		})
--		framework.ExpectNoError(err)
--		svcPortOld := svcPort
--		svcPort = int(tcpService.Spec.Ports[0].Port)
--		if svcPort == svcPortOld {
--			framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort)
--		}
--		if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
--			framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort)
--		}
--		if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
--			framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
--		}
--
--		ginkgo.By("changing the UDP service's port")
--		udpService, err = udpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Ports[0].Port++
--		})
--		framework.ExpectNoError(err)
--		if int(udpService.Spec.Ports[0].Port) != svcPort {
--			framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port)
--		}
--		if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
--			framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort)
--		}
--		if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
--			framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
--		}
--
--		framework.Logf("service port (TCP and UDP): %d", svcPort)
--
--		ginkgo.By("hitting the TCP service's NodePort")
--		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the UDP service's NodePort")
--		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the TCP service's LoadBalancer")
--		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("hitting the UDP service's LoadBalancer")
--			testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
--		}
--
--		ginkgo.By("Scaling the pods to 0")
--		err = tcpJig.Scale(0)
--		framework.ExpectNoError(err)
--		err = udpJig.Scale(0)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort")
--		testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort")
--		testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer")
--		testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer")
--			testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
--		}
--
--		ginkgo.By("Scaling the pods to 1")
--		err = tcpJig.Scale(1)
--		framework.ExpectNoError(err)
--		err = udpJig.Scale(1)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("hitting the TCP service's NodePort")
--		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the UDP service's NodePort")
--		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("hitting the TCP service's LoadBalancer")
--		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("hitting the UDP service's LoadBalancer")
--			testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
--		}
--
--		// Change the services back to ClusterIP.
--
--		ginkgo.By("changing TCP service back to type=ClusterIP")
--		_, err = tcpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Type = v1.ServiceTypeClusterIP
--			s.Spec.Ports[0].NodePort = 0
--		})
--		framework.ExpectNoError(err)
--		// Wait for the load balancer to be destroyed asynchronously
--		_, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
--		framework.ExpectNoError(err)
--
--		ginkgo.By("changing UDP service back to type=ClusterIP")
--		_, err = udpJig.UpdateService(func(s *v1.Service) {
--			s.Spec.Type = v1.ServiceTypeClusterIP
--			s.Spec.Ports[0].NodePort = 0
--		})
--		framework.ExpectNoError(err)
--		if loadBalancerSupportsUDP {
--			// Wait for the load balancer to be destroyed asynchronously
--			_, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--		}
--
--		ginkgo.By("checking the TCP NodePort is closed")
--		testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("checking the UDP NodePort is closed")
--		testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
--
--		ginkgo.By("checking the TCP LoadBalancer is closed")
--		testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
--
--		if loadBalancerSupportsUDP {
--			ginkgo.By("checking the UDP LoadBalancer is closed")
--			testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
--		}
--	})
--
- 	/*
- 		Testname: Service, update NodePort, same port different protocol
- 		Description: Create a service to accept TCP requests. By default, created service MUST be of type ClusterIP and an ClusterIP MUST be assigned to the service.
-@@ -2253,199 +1879,6 @@ var _ = SIGDescribe("Services", func() {
- 		checkReachabilityFromPod(true, normalReachabilityTimeout, namespace, dropPod.Name, svcIP)
- 	})
- 
--	ginkgo.It("should be able to create an internal type load balancer [Slow]", func() {
--		e2eskipper.SkipUnlessProviderIs("azure", "gke", "gce")
--
--		createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
--		pollInterval := framework.Poll * 10
--
--		namespace := f.Namespace.Name
--		serviceName := "lb-internal"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		ginkgo.By("creating pod to be part of service " + serviceName)
--		_, err := jig.Run(nil)
--		framework.ExpectNoError(err)
--
--		enableILB, disableILB := enableAndDisableInternalLB()
--
--		isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool {
--			ingressEndpoint := e2eservice.GetIngressPoint(lbIngress)
--			// Needs update for providers using hostname as endpoint.
--			return strings.HasPrefix(ingressEndpoint, "10.")
--		}
--
--		ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled")
--		svc, err := jig.CreateTCPService(func(svc *v1.Service) {
--			svc.Spec.Type = v1.ServiceTypeLoadBalancer
--			enableILB(svc)
--		})
--		framework.ExpectNoError(err)
--
--		defer func() {
--			ginkgo.By("Clean up loadbalancer service")
--			e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
--		}()
--
--		svc, err = jig.WaitForLoadBalancer(createTimeout)
--		framework.ExpectNoError(err)
--		lbIngress := &svc.Status.LoadBalancer.Ingress[0]
--		svcPort := int(svc.Spec.Ports[0].Port)
--		// should have an internal IP.
--		framework.ExpectEqual(isInternalEndpoint(lbIngress), true)
--
--		// ILBs are not accessible from the test orchestrator, so it's necessary to use
--		//  a pod to test the service.
--		ginkgo.By("hitting the internal load balancer from pod")
--		framework.Logf("creating pod with host network")
--		hostExec := launchHostExecPod(f.ClientSet, f.Namespace.Name, "ilb-host-exec")
--
--		framework.Logf("Waiting up to %v for service %q's internal LB to respond to requests", createTimeout, serviceName)
--		tcpIngressIP := e2eservice.GetIngressPoint(lbIngress)
--		if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
--			cmd := fmt.Sprintf(`curl -m 5 'http://%v:%v/echo?msg=hello'`, tcpIngressIP, svcPort)
--			stdout, err := framework.RunHostCmd(hostExec.Namespace, hostExec.Name, cmd)
--			if err != nil {
--				framework.Logf("error curling; stdout: %v. err: %v", stdout, err)
--				return false, nil
--			}
--
--			if !strings.Contains(stdout, "hello") {
--				framework.Logf("Expected output to contain 'hello', got %q; retrying...", stdout)
--				return false, nil
--			}
--
--			framework.Logf("Successful curl; stdout: %v", stdout)
--			return true, nil
--		}); pollErr != nil {
--			framework.Failf("ginkgo.Failed to hit ILB IP, err: %v", pollErr)
--		}
--
--		ginkgo.By("switching to external type LoadBalancer")
--		svc, err = jig.UpdateService(func(svc *v1.Service) {
--			disableILB(svc)
--		})
--		framework.ExpectNoError(err)
--		framework.Logf("Waiting up to %v for service %q to have an external LoadBalancer", createTimeout, serviceName)
--		if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
--			svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
--			if err != nil {
--				return false, err
--			}
--			lbIngress = &svc.Status.LoadBalancer.Ingress[0]
--			return !isInternalEndpoint(lbIngress), nil
--		}); pollErr != nil {
--			framework.Failf("Loadbalancer IP not changed to external.")
--		}
--		// should have an external IP.
--		gomega.Expect(isInternalEndpoint(lbIngress)).To(gomega.BeFalse())
--
--		ginkgo.By("hitting the external load balancer")
--		framework.Logf("Waiting up to %v for service %q's external LB to respond to requests", createTimeout, serviceName)
--		tcpIngressIP = e2eservice.GetIngressPoint(lbIngress)
--		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
--
--		// GCE cannot test a specific IP because the test may not own it. This cloud specific condition
--		// will be removed when GCP supports similar functionality.
--		if framework.ProviderIs("azure") {
--			ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.")
--			internalStaticIP := "10.240.11.11"
--			svc, err = jig.UpdateService(func(svc *v1.Service) {
--				svc.Spec.LoadBalancerIP = internalStaticIP
--				enableILB(svc)
--			})
--			framework.ExpectNoError(err)
--			framework.Logf("Waiting up to %v for service %q to have an internal LoadBalancer", createTimeout, serviceName)
--			if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
--				svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
--				if err != nil {
--					return false, err
--				}
--				lbIngress = &svc.Status.LoadBalancer.Ingress[0]
--				return isInternalEndpoint(lbIngress), nil
--			}); pollErr != nil {
--				framework.Failf("Loadbalancer IP not changed to internal.")
--			}
--			// should have the given static internal IP.
--			framework.ExpectEqual(e2eservice.GetIngressPoint(lbIngress), internalStaticIP)
--		}
--	})
--
--	// This test creates a load balancer, make sure its health check interval
--	// equals to gceHcCheckIntervalSeconds. Then the interval is manipulated
--	// to be something else, see if the interval will be reconciled.
--	ginkgo.It("should reconcile LB health check interval [Slow][Serial]", func() {
--		const gceHcCheckIntervalSeconds = int64(8)
--		// This test is for clusters on GCE.
--		// (It restarts kube-controller-manager, which we don't support on GKE)
--		e2eskipper.SkipUnlessProviderIs("gce")
--		e2eskipper.SkipUnlessSSHKeyPresent()
--
--		clusterID, err := gce.GetClusterID(cs)
--		if err != nil {
--			framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err)
--		}
--		gceCloud, err := gce.GetGCECloud()
--		if err != nil {
--			framework.Failf("framework.GetGCECloud() = _, %v; want nil", err)
--		}
--
--		namespace := f.Namespace.Name
--		serviceName := "lb-hc-int"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		ginkgo.By("create load balancer service")
--		// Create loadbalancer service with source range from node[0] and podAccept
--		svc, err := jig.CreateTCPService(func(svc *v1.Service) {
--			svc.Spec.Type = v1.ServiceTypeLoadBalancer
--		})
--		framework.ExpectNoError(err)
--
--		defer func() {
--			ginkgo.By("Clean up loadbalancer service")
--			e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
--		}()
--
--		svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
--		framework.ExpectNoError(err)
--
--		hcName := gcecloud.MakeNodesHealthCheckName(clusterID)
--		hc, err := gceCloud.GetHTTPHealthCheck(hcName)
--		if err != nil {
--			framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err)
--		}
--		framework.ExpectEqual(hc.CheckIntervalSec, gceHcCheckIntervalSeconds)
--
--		ginkgo.By("modify the health check interval")
--		hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1
--		if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil {
--			framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err)
--		}
--
--		ginkgo.By("restart kube-controller-manager")
--		if err := e2ekubesystem.RestartControllerManager(); err != nil {
--			framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err)
--		}
--		if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil {
--			framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err)
--		}
--
--		ginkgo.By("health check should be reconciled")
--		pollInterval := framework.Poll * 10
--		loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
--		if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) {
--			hc, err := gceCloud.GetHTTPHealthCheck(hcName)
--			if err != nil {
--				framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err)
--				return false, err
--			}
--			framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec)
--			return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil
--		}); pollErr != nil {
--			framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds)
--		}
--	})
--
- 	/*
- 		Release: v1.19
- 		Testname: Service, ClusterIP type, session affinity to ClientIP
-@@ -2999,394 +2432,6 @@ var _ = SIGDescribe("Services", func() {
- 	})
- })
- 
--var _ = SIGDescribe("ESIPP [Slow]", func() {
--	f := framework.NewDefaultFramework("esipp")
--	var loadBalancerCreateTimeout time.Duration
--
--	var cs clientset.Interface
--	serviceLBNames := []string{}
--
--	ginkgo.BeforeEach(func() {
--		// requires cloud load-balancer support - this feature currently supported only on GCE/GKE
--		e2eskipper.SkipUnlessProviderIs("gce", "gke")
--
--		cs = f.ClientSet
--		loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
--	})
--
--	ginkgo.AfterEach(func() {
--		if ginkgo.CurrentGinkgoTestDescription().Failed {
--			DescribeSvc(f.Namespace.Name)
--		}
--		for _, lb := range serviceLBNames {
--			framework.Logf("cleaning load balancer resource for %s", lb)
--			e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
--		}
--		//reset serviceLBNames
--		serviceLBNames = []string{}
--	})
--
--	ginkgo.It("should work for type=LoadBalancer", func() {
--		namespace := f.Namespace.Name
--		serviceName := "external-local-lb"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
--		framework.ExpectNoError(err)
--		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
--		healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
--		if healthCheckNodePort == 0 {
--			framework.Failf("Service HealthCheck NodePort was not allocated")
--		}
--		defer func() {
--			err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--
--			// Make sure we didn't leak the health check node port.
--			const threshold = 2
--			nodes, err := getEndpointNodesWithInternalIP(jig)
--			framework.ExpectNoError(err)
--			config := e2enetwork.NewNetworkingTestConfig(f, false, false)
--			for _, internalIP := range nodes {
--				err := testHTTPHealthCheckNodePortFromTestContainer(
--					config,
--					internalIP,
--					healthCheckNodePort,
--					e2eservice.KubeProxyLagTimeout,
--					false,
--					threshold)
--				framework.ExpectNoError(err)
--			}
--			err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err)
--		}()
--
--		svcTCPPort := int(svc.Spec.Ports[0].Port)
--		ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
--
--		ginkgo.By("reading clientIP using the TCP service's service port via its external VIP")
--		content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
--		clientIP := content.String()
--		framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP)
--
--		ginkgo.By("checking if Source IP is preserved")
--		if strings.HasPrefix(clientIP, "10.") {
--			framework.Failf("Source IP was NOT preserved")
--		}
--	})
--
--	ginkgo.It("should work for type=NodePort", func() {
--		namespace := f.Namespace.Name
--		serviceName := "external-local-nodeport"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		svc, err := jig.CreateOnlyLocalNodePortService(true)
--		framework.ExpectNoError(err)
--		defer func() {
--			err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err)
--		}()
--
--		tcpNodePort := int(svc.Spec.Ports[0].NodePort)
--
--		endpointsNodeMap, err := getEndpointNodesWithInternalIP(jig)
--		framework.ExpectNoError(err)
--
--		dialCmd := "clientip"
--		config := e2enetwork.NewNetworkingTestConfig(f, false, false)
--
--		for nodeName, nodeIP := range endpointsNodeMap {
--			ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v:%v/%v", nodeName, nodeIP, tcpNodePort, dialCmd))
--			clientIP, err := GetHTTPContentFromTestContainer(config, nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, dialCmd)
--			framework.ExpectNoError(err)
--			framework.Logf("ClientIP detected by target pod using NodePort is %s, the ip of test container is %s", clientIP, config.TestContainerPod.Status.PodIP)
--			// the clientIP returned by agnhost contains port
--			if !strings.HasPrefix(clientIP, config.TestContainerPod.Status.PodIP) {
--				framework.Failf("Source IP was NOT preserved")
--			}
--		}
--	})
--
--	ginkgo.It("should only target nodes with endpoints", func() {
--		namespace := f.Namespace.Name
--		serviceName := "external-local-nodes"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--		nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
--		framework.ExpectNoError(err)
--
--		svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false,
--			func(svc *v1.Service) {
--				// Change service port to avoid collision with opened hostPorts
--				// in other tests that run in parallel.
--				if len(svc.Spec.Ports) != 0 {
--					svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port))
--					svc.Spec.Ports[0].Port = 8081
--				}
--
--			})
--		framework.ExpectNoError(err)
--		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
--		defer func() {
--			err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--			err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err)
--		}()
--
--		healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
--		if healthCheckNodePort == 0 {
--			framework.Failf("Service HealthCheck NodePort was not allocated")
--		}
--
--		ips := e2enode.CollectAddresses(nodes, v1.NodeInternalIP)
--
--		ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
--		svcTCPPort := int(svc.Spec.Ports[0].Port)
--
--		const threshold = 2
--		config := e2enetwork.NewNetworkingTestConfig(f, false, false)
--		for i := 0; i < len(nodes.Items); i++ {
--			endpointNodeName := nodes.Items[i].Name
--
--			ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName)
--			_, err = jig.Run(func(rc *v1.ReplicationController) {
--				rc.Name = serviceName
--				if endpointNodeName != "" {
--					rc.Spec.Template.Spec.NodeName = endpointNodeName
--				}
--			})
--			framework.ExpectNoError(err)
--
--			ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName))
--			err = jig.WaitForEndpointOnNode(endpointNodeName)
--			framework.ExpectNoError(err)
--
--			// HealthCheck should pass only on the node where num(endpoints) > 0
--			// All other nodes should fail the healthcheck on the service healthCheckNodePort
--			for n, internalIP := range ips {
--				// Make sure the loadbalancer picked up the health check change.
--				// Confirm traffic can reach backend through LB before checking healthcheck nodeport.
--				e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout)
--				expectedSuccess := nodes.Items[n].Name == endpointNodeName
--				port := strconv.Itoa(healthCheckNodePort)
--				ipPort := net.JoinHostPort(internalIP, port)
--				framework.Logf("Health checking %s, http://%s/healthz, expectedSuccess %v", nodes.Items[n].Name, ipPort, expectedSuccess)
--				err := testHTTPHealthCheckNodePortFromTestContainer(
--					config,
--					internalIP,
--					healthCheckNodePort,
--					e2eservice.KubeProxyEndpointLagTimeout,
--					expectedSuccess,
--					threshold)
--				framework.ExpectNoError(err)
--			}
--			framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName))
--		}
--	})
--
--	ginkgo.It("should work from pods", func() {
--		var err error
--		namespace := f.Namespace.Name
--		serviceName := "external-local-pods"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
--		framework.ExpectNoError(err)
--		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
--		defer func() {
--			err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--			err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err)
--		}()
--
--		ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
--		port := strconv.Itoa(int(svc.Spec.Ports[0].Port))
--		ipPort := net.JoinHostPort(ingressIP, port)
--		path := fmt.Sprintf("%s/clientip", ipPort)
--
--		ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state")
--		deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1)
--		framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment")
--
--		defer func() {
--			framework.Logf("Deleting deployment")
--			err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name)
--		}()
--
--		deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
--		framework.ExpectNoError(err, "Error in retrieving pause pod deployment")
--		labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
--		framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment")
--
--		pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()})
--		framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments")
--
--		pausePod := pausePods.Items[0]
--		framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path)
--		cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path)
--
--		var srcIP string
--		loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
--		ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName))
--		if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) {
--			stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd)
--			if err != nil {
--				framework.Logf("got err: %v, retry until timeout", err)
--				return false, nil
--			}
--			srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0])
--			return srcIP == pausePod.Status.PodIP, nil
--		}); pollErr != nil {
--			framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP)
--		}
--	})
--
--	ginkgo.It("should handle updates to ExternalTrafficPolicy field", func() {
--		namespace := f.Namespace.Name
--		serviceName := "external-local-update"
--		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
--
--		nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
--		framework.ExpectNoError(err)
--		if len(nodes.Items) < 2 {
--			framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint")
--		}
--
--		svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
--		framework.ExpectNoError(err)
--		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
--		defer func() {
--			err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
--			framework.ExpectNoError(err)
--			err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
--			framework.ExpectNoError(err)
--		}()
--
--		// save the health check node port because it disappears when ESIPP is turned off.
--		healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
--
--		ginkgo.By("turning ESIPP off")
--		svc, err = jig.UpdateService(func(svc *v1.Service) {
--			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
--		})
--		framework.ExpectNoError(err)
--		if svc.Spec.HealthCheckNodePort > 0 {
--			framework.Failf("Service HealthCheck NodePort still present")
--		}
--
--		epNodes, err := jig.ListNodesWithEndpoint()
--		framework.ExpectNoError(err)
--		// map from name of nodes with endpoint to internal ip
--		// it is assumed that there is only a single node with the endpoint
--		endpointNodeMap := make(map[string]string)
--		// map from name of nodes without endpoint to internal ip
--		noEndpointNodeMap := make(map[string]string)
--		for _, node := range epNodes {
--			ips := e2enode.GetAddresses(&node, v1.NodeInternalIP)
--			if len(ips) < 1 {
--				framework.Failf("No internal ip found for node %s", node.Name)
--			}
--			endpointNodeMap[node.Name] = ips[0]
--		}
--		for _, n := range nodes.Items {
--			ips := e2enode.GetAddresses(&n, v1.NodeInternalIP)
--			if len(ips) < 1 {
--				framework.Failf("No internal ip found for node %s", n.Name)
--			}
--			if _, ok := endpointNodeMap[n.Name]; !ok {
--				noEndpointNodeMap[n.Name] = ips[0]
--			}
--		}
--		framework.ExpectNotEqual(len(endpointNodeMap), 0)
--		framework.ExpectNotEqual(len(noEndpointNodeMap), 0)
--
--		svcTCPPort := int(svc.Spec.Ports[0].Port)
--		svcNodePort := int(svc.Spec.Ports[0].NodePort)
--		ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
--		path := "/clientip"
--		dialCmd := "clientip"
--
--		config := e2enetwork.NewNetworkingTestConfig(f, false, false)
--
--		ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap))
--		for nodeName, nodeIP := range noEndpointNodeMap {
--			ginkgo.By(fmt.Sprintf("Checking %v (%v:%v/%v) proxies to endpoints on another node", nodeName, nodeIP[0], svcNodePort, dialCmd))
--			_, err := GetHTTPContentFromTestContainer(config, nodeIP, svcNodePort, e2eservice.KubeProxyLagTimeout, dialCmd)
--			framework.ExpectNoError(err, "Could not reach HTTP service through %v:%v/%v after %v", nodeIP, svcNodePort, dialCmd, e2eservice.KubeProxyLagTimeout)
--		}
--
--		for nodeName, nodeIP := range endpointNodeMap {
--			ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIP))
--			var body string
--			pollFn := func() (bool, error) {
--				// we expect connection failure here, but not other errors
--				resp, err := config.GetResponseFromTestContainer(
--					"http",
--					"healthz",
--					nodeIP,
--					healthCheckNodePort)
--				if err != nil {
--					return false, nil
--				}
--				if len(resp.Errors) > 0 {
--					return true, nil
--				}
--				if len(resp.Responses) > 0 {
--					body = resp.Responses[0]
--				}
--				return false, nil
--			}
--			if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollFn); pollErr != nil {
--				framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s",
--					nodeName, healthCheckNodePort, body)
--			}
--		}
--
--		// Poll till kube-proxy re-adds the MASQUERADE rule on the node.
--		ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP))
--		var clientIP string
--		pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
--			content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
--			clientIP = content.String()
--			if strings.HasPrefix(clientIP, "10.") {
--				return true, nil
--			}
--			return false, nil
--		})
--		if pollErr != nil {
--			framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP)
--		}
--
--		// TODO: We need to attempt to create another service with the previously
--		// allocated healthcheck nodePort. If the health check nodePort has been
--		// freed, the new service creation will succeed, upon which we cleanup.
--		// If the health check nodePort has NOT been freed, the new service
--		// creation will fail.
--
--		ginkgo.By("setting ExternalTraffic field back to OnlyLocal")
--		svc, err = jig.UpdateService(func(svc *v1.Service) {
--			svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
--			// Request the same healthCheckNodePort as before, to test the user-requested allocation path
--			svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
--		})
--		framework.ExpectNoError(err)
--		pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
--			content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path)
--			clientIP = content.String()
--			ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP))
--			if !strings.HasPrefix(clientIP, "10.") {
--				return true, nil
--			}
--			return false, nil
--		})
--		if pollErr != nil {
--			framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP)
--		}
--	})
--})
--
- // execAffinityTestForSessionAffinityTimeout is a helper function that wrap the logic of
- // affinity test for non-load-balancer services. Session afinity will be
- // enabled when the service is created and a short timeout will be configured so
-diff --git a/test/e2e/network/service_providers.go b/test/e2e/network/service_providers.go
-new file mode 100644
-index 00000000000..b7eae6feb2c
---- /dev/null
-+++ b/test/e2e/network/service_providers.go
-@@ -0,0 +1,980 @@
-+// +build !providerless
-+
-+/*
-+Copyright 2020 The Kubernetes Authors.
-+
-+Licensed under the Apache License, Version 2.0 (the "License");
-+you may not use this file except in compliance with the License.
-+You may obtain a copy of the License at
-+
-+    http://www.apache.org/licenses/LICENSE-2.0
-+
-+Unless required by applicable law or agreed to in writing, software
-+distributed under the License is distributed on an "AS IS" BASIS,
-+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+See the License for the specific language governing permissions and
-+limitations under the License.
-+*/
-+
-+package network
-+
-+import (
-+	"bytes"
-+	"context"
-+	"fmt"
-+	"net"
-+	"strconv"
-+	"strings"
-+	"time"
-+
-+	compute "google.golang.org/api/compute/v1"
-+	v1 "k8s.io/api/core/v1"
-+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-+	"k8s.io/apimachinery/pkg/util/intstr"
-+	"k8s.io/apimachinery/pkg/util/wait"
-+	clientset "k8s.io/client-go/kubernetes"
-+	cloudprovider "k8s.io/cloud-provider"
-+	"k8s.io/kubernetes/test/e2e/framework"
-+	e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
-+	e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem"
-+	e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
-+	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
-+	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
-+	e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
-+	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
-+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-+	gcecloud "k8s.io/legacy-cloud-providers/gce"
-+
-+	"github.com/onsi/ginkgo"
-+	"github.com/onsi/gomega"
-+)
-+
-+var _ = SIGDescribe("Services with Cloud LoadBalancers", func() {
-+
-+	f := framework.NewDefaultFramework("services")
-+
-+	var cs clientset.Interface
-+	serviceLBNames := []string{}
-+
-+	ginkgo.BeforeEach(func() {
-+		cs = f.ClientSet
-+	})
-+
-+	ginkgo.AfterEach(func() {
-+		if ginkgo.CurrentGinkgoTestDescription().Failed {
-+			DescribeSvc(f.Namespace.Name)
-+		}
-+		for _, lb := range serviceLBNames {
-+			framework.Logf("cleaning load balancer resource for %s", lb)
-+			e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
-+		}
-+		//reset serviceLBNames
-+		serviceLBNames = []string{}
-+	})
-+
-+	// TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed
-+	ginkgo.It("should be able to change the type and ports of a service [Slow] [DisabledForLargeClusters]", func() {
-+		// requires cloud load-balancer support
-+		e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
-+
-+		loadBalancerSupportsUDP := !framework.ProviderIs("aws")
-+
-+		loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault
-+		if framework.ProviderIs("aws") {
-+			loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS
-+		}
-+		loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
-+
-+		// This test is more monolithic than we'd like because LB turnup can be
-+		// very slow, so we lumped all the tests into one LB lifecycle.
-+
-+		serviceName := "mutability-test"
-+		ns1 := f.Namespace.Name // LB1 in ns1 on TCP
-+		framework.Logf("namespace for TCP test: %s", ns1)
-+
-+		ginkgo.By("creating a second namespace")
-+		namespacePtr, err := f.CreateNamespace("services", nil)
-+		framework.ExpectNoError(err, "failed to create namespace")
-+		ns2 := namespacePtr.Name // LB2 in ns2 on UDP
-+		framework.Logf("namespace for UDP test: %s", ns2)
-+
-+		nodeIP, err := e2enode.PickIP(cs) // for later
-+		framework.ExpectNoError(err)
-+
-+		// Test TCP and UDP Services.  Services with the same name in different
-+		// namespaces should get different node ports and load balancers.
-+
-+		ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
-+		tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
-+		tcpService, err := tcpJig.CreateTCPService(nil)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2)
-+		udpJig := e2eservice.NewTestJig(cs, ns2, serviceName)
-+		udpService, err := udpJig.CreateUDPService(nil)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("verifying that TCP and UDP use the same port")
-+		if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port {
-+			framework.Failf("expected to use the same port for TCP and UDP")
-+		}
-+		svcPort := int(tcpService.Spec.Ports[0].Port)
-+		framework.Logf("service port (TCP and UDP): %d", svcPort)
-+
-+		ginkgo.By("creating a pod to be part of the TCP service " + serviceName)
-+		_, err = tcpJig.Run(nil)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("creating a pod to be part of the UDP service " + serviceName)
-+		_, err = udpJig.Run(nil)
-+		framework.ExpectNoError(err)
-+
-+		// Change the services to NodePort.
-+
-+		ginkgo.By("changing the TCP service to type=NodePort")
-+		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Type = v1.ServiceTypeNodePort
-+		})
-+		framework.ExpectNoError(err)
-+		tcpNodePort := int(tcpService.Spec.Ports[0].NodePort)
-+		framework.Logf("TCP node port: %d", tcpNodePort)
-+
-+		ginkgo.By("changing the UDP service to type=NodePort")
-+		udpService, err = udpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Type = v1.ServiceTypeNodePort
-+		})
-+		framework.ExpectNoError(err)
-+		udpNodePort := int(udpService.Spec.Ports[0].NodePort)
-+		framework.Logf("UDP node port: %d", udpNodePort)
-+
-+		ginkgo.By("hitting the TCP service's NodePort")
-+		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the UDP service's NodePort")
-+		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		// Change the services to LoadBalancer.
-+
-+		// Here we test that LoadBalancers can receive static IP addresses.  This isn't
-+		// necessary, but is an additional feature this monolithic test checks.
-+		requestedIP := ""
-+		staticIPName := ""
-+		if framework.ProviderIs("gce", "gke") {
-+			ginkgo.By("creating a static load balancer IP")
-+			staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID)
-+			gceCloud, err := gce.GetGCECloud()
-+			framework.ExpectNoError(err, "failed to get GCE cloud provider")
-+
-+			err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region())
-+			defer func() {
-+				if staticIPName != "" {
-+					// Release GCE static IP - this is not kube-managed and will not be automatically released.
-+					if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
-+						framework.Logf("failed to release static IP %s: %v", staticIPName, err)
-+					}
-+				}
-+			}()
-+			framework.ExpectNoError(err, "failed to create region address: %s", staticIPName)
-+			reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region())
-+			framework.ExpectNoError(err, "failed to get region address: %s", staticIPName)
-+
-+			requestedIP = reservedAddr.Address
-+			framework.Logf("Allocated static load balancer IP: %s", requestedIP)
-+		}
-+
-+		ginkgo.By("changing the TCP service to type=LoadBalancer")
-+		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable
-+			s.Spec.Type = v1.ServiceTypeLoadBalancer
-+		})
-+		framework.ExpectNoError(err)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("changing the UDP service to type=LoadBalancer")
-+			udpService, err = udpJig.UpdateService(func(s *v1.Service) {
-+				s.Spec.Type = v1.ServiceTypeLoadBalancer
-+			})
-+			framework.ExpectNoError(err)
-+		}
-+		serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService))
-+		if loadBalancerSupportsUDP {
-+			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService))
-+		}
-+
-+		ginkgo.By("waiting for the TCP service to have a load balancer")
-+		// Wait for the load balancer to be created asynchronously
-+		tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
-+		framework.ExpectNoError(err)
-+		if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
-+			framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort)
-+		}
-+		if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP {
-+			framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
-+		}
-+		tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
-+		framework.Logf("TCP load balancer: %s", tcpIngressIP)
-+
-+		if framework.ProviderIs("gce", "gke") {
-+			// Do this as early as possible, which overrides the `defer` above.
-+			// This is mostly out of fear of leaking the IP in a timeout case
-+			// (as of this writing we're not 100% sure where the leaks are
-+			// coming from, so this is first-aid rather than surgery).
-+			ginkgo.By("demoting the static IP to ephemeral")
-+			if staticIPName != "" {
-+				gceCloud, err := gce.GetGCECloud()
-+				framework.ExpectNoError(err, "failed to get GCE cloud provider")
-+				// Deleting it after it is attached "demotes" it to an
-+				// ephemeral IP, which can be auto-released.
-+				if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
-+					framework.Failf("failed to release static IP %s: %v", staticIPName, err)
-+				}
-+				staticIPName = ""
-+			}
-+		}
-+
-+		var udpIngressIP string
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("waiting for the UDP service to have a load balancer")
-+			// 2nd one should be faster since they ran in parallel.
-+			udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
-+			framework.ExpectNoError(err)
-+			if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
-+				framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort)
-+			}
-+			udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])
-+			framework.Logf("UDP load balancer: %s", udpIngressIP)
-+
-+			ginkgo.By("verifying that TCP and UDP use different load balancers")
-+			if tcpIngressIP == udpIngressIP {
-+				framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
-+			}
-+		}
-+
-+		ginkgo.By("hitting the TCP service's NodePort")
-+		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the UDP service's NodePort")
-+		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the TCP service's LoadBalancer")
-+		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("hitting the UDP service's LoadBalancer")
-+			testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
-+		}
-+
-+		// Change the services' node ports.
-+
-+		ginkgo.By("changing the TCP service's NodePort")
-+		tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort)
-+		framework.ExpectNoError(err)
-+		tcpNodePortOld := tcpNodePort
-+		tcpNodePort = int(tcpService.Spec.Ports[0].NodePort)
-+		if tcpNodePort == tcpNodePortOld {
-+			framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort)
-+		}
-+		if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
-+			framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
-+		}
-+		framework.Logf("TCP node port: %d", tcpNodePort)
-+
-+		ginkgo.By("changing the UDP service's NodePort")
-+		udpService, err = udpJig.ChangeServiceNodePort(udpNodePort)
-+		framework.ExpectNoError(err)
-+		udpNodePortOld := udpNodePort
-+		udpNodePort = int(udpService.Spec.Ports[0].NodePort)
-+		if udpNodePort == udpNodePortOld {
-+			framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort)
-+		}
-+		if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
-+			framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
-+		}
-+		framework.Logf("UDP node port: %d", udpNodePort)
-+
-+		ginkgo.By("hitting the TCP service's new NodePort")
-+		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the UDP service's new NodePort")
-+		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("checking the old TCP NodePort is closed")
-+		testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("checking the old UDP NodePort is closed")
-+		testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the TCP service's LoadBalancer")
-+		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("hitting the UDP service's LoadBalancer")
-+			testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
-+		}
-+
-+		// Change the services' main ports.
-+
-+		ginkgo.By("changing the TCP service's port")
-+		tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Ports[0].Port++
-+		})
-+		framework.ExpectNoError(err)
-+		svcPortOld := svcPort
-+		svcPort = int(tcpService.Spec.Ports[0].Port)
-+		if svcPort == svcPortOld {
-+			framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort)
-+		}
-+		if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
-+			framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort)
-+		}
-+		if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
-+			framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
-+		}
-+
-+		ginkgo.By("changing the UDP service's port")
-+		udpService, err = udpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Ports[0].Port++
-+		})
-+		framework.ExpectNoError(err)
-+		if int(udpService.Spec.Ports[0].Port) != svcPort {
-+			framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port)
-+		}
-+		if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
-+			framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort)
-+		}
-+		if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
-+			framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
-+		}
-+
-+		framework.Logf("service port (TCP and UDP): %d", svcPort)
-+
-+		ginkgo.By("hitting the TCP service's NodePort")
-+		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the UDP service's NodePort")
-+		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the TCP service's LoadBalancer")
-+		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("hitting the UDP service's LoadBalancer")
-+			testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
-+		}
-+
-+		ginkgo.By("Scaling the pods to 0")
-+		err = tcpJig.Scale(0)
-+		framework.ExpectNoError(err)
-+		err = udpJig.Scale(0)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort")
-+		testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort")
-+		testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer")
-+		testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer")
-+			testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
-+		}
-+
-+		ginkgo.By("Scaling the pods to 1")
-+		err = tcpJig.Scale(1)
-+		framework.ExpectNoError(err)
-+		err = udpJig.Scale(1)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("hitting the TCP service's NodePort")
-+		e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the UDP service's NodePort")
-+		testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("hitting the TCP service's LoadBalancer")
-+		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("hitting the UDP service's LoadBalancer")
-+			testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
-+		}
-+
-+		// Change the services back to ClusterIP.
-+
-+		ginkgo.By("changing TCP service back to type=ClusterIP")
-+		_, err = tcpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Type = v1.ServiceTypeClusterIP
-+			s.Spec.Ports[0].NodePort = 0
-+		})
-+		framework.ExpectNoError(err)
-+		// Wait for the load balancer to be destroyed asynchronously
-+		_, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
-+		framework.ExpectNoError(err)
-+
-+		ginkgo.By("changing UDP service back to type=ClusterIP")
-+		_, err = udpJig.UpdateService(func(s *v1.Service) {
-+			s.Spec.Type = v1.ServiceTypeClusterIP
-+			s.Spec.Ports[0].NodePort = 0
-+		})
-+		framework.ExpectNoError(err)
-+		if loadBalancerSupportsUDP {
-+			// Wait for the load balancer to be destroyed asynchronously
-+			_, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout)
-+			framework.ExpectNoError(err)
-+		}
-+
-+		ginkgo.By("checking the TCP NodePort is closed")
-+		testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("checking the UDP NodePort is closed")
-+		testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
-+
-+		ginkgo.By("checking the TCP LoadBalancer is closed")
-+		testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
-+
-+		if loadBalancerSupportsUDP {
-+			ginkgo.By("checking the UDP LoadBalancer is closed")
-+			testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
-+		}
-+	})
-+
-+	ginkgo.It("should be able to create an internal type load balancer [Slow]", func() {
-+		e2eskipper.SkipUnlessProviderIs("azure", "gke", "gce")
-+
-+		createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
-+		pollInterval := framework.Poll * 10
-+
-+		namespace := f.Namespace.Name
-+		serviceName := "lb-internal"
-+		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+		ginkgo.By("creating pod to be part of service " + serviceName)
-+		_, err := jig.Run(nil)
-+		framework.ExpectNoError(err)
-+
-+		enableILB, disableILB := enableAndDisableInternalLB()
-+
-+		isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool {
-+			ingressEndpoint := e2eservice.GetIngressPoint(lbIngress)
-+			// Needs update for providers using hostname as endpoint.
-+			return strings.HasPrefix(ingressEndpoint, "10.")
-+		}
-+
-+		ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled")
-+		svc, err := jig.CreateTCPService(func(svc *v1.Service) {
-+			svc.Spec.Type = v1.ServiceTypeLoadBalancer
-+			enableILB(svc)
-+		})
-+		framework.ExpectNoError(err)
-+
-+		defer func() {
-+			ginkgo.By("Clean up loadbalancer service")
-+			e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
-+		}()
-+
-+		svc, err = jig.WaitForLoadBalancer(createTimeout)
-+		framework.ExpectNoError(err)
-+		lbIngress := &svc.Status.LoadBalancer.Ingress[0]
-+		svcPort := int(svc.Spec.Ports[0].Port)
-+		// should have an internal IP.
-+		framework.ExpectEqual(isInternalEndpoint(lbIngress), true)
-+
-+		// ILBs are not accessible from the test orchestrator, so it's necessary to use
-+		//  a pod to test the service.
-+		ginkgo.By("hitting the internal load balancer from pod")
-+		framework.Logf("creating pod with host network")
-+		hostExec := launchHostExecPod(f.ClientSet, f.Namespace.Name, "ilb-host-exec")
-+
-+		framework.Logf("Waiting up to %v for service %q's internal LB to respond to requests", createTimeout, serviceName)
-+		tcpIngressIP := e2eservice.GetIngressPoint(lbIngress)
-+		if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
-+			cmd := fmt.Sprintf(`curl -m 5 'http://%v:%v/echo?msg=hello'`, tcpIngressIP, svcPort)
-+			stdout, err := framework.RunHostCmd(hostExec.Namespace, hostExec.Name, cmd)
-+			if err != nil {
-+				framework.Logf("error curling; stdout: %v. err: %v", stdout, err)
-+				return false, nil
-+			}
-+
-+			if !strings.Contains(stdout, "hello") {
-+				framework.Logf("Expected output to contain 'hello', got %q; retrying...", stdout)
-+				return false, nil
-+			}
-+
-+			framework.Logf("Successful curl; stdout: %v", stdout)
-+			return true, nil
-+		}); pollErr != nil {
-+			framework.Failf("ginkgo.Failed to hit ILB IP, err: %v", pollErr)
-+		}
-+
-+		ginkgo.By("switching to external type LoadBalancer")
-+		svc, err = jig.UpdateService(func(svc *v1.Service) {
-+			disableILB(svc)
-+		})
-+		framework.ExpectNoError(err)
-+		framework.Logf("Waiting up to %v for service %q to have an external LoadBalancer", createTimeout, serviceName)
-+		if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
-+			svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
-+			if err != nil {
-+				return false, err
-+			}
-+			lbIngress = &svc.Status.LoadBalancer.Ingress[0]
-+			return !isInternalEndpoint(lbIngress), nil
-+		}); pollErr != nil {
-+			framework.Failf("Loadbalancer IP not changed to external.")
-+		}
-+		// should have an external IP.
-+		gomega.Expect(isInternalEndpoint(lbIngress)).To(gomega.BeFalse())
-+
-+		ginkgo.By("hitting the external load balancer")
-+		framework.Logf("Waiting up to %v for service %q's external LB to respond to requests", createTimeout, serviceName)
-+		tcpIngressIP = e2eservice.GetIngressPoint(lbIngress)
-+		e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
-+
-+		// GCE cannot test a specific IP because the test may not own it. This cloud specific condition
-+		// will be removed when GCP supports similar functionality.
-+		if framework.ProviderIs("azure") {
-+			ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.")
-+			internalStaticIP := "10.240.11.11"
-+			svc, err = jig.UpdateService(func(svc *v1.Service) {
-+				svc.Spec.LoadBalancerIP = internalStaticIP
-+				enableILB(svc)
-+			})
-+			framework.ExpectNoError(err)
-+			framework.Logf("Waiting up to %v for service %q to have an internal LoadBalancer", createTimeout, serviceName)
-+			if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
-+				svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
-+				if err != nil {
-+					return false, err
-+				}
-+				lbIngress = &svc.Status.LoadBalancer.Ingress[0]
-+				return isInternalEndpoint(lbIngress), nil
-+			}); pollErr != nil {
-+				framework.Failf("Loadbalancer IP not changed to internal.")
-+			}
-+			// should have the given static internal IP.
-+			framework.ExpectEqual(e2eservice.GetIngressPoint(lbIngress), internalStaticIP)
-+		}
-+	})
-+
-+	// This test creates a load balancer, make sure its health check interval
-+	// equals to gceHcCheckIntervalSeconds. Then the interval is manipulated
-+	// to be something else, see if the interval will be reconciled.
-+	ginkgo.It("should reconcile LB health check interval [Slow][Serial]", func() {
-+		const gceHcCheckIntervalSeconds = int64(8)
-+		// This test is for clusters on GCE.
-+		// (It restarts kube-controller-manager, which we don't support on GKE)
-+		e2eskipper.SkipUnlessProviderIs("gce")
-+		e2eskipper.SkipUnlessSSHKeyPresent()
-+
-+		clusterID, err := gce.GetClusterID(cs)
-+		if err != nil {
-+			framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err)
-+		}
-+		gceCloud, err := gce.GetGCECloud()
-+		if err != nil {
-+			framework.Failf("framework.GetGCECloud() = _, %v; want nil", err)
-+		}
-+
-+		namespace := f.Namespace.Name
-+		serviceName := "lb-hc-int"
-+		jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+		ginkgo.By("create load balancer service")
-+		// Create loadbalancer service with source range from node[0] and podAccept
-+		svc, err := jig.CreateTCPService(func(svc *v1.Service) {
-+			svc.Spec.Type = v1.ServiceTypeLoadBalancer
-+		})
-+		framework.ExpectNoError(err)
-+
-+		defer func() {
-+			ginkgo.By("Clean up loadbalancer service")
-+			e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
-+		}()
-+
-+		svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
-+		framework.ExpectNoError(err)
-+
-+		hcName := gcecloud.MakeNodesHealthCheckName(clusterID)
-+		hc, err := gceCloud.GetHTTPHealthCheck(hcName)
-+		if err != nil {
-+			framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err)
-+		}
-+		framework.ExpectEqual(hc.CheckIntervalSec, gceHcCheckIntervalSeconds)
-+
-+		ginkgo.By("modify the health check interval")
-+		hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1
-+		if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil {
-+			framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err)
-+		}
-+
-+		ginkgo.By("restart kube-controller-manager")
-+		if err := e2ekubesystem.RestartControllerManager(); err != nil {
-+			framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err)
-+		}
-+		if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil {
-+			framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err)
-+		}
-+
-+		ginkgo.By("health check should be reconciled")
-+		pollInterval := framework.Poll * 10
-+		loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
-+		if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) {
-+			hc, err := gceCloud.GetHTTPHealthCheck(hcName)
-+			if err != nil {
-+				framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err)
-+				return false, err
-+			}
-+			framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec)
-+			return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil
-+		}); pollErr != nil {
-+			framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds)
-+		}
-+	})
-+
-+	var _ = SIGDescribe("ESIPP [Slow]", func() {
-+		f := framework.NewDefaultFramework("esipp")
-+		var loadBalancerCreateTimeout time.Duration
-+
-+		var cs clientset.Interface
-+		serviceLBNames := []string{}
-+
-+		ginkgo.BeforeEach(func() {
-+			// requires cloud load-balancer support - this feature currently supported only on GCE/GKE
-+			e2eskipper.SkipUnlessProviderIs("gce", "gke")
-+
-+			cs = f.ClientSet
-+			loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
-+		})
-+
-+		ginkgo.AfterEach(func() {
-+			if ginkgo.CurrentGinkgoTestDescription().Failed {
-+				DescribeSvc(f.Namespace.Name)
-+			}
-+			for _, lb := range serviceLBNames {
-+				framework.Logf("cleaning load balancer resource for %s", lb)
-+				e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
-+			}
-+			//reset serviceLBNames
-+			serviceLBNames = []string{}
-+		})
-+
-+		ginkgo.It("should work for type=LoadBalancer", func() {
-+			namespace := f.Namespace.Name
-+			serviceName := "external-local-lb"
-+			jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+			svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
-+			framework.ExpectNoError(err)
-+			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
-+			healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
-+			if healthCheckNodePort == 0 {
-+				framework.Failf("Service HealthCheck NodePort was not allocated")
-+			}
-+			defer func() {
-+				err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
-+				framework.ExpectNoError(err)
-+
-+				// Make sure we didn't leak the health check node port.
-+				threshold := 2
-+				nodes, err := jig.GetEndpointNodes()
-+				framework.ExpectNoError(err)
-+				for _, ips := range nodes {
-+					err := TestHTTPHealthCheckNodePort(ips[0], healthCheckNodePort, "/healthz", e2eservice.KubeProxyEndpointLagTimeout, false, threshold)
-+					framework.ExpectNoError(err)
-+				}
-+				err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err)
-+			}()
-+
-+			svcTCPPort := int(svc.Spec.Ports[0].Port)
-+			ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
-+
-+			ginkgo.By("reading clientIP using the TCP service's service port via its external VIP")
-+			content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
-+			clientIP := content.String()
-+			framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP)
-+
-+			ginkgo.By("checking if Source IP is preserved")
-+			if strings.HasPrefix(clientIP, "10.") {
-+				framework.Failf("Source IP was NOT preserved")
-+			}
-+		})
-+
-+		ginkgo.It("should work for type=NodePort", func() {
-+			namespace := f.Namespace.Name
-+			serviceName := "external-local-nodeport"
-+			jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+			svc, err := jig.CreateOnlyLocalNodePortService(true)
-+			framework.ExpectNoError(err)
-+			defer func() {
-+				err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err)
-+			}()
-+
-+			tcpNodePort := int(svc.Spec.Ports[0].NodePort)
-+			endpointsNodeMap, err := jig.GetEndpointNodes()
-+			framework.ExpectNoError(err)
-+			path := "/clientip"
-+
-+			for nodeName, nodeIPs := range endpointsNodeMap {
-+				nodeIP := nodeIPs[0]
-+				ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v%v%v", nodeName, nodeIP, tcpNodePort, path))
-+				content := GetHTTPContent(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, path)
-+				clientIP := content.String()
-+				framework.Logf("ClientIP detected by target pod using NodePort is %s", clientIP)
-+				if strings.HasPrefix(clientIP, "10.") {
-+					framework.Failf("Source IP was NOT preserved")
-+				}
-+			}
-+		})
-+
-+		ginkgo.It("should only target nodes with endpoints", func() {
-+			namespace := f.Namespace.Name
-+			serviceName := "external-local-nodes"
-+			jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+			nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
-+			framework.ExpectNoError(err)
-+
-+			svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false,
-+				func(svc *v1.Service) {
-+					// Change service port to avoid collision with opened hostPorts
-+					// in other tests that run in parallel.
-+					if len(svc.Spec.Ports) != 0 {
-+						svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port))
-+						svc.Spec.Ports[0].Port = 8081
-+					}
-+
-+				})
-+			framework.ExpectNoError(err)
-+			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
-+			defer func() {
-+				err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
-+				framework.ExpectNoError(err)
-+				err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err)
-+			}()
-+
-+			healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
-+			if healthCheckNodePort == 0 {
-+				framework.Failf("Service HealthCheck NodePort was not allocated")
-+			}
-+
-+			ips := e2enode.CollectAddresses(nodes, v1.NodeExternalIP)
-+
-+			ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
-+			svcTCPPort := int(svc.Spec.Ports[0].Port)
-+
-+			threshold := 2
-+			path := "/healthz"
-+			for i := 0; i < len(nodes.Items); i++ {
-+				endpointNodeName := nodes.Items[i].Name
-+
-+				ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName)
-+				_, err = jig.Run(func(rc *v1.ReplicationController) {
-+					rc.Name = serviceName
-+					if endpointNodeName != "" {
-+						rc.Spec.Template.Spec.NodeName = endpointNodeName
-+					}
-+				})
-+				framework.ExpectNoError(err)
-+
-+				ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName))
-+				err = jig.WaitForEndpointOnNode(endpointNodeName)
-+				framework.ExpectNoError(err)
-+
-+				// HealthCheck should pass only on the node where num(endpoints) > 0
-+				// All other nodes should fail the healthcheck on the service healthCheckNodePort
-+				for n, publicIP := range ips {
-+					// Make sure the loadbalancer picked up the health check change.
-+					// Confirm traffic can reach backend through LB before checking healthcheck nodeport.
-+					e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout)
-+					expectedSuccess := nodes.Items[n].Name == endpointNodeName
-+					port := strconv.Itoa(healthCheckNodePort)
-+					ipPort := net.JoinHostPort(publicIP, port)
-+					framework.Logf("Health checking %s, http://%s%s, expectedSuccess %v", nodes.Items[n].Name, ipPort, path, expectedSuccess)
-+					err := TestHTTPHealthCheckNodePort(publicIP, healthCheckNodePort, path, e2eservice.KubeProxyEndpointLagTimeout, expectedSuccess, threshold)
-+					framework.ExpectNoError(err)
-+				}
-+				framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName))
-+			}
-+		})
-+
-+		ginkgo.It("should work from pods", func() {
-+			var err error
-+			namespace := f.Namespace.Name
-+			serviceName := "external-local-pods"
-+			jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+			svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
-+			framework.ExpectNoError(err)
-+			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
-+			defer func() {
-+				err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
-+				framework.ExpectNoError(err)
-+				err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err)
-+			}()
-+
-+			ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
-+			port := strconv.Itoa(int(svc.Spec.Ports[0].Port))
-+			ipPort := net.JoinHostPort(ingressIP, port)
-+			path := fmt.Sprintf("%s/clientip", ipPort)
-+
-+			ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state")
-+			deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1)
-+			framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment")
-+
-+			defer func() {
-+				framework.Logf("Deleting deployment")
-+				err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name)
-+			}()
-+
-+			deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
-+			framework.ExpectNoError(err, "Error in retrieving pause pod deployment")
-+			labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
-+			framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment")
-+
-+			pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()})
-+			framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments")
-+
-+			pausePod := pausePods.Items[0]
-+			framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path)
-+			cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path)
-+
-+			var srcIP string
-+			loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
-+			ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName))
-+			if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) {
-+				stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd)
-+				if err != nil {
-+					framework.Logf("got err: %v, retry until timeout", err)
-+					return false, nil
-+				}
-+				srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0])
-+				return srcIP == pausePod.Status.PodIP, nil
-+			}); pollErr != nil {
-+				framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP)
-+			}
-+		})
-+
-+		// TODO: Get rid of [DisabledForLargeClusters] tag when issue #90047 is fixed.
-+		ginkgo.It("should handle updates to ExternalTrafficPolicy field [DisabledForLargeClusters]", func() {
-+			namespace := f.Namespace.Name
-+			serviceName := "external-local-update"
-+			jig := e2eservice.NewTestJig(cs, namespace, serviceName)
-+
-+			nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
-+			framework.ExpectNoError(err)
-+			if len(nodes.Items) < 2 {
-+				framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint")
-+			}
-+
-+			svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
-+			framework.ExpectNoError(err)
-+			serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
-+			defer func() {
-+				err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
-+				framework.ExpectNoError(err)
-+				err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
-+				framework.ExpectNoError(err)
-+			}()
-+
-+			// save the health check node port because it disappears when ESIPP is turned off.
-+			healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
-+
-+			ginkgo.By("turning ESIPP off")
-+			svc, err = jig.UpdateService(func(svc *v1.Service) {
-+				svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
-+			})
-+			framework.ExpectNoError(err)
-+			if svc.Spec.HealthCheckNodePort > 0 {
-+				framework.Failf("Service HealthCheck NodePort still present")
-+			}
-+
-+			endpointNodeMap, err := jig.GetEndpointNodes()
-+			framework.ExpectNoError(err)
-+			noEndpointNodeMap := map[string][]string{}
-+			for _, n := range nodes.Items {
-+				if _, ok := endpointNodeMap[n.Name]; ok {
-+					continue
-+				}
-+				noEndpointNodeMap[n.Name] = e2enode.GetAddresses(&n, v1.NodeExternalIP)
-+			}
-+
-+			svcTCPPort := int(svc.Spec.Ports[0].Port)
-+			svcNodePort := int(svc.Spec.Ports[0].NodePort)
-+			ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
-+			path := "/clientip"
-+
-+			ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap))
-+			for nodeName, nodeIPs := range noEndpointNodeMap {
-+				ginkgo.By(fmt.Sprintf("Checking %v (%v:%v%v) proxies to endpoints on another node", nodeName, nodeIPs[0], svcNodePort, path))
-+				GetHTTPContent(nodeIPs[0], svcNodePort, e2eservice.KubeProxyLagTimeout, path)
-+			}
-+
-+			for nodeName, nodeIPs := range endpointNodeMap {
-+				ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIPs[0]))
-+				var body bytes.Buffer
-+				pollfn := func() (bool, error) {
-+					result := e2enetwork.PokeHTTP(nodeIPs[0], healthCheckNodePort, "/healthz", nil)
-+					if result.Code == 0 {
-+						return true, nil
-+					}
-+					body.Reset()
-+					body.Write(result.Body)
-+					return false, nil
-+				}
-+				if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollfn); pollErr != nil {
-+					framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s",
-+						nodeName, healthCheckNodePort, body.String())
-+				}
-+			}
-+
-+			// Poll till kube-proxy re-adds the MASQUERADE rule on the node.
-+			ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP))
-+			var clientIP string
-+			pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
-+				content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
-+				clientIP = content.String()
-+				if strings.HasPrefix(clientIP, "10.") {
-+					return true, nil
-+				}
-+				return false, nil
-+			})
-+			if pollErr != nil {
-+				framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP)
-+			}
-+
-+			// TODO: We need to attempt to create another service with the previously
-+			// allocated healthcheck nodePort. If the health check nodePort has been
-+			// freed, the new service creation will succeed, upon which we cleanup.
-+			// If the health check nodePort has NOT been freed, the new service
-+			// creation will fail.
-+
-+			ginkgo.By("setting ExternalTraffic field back to OnlyLocal")
-+			svc, err = jig.UpdateService(func(svc *v1.Service) {
-+				svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
-+				// Request the same healthCheckNodePort as before, to test the user-requested allocation path
-+				svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
-+			})
-+			framework.ExpectNoError(err)
-+			pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
-+				content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path)
-+				clientIP = content.String()
-+				ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP))
-+				if !strings.HasPrefix(clientIP, "10.") {
-+					return true, nil
-+				}
-+				return false, nil
-+			})
-+			if pollErr != nil {
-+				framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP)
-+			}
-+		})
-+	})
-+})
-diff --git a/test/e2e/node/recreate_node.go b/test/e2e/node/recreate_node.go
-index da3fc974485..b403fa7f737 100644
---- a/test/e2e/node/recreate_node.go
-+++ b/test/e2e/node/recreate_node.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2019 The Kubernetes Authors.
- 
-diff --git a/test/e2e/scheduling/nvidia-gpus.go b/test/e2e/scheduling/nvidia-gpus.go
-index 313e773b8e2..bbf66b59aac 100644
---- a/test/e2e/scheduling/nvidia-gpus.go
-+++ b/test/e2e/scheduling/nvidia-gpus.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/scheduling/ubernetes_lite_volumes.go b/test/e2e/scheduling/ubernetes_lite_volumes.go
-index 513ed07543f..78c0f081990 100644
---- a/test/e2e/scheduling/ubernetes_lite_volumes.go
-+++ b/test/e2e/scheduling/ubernetes_lite_volumes.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go
-index a866266c1f1..28e26a10c35 100644
---- a/test/e2e/storage/drivers/in_tree.go
-+++ b/test/e2e/storage/drivers/in_tree.go
-@@ -38,10 +38,8 @@ package drivers
- import (
- 	"context"
- 	"fmt"
--	"os/exec"
- 	"strconv"
- 	"strings"
--	"time"
- 
- 	"github.com/onsi/ginkgo"
- 	v1 "k8s.io/api/core/v1"
-@@ -57,13 +55,11 @@ import (
- 	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
- 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
- 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
--	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
- 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
- 	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
- 	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
- 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
- 	"k8s.io/kubernetes/test/e2e/storage/utils"
--	vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
- 	imageutils "k8s.io/kubernetes/test/utils/image"
- )
- 
-@@ -1044,734 +1040,6 @@ func (e *emptydirDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
- 	}, func() {}
- }
- 
--// Cinder
--// This driver assumes that OpenStack client tools are installed
--// (/usr/bin/nova, /usr/bin/cinder and /usr/bin/keystone)
--// and that the usual OpenStack authentication env. variables are set
--// (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least).
--type cinderDriver struct {
--	driverInfo testsuites.DriverInfo
--}
--
--type cinderVolume struct {
--	volumeName string
--	volumeID   string
--}
--
--var _ testsuites.TestDriver = &cinderDriver{}
--var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
--var _ testsuites.InlineVolumeTestDriver = &cinderDriver{}
--var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
--var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
--
--// InitCinderDriver returns cinderDriver that implements TestDriver interface
--func InitCinderDriver() testsuites.TestDriver {
--	return &cinderDriver{
--		driverInfo: testsuites.DriverInfo{
--			Name:             "cinder",
--			InTreePluginName: "kubernetes.io/cinder",
--			MaxFileSize:      testpatterns.FileSizeMedium,
--			SupportedSizeRange: e2evolume.SizeRange{
--				Min: "5Gi",
--			},
--			SupportedFsType: sets.NewString(
--				"", // Default fsType
--				"ext3",
--			),
--			TopologyKeys: []string{v1.LabelZoneFailureDomain},
--			Capabilities: map[testsuites.Capability]bool{
--				testsuites.CapPersistence: true,
--				testsuites.CapFsGroup:     true,
--				testsuites.CapExec:        true,
--				testsuites.CapBlock:       true,
--				// Cinder supports volume limits, but the test creates large
--				// number of volumes and times out test suites.
--				testsuites.CapVolumeLimits: false,
--				testsuites.CapTopology:     true,
--			},
--		},
--	}
--}
--
--func (c *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
--	return &c.driverInfo
--}
--
--func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
--	e2eskipper.SkipUnlessProviderIs("openstack")
--}
--
--func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
--	cv, ok := e2evolume.(*cinderVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
--
--	volSource := v1.VolumeSource{
--		Cinder: &v1.CinderVolumeSource{
--			VolumeID: cv.volumeID,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		volSource.Cinder.FSType = fsType
--	}
--	return &volSource
--}
--
--func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
--	cv, ok := e2evolume.(*cinderVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
--
--	pvSource := v1.PersistentVolumeSource{
--		Cinder: &v1.CinderPersistentVolumeSource{
--			VolumeID: cv.volumeID,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		pvSource.Cinder.FSType = fsType
--	}
--	return &pvSource, nil
--}
--
--func (c *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
--	provisioner := "kubernetes.io/cinder"
--	parameters := map[string]string{}
--	if fsType != "" {
--		parameters["fsType"] = fsType
--	}
--	ns := config.Framework.Namespace.Name
--	suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name)
--
--	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
--}
--
--func (c *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
--	return &testsuites.PerTestConfig{
--		Driver:    c,
--		Prefix:    "cinder",
--		Framework: f,
--	}, func() {}
--}
--
--func (c *cinderDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
--	f := config.Framework
--	ns := f.Namespace
--
--	// We assume that namespace.Name is a random string
--	volumeName := ns.Name
--	ginkgo.By("creating a test Cinder volume")
--	output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput()
--	outputString := string(output[:])
--	framework.Logf("cinder output:\n%s", outputString)
--	framework.ExpectNoError(err)
--
--	// Parse 'id'' from stdout. Expected format:
--	// |     attachments     |                  []                  |
--	// |  availability_zone  |                 nova                 |
--	// ...
--	// |          id         | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 |
--	volumeID := ""
--	for _, line := range strings.Split(outputString, "\n") {
--		fields := strings.Fields(line)
--		if len(fields) != 5 {
--			continue
--		}
--		if fields[1] != "id" {
--			continue
--		}
--		volumeID = fields[3]
--		break
--	}
--	framework.Logf("Volume ID: %s", volumeID)
--	framework.ExpectNotEqual(volumeID, "")
--	return &cinderVolume{
--		volumeName: volumeName,
--		volumeID:   volumeID,
--	}
--}
--
--func (v *cinderVolume) DeleteVolume() {
--	name := v.volumeName
--
--	// Try to delete the volume for several seconds - it takes
--	// a while for the plugin to detach it.
--	var output []byte
--	var err error
--	timeout := time.Second * 120
--
--	framework.Logf("Waiting up to %v for removal of cinder volume %s", timeout, name)
--	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
--		output, err = exec.Command("cinder", "delete", name).CombinedOutput()
--		if err == nil {
--			framework.Logf("Cinder volume %s deleted", name)
--			return
--		}
--		framework.Logf("Failed to delete volume %s: %v", name, err)
--	}
--	framework.Logf("Giving up deleting volume %s: %v\n%s", name, err, string(output[:]))
--}
--
--// GCE
--type gcePdDriver struct {
--	driverInfo testsuites.DriverInfo
--}
--
--type gcePdVolume struct {
--	volumeName string
--}
--
--var _ testsuites.TestDriver = &gcePdDriver{}
--var _ testsuites.PreprovisionedVolumeTestDriver = &gcePdDriver{}
--var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
--var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
--var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
--
--// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
--func InitGcePdDriver() testsuites.TestDriver {
--	// In current test structure, it first initialize the driver and then set up
--	// the new framework, so we cannot get the correct OS here. So here set to
--	// support all fs types including both linux and windows. We have code to check Node OS later
--	// during test.
--	supportedTypes := sets.NewString(
--		"", // Default fsType
--		"ext2",
--		"ext3",
--		"ext4",
--		"xfs",
--		"ntfs",
--	)
--	return &gcePdDriver{
--		driverInfo: testsuites.DriverInfo{
--			Name:             "gcepd",
--			InTreePluginName: "kubernetes.io/gce-pd",
--			MaxFileSize:      testpatterns.FileSizeMedium,
--			SupportedSizeRange: e2evolume.SizeRange{
--				Min: "5Gi",
--			},
--			SupportedFsType:      supportedTypes,
--			SupportedMountOption: sets.NewString("debug", "nouid32"),
--			TopologyKeys:         []string{v1.LabelZoneFailureDomain},
--			Capabilities: map[testsuites.Capability]bool{
--				testsuites.CapPersistence:         true,
--				testsuites.CapFsGroup:             true,
--				testsuites.CapBlock:               true,
--				testsuites.CapExec:                true,
--				testsuites.CapMultiPODs:           true,
--				testsuites.CapControllerExpansion: true,
--				testsuites.CapNodeExpansion:       true,
--				// GCE supports volume limits, but the test creates large
--				// number of volumes and times out test suites.
--				testsuites.CapVolumeLimits: false,
--				testsuites.CapTopology:     true,
--			},
--		},
--	}
--}
--
--func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
--	return &g.driverInfo
--}
--
--func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
--	e2eskipper.SkipUnlessProviderIs("gce", "gke")
--	if pattern.FeatureTag == "[sig-windows]" {
--		e2eskipper.SkipUnlessNodeOSDistroIs("windows")
--	}
--}
--
--func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
--	gv, ok := e2evolume.(*gcePdVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
--	volSource := v1.VolumeSource{
--		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
--			PDName:   gv.volumeName,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		volSource.GCEPersistentDisk.FSType = fsType
--	}
--	return &volSource
--}
--
--func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
--	gv, ok := e2evolume.(*gcePdVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
--	pvSource := v1.PersistentVolumeSource{
--		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
--			PDName:   gv.volumeName,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		pvSource.GCEPersistentDisk.FSType = fsType
--	}
--	return &pvSource, nil
--}
--
--func (g *gcePdDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
--	provisioner := "kubernetes.io/gce-pd"
--	parameters := map[string]string{}
--	if fsType != "" {
--		parameters["fsType"] = fsType
--	}
--	ns := config.Framework.Namespace.Name
--	suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
--	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
--
--	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
--}
--
--func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
--	config := &testsuites.PerTestConfig{
--		Driver:    g,
--		Prefix:    "gcepd",
--		Framework: f,
--	}
--
--	if framework.NodeOSDistroIs("windows") {
--		config.ClientNodeSelection = e2epod.NodeSelection{
--			Selector: map[string]string{
--				"kubernetes.io/os": "windows",
--			},
--		}
--	}
--	return config, func() {}
--
--}
--
--func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
--	zone := getInlineVolumeZone(config.Framework)
--	if volType == testpatterns.InlineVolume {
--		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
--		// so pods should be also scheduled there.
--		config.ClientNodeSelection = e2epod.NodeSelection{
--			Selector: map[string]string{
--				v1.LabelZoneFailureDomain: zone,
--			},
--		}
--	}
--	ginkgo.By("creating a test gce pd volume")
--	vname, err := e2epv.CreatePDWithRetryAndZone(zone)
--	framework.ExpectNoError(err)
--	return &gcePdVolume{
--		volumeName: vname,
--	}
--}
--
--func (v *gcePdVolume) DeleteVolume() {
--	e2epv.DeletePDWithRetry(v.volumeName)
--}
--
--// vSphere
--type vSphereDriver struct {
--	driverInfo testsuites.DriverInfo
--}
--
--type vSphereVolume struct {
--	volumePath string
--	nodeInfo   *vspheretest.NodeInfo
--}
--
--var _ testsuites.TestDriver = &vSphereDriver{}
--var _ testsuites.PreprovisionedVolumeTestDriver = &vSphereDriver{}
--var _ testsuites.InlineVolumeTestDriver = &vSphereDriver{}
--var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{}
--var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
--
--// InitVSphereDriver returns vSphereDriver that implements TestDriver interface
--func InitVSphereDriver() testsuites.TestDriver {
--	return &vSphereDriver{
--		driverInfo: testsuites.DriverInfo{
--			Name:             "vsphere",
--			InTreePluginName: "kubernetes.io/vsphere-volume",
--			MaxFileSize:      testpatterns.FileSizeMedium,
--			SupportedSizeRange: e2evolume.SizeRange{
--				Min: "5Gi",
--			},
--			SupportedFsType: sets.NewString(
--				"", // Default fsType
--				"ext4",
--			),
--			TopologyKeys: []string{v1.LabelZoneFailureDomain},
--			Capabilities: map[testsuites.Capability]bool{
--				testsuites.CapPersistence: true,
--				testsuites.CapFsGroup:     true,
--				testsuites.CapExec:        true,
--				testsuites.CapMultiPODs:   true,
--				testsuites.CapTopology:    true,
--			},
--		},
--	}
--}
--func (v *vSphereDriver) GetDriverInfo() *testsuites.DriverInfo {
--	return &v.driverInfo
--}
--
--func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
--	e2eskipper.SkipUnlessProviderIs("vsphere")
--}
--
--func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
--	vsv, ok := e2evolume.(*vSphereVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
--
--	// vSphere driver doesn't seem to support readOnly volume
--	// TODO: check if it is correct
--	if readOnly {
--		return nil
--	}
--	volSource := v1.VolumeSource{
--		VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
--			VolumePath: vsv.volumePath,
--		},
--	}
--	if fsType != "" {
--		volSource.VsphereVolume.FSType = fsType
--	}
--	return &volSource
--}
--
--func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
--	vsv, ok := e2evolume.(*vSphereVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
--
--	// vSphere driver doesn't seem to support readOnly volume
--	// TODO: check if it is correct
--	if readOnly {
--		return nil, nil
--	}
--	pvSource := v1.PersistentVolumeSource{
--		VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
--			VolumePath: vsv.volumePath,
--		},
--	}
--	if fsType != "" {
--		pvSource.VsphereVolume.FSType = fsType
--	}
--	return &pvSource, nil
--}
--
--func (v *vSphereDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
--	provisioner := "kubernetes.io/vsphere-volume"
--	parameters := map[string]string{}
--	if fsType != "" {
--		parameters["fsType"] = fsType
--	}
--	ns := config.Framework.Namespace.Name
--	suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name)
--
--	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
--}
--
--func (v *vSphereDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
--	return &testsuites.PerTestConfig{
--		Driver:    v,
--		Prefix:    "vsphere",
--		Framework: f,
--	}, func() {}
--}
--
--func (v *vSphereDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
--	f := config.Framework
--	vspheretest.Bootstrap(f)
--	nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo()
--	volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef)
--	framework.ExpectNoError(err)
--	return &vSphereVolume{
--		volumePath: volumePath,
--		nodeInfo:   nodeInfo,
--	}
--}
--
--func (v *vSphereVolume) DeleteVolume() {
--	v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef)
--}
--
--// Azure Disk
--type azureDiskDriver struct {
--	driverInfo testsuites.DriverInfo
--}
--
--type azureDiskVolume struct {
--	volumeName string
--}
--
--var _ testsuites.TestDriver = &azureDiskDriver{}
--var _ testsuites.PreprovisionedVolumeTestDriver = &azureDiskDriver{}
--var _ testsuites.InlineVolumeTestDriver = &azureDiskDriver{}
--var _ testsuites.PreprovisionedPVTestDriver = &azureDiskDriver{}
--var _ testsuites.DynamicPVTestDriver = &azureDiskDriver{}
--
--// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
--func InitAzureDiskDriver() testsuites.TestDriver {
--	return &azureDiskDriver{
--		driverInfo: testsuites.DriverInfo{
--			Name:             "azure-disk",
--			InTreePluginName: "kubernetes.io/azure-disk",
--			MaxFileSize:      testpatterns.FileSizeMedium,
--			SupportedSizeRange: e2evolume.SizeRange{
--				Min: "5Gi",
--			},
--			SupportedFsType: sets.NewString(
--				"", // Default fsType
--				"ext3",
--				"ext4",
--				"xfs",
--			),
--			TopologyKeys: []string{v1.LabelZoneFailureDomain},
--			Capabilities: map[testsuites.Capability]bool{
--				testsuites.CapPersistence: true,
--				testsuites.CapFsGroup:     true,
--				testsuites.CapBlock:       true,
--				testsuites.CapExec:        true,
--				testsuites.CapMultiPODs:   true,
--				// Azure supports volume limits, but the test creates large
--				// number of volumes and times out test suites.
--				testsuites.CapVolumeLimits: false,
--				testsuites.CapTopology:     true,
--			},
--		},
--	}
--}
--
--func (a *azureDiskDriver) GetDriverInfo() *testsuites.DriverInfo {
--	return &a.driverInfo
--}
--
--func (a *azureDiskDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
--	e2eskipper.SkipUnlessProviderIs("azure")
--}
--
--func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
--	av, ok := e2evolume.(*azureDiskVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
--	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
--
--	kind := v1.AzureManagedDisk
--	volSource := v1.VolumeSource{
--		AzureDisk: &v1.AzureDiskVolumeSource{
--			DiskName:    diskName,
--			DataDiskURI: av.volumeName,
--			Kind:        &kind,
--			ReadOnly:    &readOnly,
--		},
--	}
--	if fsType != "" {
--		volSource.AzureDisk.FSType = &fsType
--	}
--	return &volSource
--}
--
--func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
--	av, ok := e2evolume.(*azureDiskVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
--
--	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
--
--	kind := v1.AzureManagedDisk
--	pvSource := v1.PersistentVolumeSource{
--		AzureDisk: &v1.AzureDiskVolumeSource{
--			DiskName:    diskName,
--			DataDiskURI: av.volumeName,
--			Kind:        &kind,
--			ReadOnly:    &readOnly,
--		},
--	}
--	if fsType != "" {
--		pvSource.AzureDisk.FSType = &fsType
--	}
--	return &pvSource, nil
--}
--
--func (a *azureDiskDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
--	provisioner := "kubernetes.io/azure-disk"
--	parameters := map[string]string{}
--	if fsType != "" {
--		parameters["fsType"] = fsType
--	}
--	ns := config.Framework.Namespace.Name
--	suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
--	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
--
--	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
--}
--
--func (a *azureDiskDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
--	return &testsuites.PerTestConfig{
--		Driver:    a,
--		Prefix:    "azure",
--		Framework: f,
--	}, func() {}
--}
--
--func (a *azureDiskDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
--	ginkgo.By("creating a test azure disk volume")
--	zone := getInlineVolumeZone(config.Framework)
--	if volType == testpatterns.InlineVolume {
--		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
--		// so pods should be also scheduled there.
--		config.ClientNodeSelection = e2epod.NodeSelection{
--			Selector: map[string]string{
--				v1.LabelZoneFailureDomain: zone,
--			},
--		}
--	}
--	volumeName, err := e2epv.CreatePDWithRetryAndZone(zone)
--	framework.ExpectNoError(err)
--	return &azureDiskVolume{
--		volumeName: volumeName,
--	}
--}
--
--func (v *azureDiskVolume) DeleteVolume() {
--	e2epv.DeletePDWithRetry(v.volumeName)
--}
--
--// AWS
--type awsDriver struct {
--	driverInfo testsuites.DriverInfo
--}
--
--type awsVolume struct {
--	volumeName string
--}
--
--var _ testsuites.TestDriver = &awsDriver{}
--
--var _ testsuites.PreprovisionedVolumeTestDriver = &awsDriver{}
--var _ testsuites.InlineVolumeTestDriver = &awsDriver{}
--var _ testsuites.PreprovisionedPVTestDriver = &awsDriver{}
--var _ testsuites.DynamicPVTestDriver = &awsDriver{}
--
--// InitAwsDriver returns awsDriver that implements TestDriver interface
--func InitAwsDriver() testsuites.TestDriver {
--	return &awsDriver{
--		driverInfo: testsuites.DriverInfo{
--			Name:             "aws",
--			InTreePluginName: "kubernetes.io/aws-ebs",
--			MaxFileSize:      testpatterns.FileSizeMedium,
--			SupportedSizeRange: e2evolume.SizeRange{
--				Min: "5Gi",
--			},
--			SupportedFsType: sets.NewString(
--				"", // Default fsType
--				"ext2",
--				"ext3",
--				"ext4",
--				"xfs",
--				"ntfs",
--			),
--			SupportedMountOption: sets.NewString("debug", "nouid32"),
--			TopologyKeys:         []string{v1.LabelZoneFailureDomain},
--			Capabilities: map[testsuites.Capability]bool{
--				testsuites.CapPersistence:         true,
--				testsuites.CapFsGroup:             true,
--				testsuites.CapBlock:               true,
--				testsuites.CapExec:                true,
--				testsuites.CapMultiPODs:           true,
--				testsuites.CapControllerExpansion: true,
--				testsuites.CapNodeExpansion:       true,
--				// AWS supports volume limits, but the test creates large
--				// number of volumes and times out test suites.
--				testsuites.CapVolumeLimits: false,
--				testsuites.CapTopology:     true,
--			},
--		},
--	}
--}
--
--func (a *awsDriver) GetDriverInfo() *testsuites.DriverInfo {
--	return &a.driverInfo
--}
--
--func (a *awsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
--	e2eskipper.SkipUnlessProviderIs("aws")
--}
--
--func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
--	av, ok := e2evolume.(*awsVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
--	volSource := v1.VolumeSource{
--		AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
--			VolumeID: av.volumeName,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		volSource.AWSElasticBlockStore.FSType = fsType
--	}
--	return &volSource
--}
--
--func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
--	av, ok := e2evolume.(*awsVolume)
--	framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
--	pvSource := v1.PersistentVolumeSource{
--		AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
--			VolumeID: av.volumeName,
--			ReadOnly: readOnly,
--		},
--	}
--	if fsType != "" {
--		pvSource.AWSElasticBlockStore.FSType = fsType
--	}
--	return &pvSource, nil
--}
--
--func (a *awsDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
--	provisioner := "kubernetes.io/aws-ebs"
--	parameters := map[string]string{}
--	if fsType != "" {
--		parameters["fsType"] = fsType
--	}
--	ns := config.Framework.Namespace.Name
--	suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
--	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
--
--	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
--}
--
--func (a *awsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
--	config := &testsuites.PerTestConfig{
--		Driver:    a,
--		Prefix:    "aws",
--		Framework: f,
--	}
--
--	if framework.NodeOSDistroIs("windows") {
--		config.ClientNodeSelection = e2epod.NodeSelection{
--			Selector: map[string]string{
--				"kubernetes.io/os": "windows",
--			},
--		}
--	}
--	return config, func() {}
--}
--
--func (a *awsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
--	zone := getInlineVolumeZone(config.Framework)
--	if volType == testpatterns.InlineVolume {
--		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
--		// so pods should be also scheduled there.
--		config.ClientNodeSelection = e2epod.NodeSelection{
--			Selector: map[string]string{
--				v1.LabelZoneFailureDomain: zone,
--			},
--		}
--	}
--	ginkgo.By("creating a test aws volume")
--	vname, err := e2epv.CreatePDWithRetryAndZone(zone)
--	framework.ExpectNoError(err)
--	return &awsVolume{
--		volumeName: vname,
--	}
--}
--
--func (v *awsVolume) DeleteVolume() {
--	e2epv.DeletePDWithRetry(v.volumeName)
--}
--
- // local
- type localDriver struct {
- 	driverInfo testsuites.DriverInfo
-diff --git a/test/e2e/storage/drivers/in_tree_providers.go b/test/e2e/storage/drivers/in_tree_providers.go
-new file mode 100644
-index 00000000000..c7f5dd3052e
---- /dev/null
-+++ b/test/e2e/storage/drivers/in_tree_providers.go
-@@ -0,0 +1,751 @@
-+// +build !providerless
-+
-+package drivers
-+
-+import (
-+	"fmt"
-+	"os/exec"
-+	"strings"
-+	"time"
-+
-+	"github.com/onsi/ginkgo"
-+	v1 "k8s.io/api/core/v1"
-+	storagev1 "k8s.io/api/storage/v1"
-+	"k8s.io/apimachinery/pkg/util/sets"
-+	"k8s.io/kubernetes/test/e2e/framework"
-+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-+	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
-+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-+	e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
-+	"k8s.io/kubernetes/test/e2e/storage/testpatterns"
-+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
-+	vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
-+)
-+
-+// Cinder
-+// This driver assumes that OpenStack client tools are installed
-+// (/usr/bin/nova, /usr/bin/cinder and /usr/bin/keystone)
-+// and that the usual OpenStack authentication env. variables are set
-+// (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least).
-+type cinderDriver struct {
-+	driverInfo testsuites.DriverInfo
-+}
-+
-+type cinderVolume struct {
-+	volumeName string
-+	volumeID   string
-+}
-+
-+var _ testsuites.TestDriver = &cinderDriver{}
-+var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
-+var _ testsuites.InlineVolumeTestDriver = &cinderDriver{}
-+var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
-+var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
-+
-+// InitCinderDriver returns cinderDriver that implements TestDriver interface
-+func InitCinderDriver() testsuites.TestDriver {
-+	return &cinderDriver{
-+		driverInfo: testsuites.DriverInfo{
-+			Name:             "cinder",
-+			InTreePluginName: "kubernetes.io/cinder",
-+			MaxFileSize:      testpatterns.FileSizeMedium,
-+			SupportedSizeRange: e2evolume.SizeRange{
-+				Min: "5Gi",
-+			},
-+			SupportedFsType: sets.NewString(
-+				"", // Default fsType
-+				"ext3",
-+			),
-+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
-+			Capabilities: map[testsuites.Capability]bool{
-+				testsuites.CapPersistence: true,
-+				testsuites.CapFsGroup:     true,
-+				testsuites.CapExec:        true,
-+				testsuites.CapBlock:       true,
-+				// Cinder supports volume limits, but the test creates large
-+				// number of volumes and times out test suites.
-+				testsuites.CapVolumeLimits: false,
-+				testsuites.CapTopology:     true,
-+			},
-+		},
-+	}
-+}
-+
-+func (c *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
-+	return &c.driverInfo
-+}
-+
-+func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
-+	e2eskipper.SkipUnlessProviderIs("openstack")
-+}
-+
-+func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
-+	cv, ok := e2evolume.(*cinderVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
-+
-+	volSource := v1.VolumeSource{
-+		Cinder: &v1.CinderVolumeSource{
-+			VolumeID: cv.volumeID,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		volSource.Cinder.FSType = fsType
-+	}
-+	return &volSource
-+}
-+
-+func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
-+	cv, ok := e2evolume.(*cinderVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
-+
-+	pvSource := v1.PersistentVolumeSource{
-+		Cinder: &v1.CinderPersistentVolumeSource{
-+			VolumeID: cv.volumeID,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		pvSource.Cinder.FSType = fsType
-+	}
-+	return &pvSource, nil
-+}
-+
-+func (c *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
-+	provisioner := "kubernetes.io/cinder"
-+	parameters := map[string]string{}
-+	if fsType != "" {
-+		parameters["fsType"] = fsType
-+	}
-+	ns := config.Framework.Namespace.Name
-+	suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name)
-+
-+	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
-+}
-+
-+func (c *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-+	return &testsuites.PerTestConfig{
-+		Driver:    c,
-+		Prefix:    "cinder",
-+		Framework: f,
-+	}, func() {}
-+}
-+
-+func (c *cinderDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
-+	f := config.Framework
-+	ns := f.Namespace
-+
-+	// We assume that namespace.Name is a random string
-+	volumeName := ns.Name
-+	ginkgo.By("creating a test Cinder volume")
-+	output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput()
-+	outputString := string(output[:])
-+	framework.Logf("cinder output:\n%s", outputString)
-+	framework.ExpectNoError(err)
-+
-+	// Parse 'id'' from stdout. Expected format:
-+	// |     attachments     |                  []                  |
-+	// |  availability_zone  |                 nova                 |
-+	// ...
-+	// |          id         | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 |
-+	volumeID := ""
-+	for _, line := range strings.Split(outputString, "\n") {
-+		fields := strings.Fields(line)
-+		if len(fields) != 5 {
-+			continue
-+		}
-+		if fields[1] != "id" {
-+			continue
-+		}
-+		volumeID = fields[3]
-+		break
-+	}
-+	framework.Logf("Volume ID: %s", volumeID)
-+	framework.ExpectNotEqual(volumeID, "")
-+	return &cinderVolume{
-+		volumeName: volumeName,
-+		volumeID:   volumeID,
-+	}
-+}
-+
-+func (v *cinderVolume) DeleteVolume() {
-+	name := v.volumeName
-+
-+	// Try to delete the volume for several seconds - it takes
-+	// a while for the plugin to detach it.
-+	var output []byte
-+	var err error
-+	timeout := time.Second * 120
-+
-+	framework.Logf("Waiting up to %v for removal of cinder volume %s", timeout, name)
-+	for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
-+		output, err = exec.Command("cinder", "delete", name).CombinedOutput()
-+		if err == nil {
-+			framework.Logf("Cinder volume %s deleted", name)
-+			return
-+		}
-+		framework.Logf("Failed to delete volume %s: %v", name, err)
-+	}
-+	framework.Logf("Giving up deleting volume %s: %v\n%s", name, err, string(output[:]))
-+}
-+
-+// GCE
-+type gcePdDriver struct {
-+	driverInfo testsuites.DriverInfo
-+}
-+
-+type gcePdVolume struct {
-+	volumeName string
-+}
-+
-+var _ testsuites.TestDriver = &gcePdDriver{}
-+var _ testsuites.PreprovisionedVolumeTestDriver = &gcePdDriver{}
-+var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
-+var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
-+var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
-+
-+// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
-+func InitGcePdDriver() testsuites.TestDriver {
-+	// In current test structure, it first initialize the driver and then set up
-+	// the new framework, so we cannot get the correct OS here. So here set to
-+	// support all fs types including both linux and windows. We have code to check Node OS later
-+	// during test.
-+	supportedTypes := sets.NewString(
-+		"", // Default fsType
-+		"ext2",
-+		"ext3",
-+		"ext4",
-+		"xfs",
-+		"ntfs",
-+	)
-+	return &gcePdDriver{
-+		driverInfo: testsuites.DriverInfo{
-+			Name:             "gcepd",
-+			InTreePluginName: "kubernetes.io/gce-pd",
-+			MaxFileSize:      testpatterns.FileSizeMedium,
-+			SupportedSizeRange: e2evolume.SizeRange{
-+				Min: "5Gi",
-+			},
-+			SupportedFsType:      supportedTypes,
-+			SupportedMountOption: sets.NewString("debug", "nouid32"),
-+			TopologyKeys:         []string{v1.LabelZoneFailureDomain},
-+			Capabilities: map[testsuites.Capability]bool{
-+				testsuites.CapPersistence:         true,
-+				testsuites.CapFsGroup:             true,
-+				testsuites.CapBlock:               true,
-+				testsuites.CapExec:                true,
-+				testsuites.CapMultiPODs:           true,
-+				testsuites.CapControllerExpansion: true,
-+				testsuites.CapNodeExpansion:       true,
-+				// GCE supports volume limits, but the test creates large
-+				// number of volumes and times out test suites.
-+				testsuites.CapVolumeLimits: false,
-+				testsuites.CapTopology:     true,
-+			},
-+		},
-+	}
-+}
-+
-+func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
-+	return &g.driverInfo
-+}
-+
-+func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
-+	e2eskipper.SkipUnlessProviderIs("gce", "gke")
-+	if pattern.FeatureTag == "[sig-windows]" {
-+		e2eskipper.SkipUnlessNodeOSDistroIs("windows")
-+	}
-+}
-+
-+func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
-+	gv, ok := e2evolume.(*gcePdVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
-+	volSource := v1.VolumeSource{
-+		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
-+			PDName:   gv.volumeName,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		volSource.GCEPersistentDisk.FSType = fsType
-+	}
-+	return &volSource
-+}
-+
-+func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
-+	gv, ok := e2evolume.(*gcePdVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
-+	pvSource := v1.PersistentVolumeSource{
-+		GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
-+			PDName:   gv.volumeName,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		pvSource.GCEPersistentDisk.FSType = fsType
-+	}
-+	return &pvSource, nil
-+}
-+
-+func (g *gcePdDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
-+	provisioner := "kubernetes.io/gce-pd"
-+	parameters := map[string]string{}
-+	if fsType != "" {
-+		parameters["fsType"] = fsType
-+	}
-+	ns := config.Framework.Namespace.Name
-+	suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
-+	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
-+
-+	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
-+}
-+
-+func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-+	config := &testsuites.PerTestConfig{
-+		Driver:    g,
-+		Prefix:    "gcepd",
-+		Framework: f,
-+	}
-+
-+	if framework.NodeOSDistroIs("windows") {
-+		config.ClientNodeSelection = e2epod.NodeSelection{
-+			Selector: map[string]string{
-+				"kubernetes.io/os": "windows",
-+			},
-+		}
-+	}
-+	return config, func() {}
-+
-+}
-+
-+func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
-+	zone := getInlineVolumeZone(config.Framework)
-+	if volType == testpatterns.InlineVolume {
-+		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
-+		// so pods should be also scheduled there.
-+		config.ClientNodeSelection = e2epod.NodeSelection{
-+			Selector: map[string]string{
-+				v1.LabelZoneFailureDomain: zone,
-+			},
-+		}
-+	}
-+	ginkgo.By("creating a test gce pd volume")
-+	vname, err := e2epv.CreatePDWithRetryAndZone(zone)
-+	framework.ExpectNoError(err)
-+	return &gcePdVolume{
-+		volumeName: vname,
-+	}
-+}
-+
-+func (v *gcePdVolume) DeleteVolume() {
-+	e2epv.DeletePDWithRetry(v.volumeName)
-+}
-+
-+// vSphere
-+type vSphereDriver struct {
-+	driverInfo testsuites.DriverInfo
-+}
-+
-+type vSphereVolume struct {
-+	volumePath string
-+	nodeInfo   *vspheretest.NodeInfo
-+}
-+
-+var _ testsuites.TestDriver = &vSphereDriver{}
-+var _ testsuites.PreprovisionedVolumeTestDriver = &vSphereDriver{}
-+var _ testsuites.InlineVolumeTestDriver = &vSphereDriver{}
-+var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{}
-+var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
-+
-+// InitVSphereDriver returns vSphereDriver that implements TestDriver interface
-+func InitVSphereDriver() testsuites.TestDriver {
-+	return &vSphereDriver{
-+		driverInfo: testsuites.DriverInfo{
-+			Name:             "vsphere",
-+			InTreePluginName: "kubernetes.io/vsphere-volume",
-+			MaxFileSize:      testpatterns.FileSizeMedium,
-+			SupportedSizeRange: e2evolume.SizeRange{
-+				Min: "5Gi",
-+			},
-+			SupportedFsType: sets.NewString(
-+				"", // Default fsType
-+				"ext4",
-+			),
-+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
-+			Capabilities: map[testsuites.Capability]bool{
-+				testsuites.CapPersistence: true,
-+				testsuites.CapFsGroup:     true,
-+				testsuites.CapExec:        true,
-+				testsuites.CapMultiPODs:   true,
-+				testsuites.CapTopology:    true,
-+			},
-+		},
-+	}
-+}
-+func (v *vSphereDriver) GetDriverInfo() *testsuites.DriverInfo {
-+	return &v.driverInfo
-+}
-+
-+func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
-+	e2eskipper.SkipUnlessProviderIs("vsphere")
-+}
-+
-+func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
-+	vsv, ok := e2evolume.(*vSphereVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
-+
-+	// vSphere driver doesn't seem to support readOnly volume
-+	// TODO: check if it is correct
-+	if readOnly {
-+		return nil
-+	}
-+	volSource := v1.VolumeSource{
-+		VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
-+			VolumePath: vsv.volumePath,
-+		},
-+	}
-+	if fsType != "" {
-+		volSource.VsphereVolume.FSType = fsType
-+	}
-+	return &volSource
-+}
-+
-+func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
-+	vsv, ok := e2evolume.(*vSphereVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
-+
-+	// vSphere driver doesn't seem to support readOnly volume
-+	// TODO: check if it is correct
-+	if readOnly {
-+		return nil, nil
-+	}
-+	pvSource := v1.PersistentVolumeSource{
-+		VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
-+			VolumePath: vsv.volumePath,
-+		},
-+	}
-+	if fsType != "" {
-+		pvSource.VsphereVolume.FSType = fsType
-+	}
-+	return &pvSource, nil
-+}
-+
-+func (v *vSphereDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
-+	provisioner := "kubernetes.io/vsphere-volume"
-+	parameters := map[string]string{}
-+	if fsType != "" {
-+		parameters["fsType"] = fsType
-+	}
-+	ns := config.Framework.Namespace.Name
-+	suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name)
-+
-+	return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
-+}
-+
-+func (v *vSphereDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-+	return &testsuites.PerTestConfig{
-+		Driver:    v,
-+		Prefix:    "vsphere",
-+		Framework: f,
-+	}, func() {}
-+}
-+
-+func (v *vSphereDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
-+	f := config.Framework
-+	vspheretest.Bootstrap(f)
-+	nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo()
-+	volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef)
-+	framework.ExpectNoError(err)
-+	return &vSphereVolume{
-+		volumePath: volumePath,
-+		nodeInfo:   nodeInfo,
-+	}
-+}
-+
-+func (v *vSphereVolume) DeleteVolume() {
-+	v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef)
-+}
-+
-+// Azure Disk
-+type azureDiskDriver struct {
-+	driverInfo testsuites.DriverInfo
-+}
-+
-+type azureDiskVolume struct {
-+	volumeName string
-+}
-+
-+var _ testsuites.TestDriver = &azureDiskDriver{}
-+var _ testsuites.PreprovisionedVolumeTestDriver = &azureDiskDriver{}
-+var _ testsuites.InlineVolumeTestDriver = &azureDiskDriver{}
-+var _ testsuites.PreprovisionedPVTestDriver = &azureDiskDriver{}
-+var _ testsuites.DynamicPVTestDriver = &azureDiskDriver{}
-+
-+// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
-+func InitAzureDiskDriver() testsuites.TestDriver {
-+	return &azureDiskDriver{
-+		driverInfo: testsuites.DriverInfo{
-+			Name:             "azure-disk",
-+			InTreePluginName: "kubernetes.io/azure-disk",
-+			MaxFileSize:      testpatterns.FileSizeMedium,
-+			SupportedSizeRange: e2evolume.SizeRange{
-+				Min: "5Gi",
-+			},
-+			SupportedFsType: sets.NewString(
-+				"", // Default fsType
-+				"ext3",
-+				"ext4",
-+				"xfs",
-+			),
-+			TopologyKeys: []string{v1.LabelZoneFailureDomain},
-+			Capabilities: map[testsuites.Capability]bool{
-+				testsuites.CapPersistence: true,
-+				testsuites.CapFsGroup:     true,
-+				testsuites.CapBlock:       true,
-+				testsuites.CapExec:        true,
-+				testsuites.CapMultiPODs:   true,
-+				// Azure supports volume limits, but the test creates large
-+				// number of volumes and times out test suites.
-+				testsuites.CapVolumeLimits: false,
-+				testsuites.CapTopology:     true,
-+			},
-+		},
-+	}
-+}
-+
-+func (a *azureDiskDriver) GetDriverInfo() *testsuites.DriverInfo {
-+	return &a.driverInfo
-+}
-+
-+func (a *azureDiskDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
-+	e2eskipper.SkipUnlessProviderIs("azure")
-+}
-+
-+func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
-+	av, ok := e2evolume.(*azureDiskVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
-+	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
-+
-+	kind := v1.AzureManagedDisk
-+	volSource := v1.VolumeSource{
-+		AzureDisk: &v1.AzureDiskVolumeSource{
-+			DiskName:    diskName,
-+			DataDiskURI: av.volumeName,
-+			Kind:        &kind,
-+			ReadOnly:    &readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		volSource.AzureDisk.FSType = &fsType
-+	}
-+	return &volSource
-+}
-+
-+func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
-+	av, ok := e2evolume.(*azureDiskVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
-+
-+	diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
-+
-+	kind := v1.AzureManagedDisk
-+	pvSource := v1.PersistentVolumeSource{
-+		AzureDisk: &v1.AzureDiskVolumeSource{
-+			DiskName:    diskName,
-+			DataDiskURI: av.volumeName,
-+			Kind:        &kind,
-+			ReadOnly:    &readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		pvSource.AzureDisk.FSType = &fsType
-+	}
-+	return &pvSource, nil
-+}
-+
-+func (a *azureDiskDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
-+	provisioner := "kubernetes.io/azure-disk"
-+	parameters := map[string]string{}
-+	if fsType != "" {
-+		parameters["fsType"] = fsType
-+	}
-+	ns := config.Framework.Namespace.Name
-+	suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
-+	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
-+
-+	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
-+}
-+
-+func (a *azureDiskDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-+	return &testsuites.PerTestConfig{
-+		Driver:    a,
-+		Prefix:    "azure",
-+		Framework: f,
-+	}, func() {}
-+}
-+
-+func (a *azureDiskDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
-+	ginkgo.By("creating a test azure disk volume")
-+	zone := getInlineVolumeZone(config.Framework)
-+	if volType == testpatterns.InlineVolume {
-+		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
-+		// so pods should be also scheduled there.
-+		config.ClientNodeSelection = e2epod.NodeSelection{
-+			Selector: map[string]string{
-+				v1.LabelZoneFailureDomain: zone,
-+			},
-+		}
-+	}
-+	volumeName, err := e2epv.CreatePDWithRetryAndZone(zone)
-+	framework.ExpectNoError(err)
-+	return &azureDiskVolume{
-+		volumeName: volumeName,
-+	}
-+}
-+
-+func (v *azureDiskVolume) DeleteVolume() {
-+	e2epv.DeletePDWithRetry(v.volumeName)
-+}
-+
-+// AWS
-+type awsDriver struct {
-+	driverInfo testsuites.DriverInfo
-+}
-+
-+type awsVolume struct {
-+	volumeName string
-+}
-+
-+var _ testsuites.TestDriver = &awsDriver{}
-+
-+var _ testsuites.PreprovisionedVolumeTestDriver = &awsDriver{}
-+var _ testsuites.InlineVolumeTestDriver = &awsDriver{}
-+var _ testsuites.PreprovisionedPVTestDriver = &awsDriver{}
-+var _ testsuites.DynamicPVTestDriver = &awsDriver{}
-+
-+// InitAwsDriver returns awsDriver that implements TestDriver interface
-+func InitAwsDriver() testsuites.TestDriver {
-+	return &awsDriver{
-+		driverInfo: testsuites.DriverInfo{
-+			Name:             "aws",
-+			InTreePluginName: "kubernetes.io/aws-ebs",
-+			MaxFileSize:      testpatterns.FileSizeMedium,
-+			SupportedSizeRange: e2evolume.SizeRange{
-+				Min: "5Gi",
-+			},
-+			SupportedFsType: sets.NewString(
-+				"", // Default fsType
-+				"ext2",
-+				"ext3",
-+				"ext4",
-+				"xfs",
-+				"ntfs",
-+			),
-+			SupportedMountOption: sets.NewString("debug", "nouid32"),
-+			TopologyKeys:         []string{v1.LabelZoneFailureDomain},
-+			Capabilities: map[testsuites.Capability]bool{
-+				testsuites.CapPersistence:         true,
-+				testsuites.CapFsGroup:             true,
-+				testsuites.CapBlock:               true,
-+				testsuites.CapExec:                true,
-+				testsuites.CapMultiPODs:           true,
-+				testsuites.CapControllerExpansion: true,
-+				testsuites.CapNodeExpansion:       true,
-+				// AWS supports volume limits, but the test creates large
-+				// number of volumes and times out test suites.
-+				testsuites.CapVolumeLimits: false,
-+				testsuites.CapTopology:     true,
-+			},
-+		},
-+	}
-+}
-+
-+func (a *awsDriver) GetDriverInfo() *testsuites.DriverInfo {
-+	return &a.driverInfo
-+}
-+
-+func (a *awsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
-+	e2eskipper.SkipUnlessProviderIs("aws")
-+}
-+
-+func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
-+	av, ok := e2evolume.(*awsVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
-+	volSource := v1.VolumeSource{
-+		AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
-+			VolumeID: av.volumeName,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		volSource.AWSElasticBlockStore.FSType = fsType
-+	}
-+	return &volSource
-+}
-+
-+func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
-+	av, ok := e2evolume.(*awsVolume)
-+	framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
-+	pvSource := v1.PersistentVolumeSource{
-+		AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
-+			VolumeID: av.volumeName,
-+			ReadOnly: readOnly,
-+		},
-+	}
-+	if fsType != "" {
-+		pvSource.AWSElasticBlockStore.FSType = fsType
-+	}
-+	return &pvSource, nil
-+}
-+
-+func (a *awsDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
-+	provisioner := "kubernetes.io/aws-ebs"
-+	parameters := map[string]string{}
-+	if fsType != "" {
-+		parameters["fsType"] = fsType
-+	}
-+	ns := config.Framework.Namespace.Name
-+	suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
-+	delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
-+
-+	return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
-+}
-+
-+func (a *awsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
-+	config := &testsuites.PerTestConfig{
-+		Driver:    a,
-+		Prefix:    "aws",
-+		Framework: f,
-+	}
-+
-+	if framework.NodeOSDistroIs("windows") {
-+		config.ClientNodeSelection = e2epod.NodeSelection{
-+			Selector: map[string]string{
-+				"kubernetes.io/os": "windows",
-+			},
-+		}
-+	}
-+	return config, func() {}
-+}
-+
-+func (a *awsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
-+	zone := getInlineVolumeZone(config.Framework)
-+	if volType == testpatterns.InlineVolume {
-+		// PD will be created in framework.TestContext.CloudConfig.Zone zone,
-+		// so pods should be also scheduled there.
-+		config.ClientNodeSelection = e2epod.NodeSelection{
-+			Selector: map[string]string{
-+				v1.LabelZoneFailureDomain: zone,
-+			},
-+		}
-+	}
-+	ginkgo.By("creating a test aws volume")
-+	vname, err := e2epv.CreatePDWithRetryAndZone(zone)
-+	framework.ExpectNoError(err)
-+	return &awsVolume{
-+		volumeName: vname,
-+	}
-+}
-+
-+func (v *awsVolume) DeleteVolume() {
-+	e2epv.DeletePDWithRetry(v.volumeName)
-+}
-diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go
-index 19372062407..8322db743cd 100644
---- a/test/e2e/storage/in_tree_volumes.go
-+++ b/test/e2e/storage/in_tree_volumes.go
-@@ -33,11 +33,6 @@ var testDrivers = []func() testsuites.TestDriver{
- 	drivers.InitHostPathDriver,
- 	drivers.InitHostPathSymlinkDriver,
- 	drivers.InitEmptydirDriver,
--	drivers.InitCinderDriver,
--	drivers.InitGcePdDriver,
--	drivers.InitVSphereDriver,
--	drivers.InitAzureDiskDriver,
--	drivers.InitAwsDriver,
- 	drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectory),
- 	drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryLink),
- 	drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryBindMounted),
-diff --git a/test/e2e/storage/in_tree_volumes_providers.go b/test/e2e/storage/in_tree_volumes_providers.go
-new file mode 100644
-index 00000000000..d6a5dbca191
---- /dev/null
-+++ b/test/e2e/storage/in_tree_volumes_providers.go
-@@ -0,0 +1,46 @@
-+// +build !providerless
-+
-+/*
-+Copyright 2020 The Kubernetes Authors.
-+
-+Licensed under the Apache License, Version 2.0 (the "License");
-+you may not use this file except in compliance with the License.
-+You may obtain a copy of the License at
-+
-+    http://www.apache.org/licenses/LICENSE-2.0
-+
-+Unless required by applicable law or agreed to in writing, software
-+distributed under the License is distributed on an "AS IS" BASIS,
-+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+See the License for the specific language governing permissions and
-+limitations under the License.
-+*/
-+
-+package storage
-+
-+import (
-+	"github.com/onsi/ginkgo"
-+	"k8s.io/kubernetes/test/e2e/storage/drivers"
-+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
-+	"k8s.io/kubernetes/test/e2e/storage/utils"
-+)
-+
-+// List of testDrivers to be executed in below loop
-+var testDriversProviders = []func() testsuites.TestDriver{
-+	drivers.InitCinderDriver,
-+	drivers.InitGcePdDriver,
-+	drivers.InitVSphereDriver,
-+	drivers.InitAzureDiskDriver,
-+	drivers.InitAwsDriver,
-+}
-+
-+// This executes testSuites for in-tree volumes.
-+var _ = utils.SIGDescribe("In-tree Volumes for Cloud Providers", func() {
-+	for _, initDriver := range testDriversProviders {
-+		curDriver := initDriver()
-+
-+		ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() {
-+			testsuites.DefineTestSuite(curDriver, testsuites.BaseSuites)
-+		})
-+	}
-+})
-diff --git a/test/e2e/storage/nfs_persistent_volume-disruptive.go b/test/e2e/storage/nfs_persistent_volume-disruptive.go
-index 5afebb5e903..b197eee99a6 100644
---- a/test/e2e/storage/nfs_persistent_volume-disruptive.go
-+++ b/test/e2e/storage/nfs_persistent_volume-disruptive.go
-@@ -91,7 +91,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() {
- 
- 	ginkgo.BeforeEach(func() {
- 		// To protect the NFS volume pod from the kubelet restart, we isolate it on its own node.
--		e2eskipper.SkipUnlessNodeCountIsAtLeast(minNodes)
-+		e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
- 		e2eskipper.SkipIfProviderIs("local")
- 
- 		c = f.ClientSet
-diff --git a/test/e2e/storage/pd.go b/test/e2e/storage/pd.go
-index f5b6060a834..addd304147c 100644
---- a/test/e2e/storage/pd.go
-+++ b/test/e2e/storage/pd.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2015 The Kubernetes Authors.
- 
-diff --git a/test/e2e/storage/persistent_volumes-gce.go b/test/e2e/storage/persistent_volumes-gce.go
-index b8bc887384e..f572754c5e8 100644
---- a/test/e2e/storage/persistent_volumes-gce.go
-+++ b/test/e2e/storage/persistent_volumes-gce.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2017 The Kubernetes Authors.
- 
-diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go
-index 7763afaf6b1..a042dcc9d4a 100644
---- a/test/e2e/storage/regional_pd.go
-+++ b/test/e2e/storage/regional_pd.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2016 The Kubernetes Authors.
- 
-@@ -18,6 +20,7 @@ package storage
- 
- import (
- 	"context"
-+
- 	"github.com/onsi/ginkgo"
- 	"github.com/onsi/gomega"
- 
-diff --git a/test/e2e/storage/utils/BUILD b/test/e2e/storage/utils/BUILD
-index bdc78982dcb..c57ff325f76 100644
---- a/test/e2e/storage/utils/BUILD
-+++ b/test/e2e/storage/utils/BUILD
-@@ -7,7 +7,6 @@ go_library(
-     srcs = [
-         "create.go",
-         "deployment.go",
--        "ebs.go",
-         "framework.go",
-         "host_exec.go",
-         "local.go",
-@@ -37,8 +36,6 @@ go_library(
-         "//test/e2e/framework/ssh:go_default_library",
-         "//test/e2e/framework/testfiles:go_default_library",
-         "//test/utils/image:go_default_library",
--        "//vendor/github.com/aws/aws-sdk-go/aws:go_default_library",
--        "//vendor/github.com/aws/aws-sdk-go/service/ec2:go_default_library",
-         "//vendor/github.com/onsi/ginkgo:go_default_library",
-         "//vendor/github.com/onsi/gomega:go_default_library",
-         "//vendor/github.com/pkg/errors:go_default_library",
-diff --git a/test/e2e/storage/utils/ebs.go b/test/e2e/storage/utils/ebs.go
-index 39e223f36aa..55065ea07b7 100644
---- a/test/e2e/storage/utils/ebs.go
-+++ b/test/e2e/storage/utils/ebs.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2020 The Kubernetes Authors.
- 
-diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go
-index a8b494eb3ac..c070a81283c 100644
---- a/test/e2e/storage/volume_provisioning.go
-+++ b/test/e2e/storage/volume_provisioning.go
-@@ -24,11 +24,6 @@ import (
- 	"time"
- 
- 	"github.com/onsi/ginkgo"
--	"github.com/onsi/gomega"
--
--	"github.com/aws/aws-sdk-go/aws"
--	"github.com/aws/aws-sdk-go/aws/session"
--	"github.com/aws/aws-sdk-go/service/ec2"
- 
- 	v1 "k8s.io/api/core/v1"
- 	rbacv1 "k8s.io/api/rbac/v1"
-@@ -37,9 +32,7 @@ import (
- 	apierrors "k8s.io/apimachinery/pkg/api/errors"
- 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
- 	"k8s.io/apimachinery/pkg/runtime/schema"
--	"k8s.io/apimachinery/pkg/types"
- 	"k8s.io/apimachinery/pkg/util/rand"
--	"k8s.io/apimachinery/pkg/util/sets"
- 	"k8s.io/apimachinery/pkg/util/wait"
- 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
- 	clientset "k8s.io/client-go/kubernetes"
-@@ -48,7 +41,6 @@ import (
- 	e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
- 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
- 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
--	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
- 	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
- 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
- 	"k8s.io/kubernetes/test/e2e/storage/testsuites"
-@@ -61,80 +53,6 @@ const (
- 	externalPluginName = "example.com/nfs"
- )
- 
--// checkAWSEBS checks properties of an AWS EBS. Test framework does not
--// instantiate full AWS provider, therefore we need use ec2 API directly.
--func checkAWSEBS(volume *v1.PersistentVolume, volumeType string, encrypted bool) error {
--	diskName := volume.Spec.AWSElasticBlockStore.VolumeID
--
--	var client *ec2.EC2
--
--	tokens := strings.Split(diskName, "/")
--	volumeID := tokens[len(tokens)-1]
--
--	zone := framework.TestContext.CloudConfig.Zone
--
--	awsSession, err := session.NewSession()
--	if err != nil {
--		return fmt.Errorf("error creating session: %v", err)
--	}
--
--	if len(zone) > 0 {
--		region := zone[:len(zone)-1]
--		cfg := aws.Config{Region: &region}
--		framework.Logf("using region %s", region)
--		client = ec2.New(awsSession, &cfg)
--	} else {
--		framework.Logf("no region configured")
--		client = ec2.New(awsSession)
--	}
--
--	request := &ec2.DescribeVolumesInput{
--		VolumeIds: []*string{&volumeID},
--	}
--	info, err := client.DescribeVolumes(request)
--	if err != nil {
--		return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
--	}
--	if len(info.Volumes) == 0 {
--		return fmt.Errorf("no volumes found for volume %q", volumeID)
--	}
--	if len(info.Volumes) > 1 {
--		return fmt.Errorf("multiple volumes found for volume %q", volumeID)
--	}
--
--	awsVolume := info.Volumes[0]
--	if awsVolume.VolumeType == nil {
--		return fmt.Errorf("expected volume type %q, got nil", volumeType)
--	}
--	if *awsVolume.VolumeType != volumeType {
--		return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
--	}
--	if encrypted && awsVolume.Encrypted == nil {
--		return fmt.Errorf("expected encrypted volume, got no encryption")
--	}
--	if encrypted && !*awsVolume.Encrypted {
--		return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
--	}
--	return nil
--}
--
--func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
--	cloud, err := gce.GetGCECloud()
--	if err != nil {
--		return err
--	}
--	diskName := volume.Spec.GCEPersistentDisk.PDName
--	disk, err := cloud.GetDiskByNameUnknownZone(diskName)
--	if err != nil {
--		return err
--	}
--
--	if !strings.HasSuffix(disk.Type, volumeType) {
--		return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
--	}
--	return nil
--}
--
- var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
- 	f := framework.NewDefaultFramework("volume-provisioning")
- 
-@@ -147,451 +65,6 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
- 		ns = f.Namespace.Name
- 	})
- 
--	ginkgo.Describe("DynamicProvisioner [Slow]", func() {
--		ginkgo.It("should provision storage with different parameters", func() {
--
--			// This test checks that dynamic provisioning can provision a volume
--			// that can be used to persist data among pods.
--			tests := []testsuites.StorageClassTest{
--				// GCE/GKE
--				{
--					Name:           "SSD PD on GCE/GKE",
--					CloudProviders: []string{"gce", "gke"},
--					Provisioner:    "kubernetes.io/gce-pd",
--					Parameters: map[string]string{
--						"type": "pd-ssd",
--						"zone": getRandomClusterZone(c),
--					},
--					ClaimSize:    "1.5Gi",
--					ExpectedSize: "2Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkGCEPD(volume, "pd-ssd")
--						framework.ExpectNoError(err, "checkGCEPD pd-ssd")
--					},
--				},
--				{
--					Name:           "HDD PD on GCE/GKE",
--					CloudProviders: []string{"gce", "gke"},
--					Provisioner:    "kubernetes.io/gce-pd",
--					Parameters: map[string]string{
--						"type": "pd-standard",
--					},
--					ClaimSize:    "1.5Gi",
--					ExpectedSize: "2Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkGCEPD(volume, "pd-standard")
--						framework.ExpectNoError(err, "checkGCEPD pd-standard")
--					},
--				},
--				// AWS
--				{
--					Name:           "gp2 EBS on AWS",
--					CloudProviders: []string{"aws"},
--					Provisioner:    "kubernetes.io/aws-ebs",
--					Parameters: map[string]string{
--						"type": "gp2",
--						"zone": getRandomClusterZone(c),
--					},
--					ClaimSize:    "1.5Gi",
--					ExpectedSize: "2Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkAWSEBS(volume, "gp2", false)
--						framework.ExpectNoError(err, "checkAWSEBS gp2")
--					},
--				},
--				{
--					Name:           "io1 EBS on AWS",
--					CloudProviders: []string{"aws"},
--					Provisioner:    "kubernetes.io/aws-ebs",
--					Parameters: map[string]string{
--						"type":      "io1",
--						"iopsPerGB": "50",
--					},
--					ClaimSize:    "3.5Gi",
--					ExpectedSize: "4Gi", // 4 GiB is minimum for io1
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkAWSEBS(volume, "io1", false)
--						framework.ExpectNoError(err, "checkAWSEBS io1")
--					},
--				},
--				{
--					Name:           "sc1 EBS on AWS",
--					CloudProviders: []string{"aws"},
--					Provisioner:    "kubernetes.io/aws-ebs",
--					Parameters: map[string]string{
--						"type": "sc1",
--					},
--					ClaimSize:    "500Gi", // minimum for sc1
--					ExpectedSize: "500Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkAWSEBS(volume, "sc1", false)
--						framework.ExpectNoError(err, "checkAWSEBS sc1")
--					},
--				},
--				{
--					Name:           "st1 EBS on AWS",
--					CloudProviders: []string{"aws"},
--					Provisioner:    "kubernetes.io/aws-ebs",
--					Parameters: map[string]string{
--						"type": "st1",
--					},
--					ClaimSize:    "500Gi", // minimum for st1
--					ExpectedSize: "500Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkAWSEBS(volume, "st1", false)
--						framework.ExpectNoError(err, "checkAWSEBS st1")
--					},
--				},
--				{
--					Name:           "encrypted EBS on AWS",
--					CloudProviders: []string{"aws"},
--					Provisioner:    "kubernetes.io/aws-ebs",
--					Parameters: map[string]string{
--						"encrypted": "true",
--					},
--					ClaimSize:    "1Gi",
--					ExpectedSize: "1Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--						err := checkAWSEBS(volume, "gp2", true)
--						framework.ExpectNoError(err, "checkAWSEBS gp2 encrypted")
--					},
--				},
--				// OpenStack generic tests (works on all OpenStack deployments)
--				{
--					Name:           "generic Cinder volume on OpenStack",
--					CloudProviders: []string{"openstack"},
--					Provisioner:    "kubernetes.io/cinder",
--					Parameters:     map[string]string{},
--					ClaimSize:      "1.5Gi",
--					ExpectedSize:   "2Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--					},
--				},
--				{
--					Name:           "Cinder volume with empty volume type and zone on OpenStack",
--					CloudProviders: []string{"openstack"},
--					Provisioner:    "kubernetes.io/cinder",
--					Parameters: map[string]string{
--						"type":         "",
--						"availability": "",
--					},
--					ClaimSize:    "1.5Gi",
--					ExpectedSize: "2Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--					},
--				},
--				// vSphere generic test
--				{
--					Name:           "generic vSphere volume",
--					CloudProviders: []string{"vsphere"},
--					Provisioner:    "kubernetes.io/vsphere-volume",
--					Parameters:     map[string]string{},
--					ClaimSize:      "1.5Gi",
--					ExpectedSize:   "1.5Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--					},
--				},
--				// Azure
--				{
--					Name:           "Azure disk volume with empty sku and location",
--					CloudProviders: []string{"azure"},
--					Provisioner:    "kubernetes.io/azure-disk",
--					Parameters:     map[string]string{},
--					ClaimSize:      "1Gi",
--					ExpectedSize:   "1Gi",
--					PvCheck: func(claim *v1.PersistentVolumeClaim) {
--						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--					},
--				},
--			}
--
--			var betaTest *testsuites.StorageClassTest
--			for i, t := range tests {
--				// Beware of clojure, use local variables instead of those from
--				// outer scope
--				test := t
--
--				if !framework.ProviderIs(test.CloudProviders...) {
--					framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
--					continue
--				}
--
--				// Remember the last supported test for subsequent test of beta API
--				betaTest = &test
--
--				ginkgo.By("Testing " + test.Name)
--				suffix := fmt.Sprintf("%d", i)
--				test.Client = c
--				test.Class = newStorageClass(test, ns, suffix)
--				test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
--					ClaimSize:        test.ClaimSize,
--					StorageClassName: &test.Class.Name,
--					VolumeMode:       &test.VolumeMode,
--				}, ns)
--				test.TestDynamicProvisioning()
--			}
--
--			// Run the last test with storage.k8s.io/v1beta1 on pvc
--			if betaTest != nil {
--				ginkgo.By("Testing " + betaTest.Name + " with beta volume provisioning")
--				class := newBetaStorageClass(*betaTest, "beta")
--				// we need to create the class manually, testDynamicProvisioning does not accept beta class
--				class, err := c.StorageV1beta1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
--				framework.ExpectNoError(err)
--				defer deleteStorageClass(c, class.Name)
--
--				betaTest.Client = c
--				betaTest.Class = nil
--				betaTest.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
--					ClaimSize:        betaTest.ClaimSize,
--					StorageClassName: &class.Name,
--					VolumeMode:       &betaTest.VolumeMode,
--				}, ns)
--				betaTest.Claim.Spec.StorageClassName = &(class.Name)
--				(*betaTest).TestDynamicProvisioning()
--			}
--		})
--
--		ginkgo.It("should provision storage with non-default reclaim policy Retain", func() {
--			e2eskipper.SkipUnlessProviderIs("gce", "gke")
--
--			test := testsuites.StorageClassTest{
--				Client:         c,
--				Name:           "HDD PD on GCE/GKE",
--				CloudProviders: []string{"gce", "gke"},
--				Provisioner:    "kubernetes.io/gce-pd",
--				Parameters: map[string]string{
--					"type": "pd-standard",
--				},
--				ClaimSize:    "1Gi",
--				ExpectedSize: "1Gi",
--				PvCheck: func(claim *v1.PersistentVolumeClaim) {
--					volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
--					gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
--
--					err := checkGCEPD(volume, "pd-standard")
--					framework.ExpectNoError(err, "checkGCEPD")
--				},
--			}
--			test.Class = newStorageClass(test, ns, "reclaimpolicy")
--			retain := v1.PersistentVolumeReclaimRetain
--			test.Class.ReclaimPolicy = &retain
--			test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
--				ClaimSize:        test.ClaimSize,
--				StorageClassName: &test.Class.Name,
--				VolumeMode:       &test.VolumeMode,
--			}, ns)
--			pv := test.TestDynamicProvisioning()
--
--			ginkgo.By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
--			framework.ExpectNoError(e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
--
--			ginkgo.By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
--			framework.ExpectNoError(e2epv.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName))
--
--			ginkgo.By(fmt.Sprintf("deleting the PV %q", pv.Name))
--			framework.ExpectNoError(e2epv.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
--			framework.ExpectNoError(e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second))
--		})
--
--		ginkgo.It("should not provision a volume in an unmanaged GCE zone.", func() {
--			e2eskipper.SkipUnlessProviderIs("gce", "gke")
--			var suffix string = "unmananged"
--
--			ginkgo.By("Discovering an unmanaged zone")
--			allZones := sets.NewString() // all zones in the project
--
--			gceCloud, err := gce.GetGCECloud()
--			framework.ExpectNoError(err)
--
--			// Get all k8s managed zones (same as zones with nodes in them for test)
--			managedZones, err := gceCloud.GetAllZonesFromCloudProvider()
--			framework.ExpectNoError(err)
--
--			// Get a list of all zones in the project
--			zones, err := gceCloud.ComputeServices().GA.Zones.List(framework.TestContext.CloudConfig.ProjectID).Do()
--			framework.ExpectNoError(err)
--			for _, z := range zones.Items {
--				allZones.Insert(z.Name)
--			}
--
--			// Get the subset of zones not managed by k8s
--			var unmanagedZone string
--			var popped bool
--			unmanagedZones := allZones.Difference(managedZones)
--			// And select one of them at random.
--			if unmanagedZone, popped = unmanagedZones.PopAny(); !popped {
--				e2eskipper.Skipf("No unmanaged zones found.")
--			}
--
--			ginkgo.By("Creating a StorageClass for the unmanaged zone")
--			test := testsuites.StorageClassTest{
--				Name:        "unmanaged_zone",
--				Provisioner: "kubernetes.io/gce-pd",
--				Parameters:  map[string]string{"zone": unmanagedZone},
--				ClaimSize:   "1Gi",
--			}
--			sc := newStorageClass(test, ns, suffix)
--			sc, err = c.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
--			framework.ExpectNoError(err)
--			defer deleteStorageClass(c, sc.Name)
--
--			ginkgo.By("Creating a claim and expecting it to timeout")
--			pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
--				ClaimSize:        test.ClaimSize,
--				StorageClassName: &sc.Name,
--				VolumeMode:       &test.VolumeMode,
--			}, ns)
--			pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{})
--			framework.ExpectNoError(err)
--			defer func() {
--				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
--			}()
--
--			// The claim should timeout phase:Pending
--			err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
--			framework.ExpectError(err)
--			framework.Logf(err.Error())
--		})
--
--		ginkgo.It("should test that deleting a claim before the volume is provisioned deletes the volume.", func() {
--			// This case tests for the regressions of a bug fixed by PR #21268
--			// REGRESSION: Deleting the PVC before the PV is provisioned can result in the PV
--			// not being deleted.
--			// NOTE:  Polls until no PVs are detected, times out at 5 minutes.
--
--			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
--
--			const raceAttempts int = 100
--			var residualPVs []*v1.PersistentVolume
--			ginkgo.By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
--			test := testsuites.StorageClassTest{
--				Name:        "deletion race",
--				Provisioner: "", // Use a native one based on current cloud provider
--				ClaimSize:   "1Gi",
--			}
--
--			class := newStorageClass(test, ns, "race")
--			class, err := c.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
--			framework.ExpectNoError(err)
--			defer deleteStorageClass(c, class.Name)
--
--			// To increase chance of detection, attempt multiple iterations
--			for i := 0; i < raceAttempts; i++ {
--				prefix := fmt.Sprintf("race-%d", i)
--				claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
--					NamePrefix:       prefix,
--					ClaimSize:        test.ClaimSize,
--					StorageClassName: &class.Name,
--					VolumeMode:       &test.VolumeMode,
--				}, ns)
--				tmpClaim, err := e2epv.CreatePVC(c, ns, claim)
--				framework.ExpectNoError(err)
--				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns))
--			}
--
--			ginkgo.By(fmt.Sprintf("Checking for residual PersistentVolumes associated with StorageClass %s", class.Name))
--			residualPVs, err = waitForProvisionedVolumesDeleted(c, class.Name)
--			// Cleanup the test resources before breaking
--			defer deleteProvisionedVolumesAndDisks(c, residualPVs)
--			framework.ExpectNoError(err, "PersistentVolumes were not deleted as expected. %d remain", len(residualPVs))
--
--			framework.Logf("0 PersistentVolumes remain.")
--		})
--
--		ginkgo.It("deletion should be idempotent", func() {
--			// This test ensures that deletion of a volume is idempotent.
--			// It creates a PV with Retain policy, deletes underlying AWS / GCE
--			// volume and changes the reclaim policy to Delete.
--			// PV controller should delete the PV even though the underlying volume
--			// is already deleted.
--			e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
--			ginkgo.By("creating PD")
--			diskName, err := e2epv.CreatePDWithRetry()
--			framework.ExpectNoError(err)
--
--			ginkgo.By("creating PV")
--			pv := e2epv.MakePersistentVolume(e2epv.PersistentVolumeConfig{
--				NamePrefix: "volume-idempotent-delete-",
--				// Use Retain to keep the PV, the test will change it to Delete
--				// when the time comes.
--				ReclaimPolicy: v1.PersistentVolumeReclaimRetain,
--				AccessModes: []v1.PersistentVolumeAccessMode{
--					v1.ReadWriteOnce,
--				},
--				Capacity: "1Gi",
--				// PV is bound to non-existing PVC, so it's reclaim policy is
--				// executed immediately
--				Prebind: &v1.PersistentVolumeClaim{
--					ObjectMeta: metav1.ObjectMeta{
--						Name:      "dummy-claim-name",
--						Namespace: ns,
--						UID:       types.UID("01234567890"),
--					},
--				},
--			})
--			switch framework.TestContext.Provider {
--			case "aws":
--				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
--					AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
--						VolumeID: diskName,
--					},
--				}
--			case "gce", "gke":
--				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
--					GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
--						PDName: diskName,
--					},
--				}
--			}
--			pv, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
--			framework.ExpectNoError(err)
--
--			ginkgo.By("waiting for the PV to get Released")
--			err = e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 2*time.Second, e2epv.PVReclaimingTimeout)
--			framework.ExpectNoError(err)
--
--			ginkgo.By("deleting the PD")
--			err = e2epv.DeletePVSource(&pv.Spec.PersistentVolumeSource)
--			framework.ExpectNoError(err)
--
--			ginkgo.By("changing the PV reclaim policy")
--			pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
--			framework.ExpectNoError(err)
--			pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
--			pv, err = c.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{})
--			framework.ExpectNoError(err)
--
--			ginkgo.By("waiting for the PV to get deleted")
--			err = e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 5*time.Second, e2epv.PVDeletingTimeout)
--			framework.ExpectNoError(err)
--		})
--	})
--
- 	ginkgo.Describe("DynamicProvisioner External", func() {
- 		ginkgo.It("should let an external dynamic provisioner create and delete persistent volumes [Slow]", func() {
- 			// external dynamic provisioner pods need additional permissions provided by the
-diff --git a/test/e2e/storage/volume_provisioning_providers.go b/test/e2e/storage/volume_provisioning_providers.go
-new file mode 100644
-index 00000000000..932c644af7a
---- /dev/null
-+++ b/test/e2e/storage/volume_provisioning_providers.go
-@@ -0,0 +1,577 @@
-+// +build !providerless
-+
-+/*
-+Copyright 2016 The Kubernetes Authors.
-+
-+Licensed under the Apache License, Version 2.0 (the "License");
-+you may not use this file except in compliance with the License.
-+You may obtain a copy of the License at
-+
-+    http://www.apache.org/licenses/LICENSE-2.0
-+
-+Unless required by applicable law or agreed to in writing, software
-+distributed under the License is distributed on an "AS IS" BASIS,
-+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-+See the License for the specific language governing permissions and
-+limitations under the License.
-+*/
-+
-+package storage
-+
-+import (
-+	"context"
-+	"fmt"
-+	"strings"
-+	"time"
-+
-+	"github.com/aws/aws-sdk-go/aws"
-+	"github.com/aws/aws-sdk-go/aws/session"
-+	"github.com/aws/aws-sdk-go/service/ec2"
-+	"github.com/onsi/ginkgo"
-+	"github.com/onsi/gomega"
-+
-+	v1 "k8s.io/api/core/v1"
-+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-+	"k8s.io/apimachinery/pkg/types"
-+	"k8s.io/apimachinery/pkg/util/sets"
-+	clientset "k8s.io/client-go/kubernetes"
-+	"k8s.io/kubernetes/test/e2e/framework"
-+	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
-+	"k8s.io/kubernetes/test/e2e/framework/providers/gce"
-+	e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
-+	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
-+	"k8s.io/kubernetes/test/e2e/storage/testsuites"
-+	"k8s.io/kubernetes/test/e2e/storage/utils"
-+)
-+
-+// checkAWSEBS checks properties of an AWS EBS. Test framework does not
-+// instantiate full AWS provider, therefore we need use ec2 API directly.
-+func checkAWSEBS(volume *v1.PersistentVolume, volumeType string, encrypted bool) error {
-+	diskName := volume.Spec.AWSElasticBlockStore.VolumeID
-+
-+	var client *ec2.EC2
-+
-+	tokens := strings.Split(diskName, "/")
-+	volumeID := tokens[len(tokens)-1]
-+
-+	zone := framework.TestContext.CloudConfig.Zone
-+
-+	awsSession, err := session.NewSession()
-+	if err != nil {
-+		return fmt.Errorf("error creating session: %v", err)
-+	}
-+
-+	if len(zone) > 0 {
-+		region := zone[:len(zone)-1]
-+		cfg := aws.Config{Region: &region}
-+		framework.Logf("using region %s", region)
-+		client = ec2.New(awsSession, &cfg)
-+	} else {
-+		framework.Logf("no region configured")
-+		client = ec2.New(awsSession)
-+	}
-+
-+	request := &ec2.DescribeVolumesInput{
-+		VolumeIds: []*string{&volumeID},
-+	}
-+	info, err := client.DescribeVolumes(request)
-+	if err != nil {
-+		return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
-+	}
-+	if len(info.Volumes) == 0 {
-+		return fmt.Errorf("no volumes found for volume %q", volumeID)
-+	}
-+	if len(info.Volumes) > 1 {
-+		return fmt.Errorf("multiple volumes found for volume %q", volumeID)
-+	}
-+
-+	awsVolume := info.Volumes[0]
-+	if awsVolume.VolumeType == nil {
-+		return fmt.Errorf("expected volume type %q, got nil", volumeType)
-+	}
-+	if *awsVolume.VolumeType != volumeType {
-+		return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
-+	}
-+	if encrypted && awsVolume.Encrypted == nil {
-+		return fmt.Errorf("expected encrypted volume, got no encryption")
-+	}
-+	if encrypted && !*awsVolume.Encrypted {
-+		return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
-+	}
-+	return nil
-+}
-+
-+func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
-+	cloud, err := gce.GetGCECloud()
-+	if err != nil {
-+		return err
-+	}
-+	diskName := volume.Spec.GCEPersistentDisk.PDName
-+	disk, err := cloud.GetDiskByNameUnknownZone(diskName)
-+	if err != nil {
-+		return err
-+	}
-+
-+	if !strings.HasSuffix(disk.Type, volumeType) {
-+		return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
-+	}
-+	return nil
-+}
-+
-+var _ = utils.SIGDescribe("Dynamic Provisioning with cloud providers", func() {
-+	f := framework.NewDefaultFramework("volume-provisioning")
-+
-+	// filled in BeforeEach
-+	var c clientset.Interface
-+	var ns string
-+
-+	ginkgo.BeforeEach(func() {
-+		c = f.ClientSet
-+		ns = f.Namespace.Name
-+	})
-+
-+	ginkgo.Describe("DynamicProvisioner [Slow]", func() {
-+		ginkgo.It("should provision storage with different parameters", func() {
-+
-+			// This test checks that dynamic provisioning can provision a volume
-+			// that can be used to persist data among pods.
-+			tests := []testsuites.StorageClassTest{
-+				// GCE/GKE
-+				{
-+					Name:           "SSD PD on GCE/GKE",
-+					CloudProviders: []string{"gce", "gke"},
-+					Provisioner:    "kubernetes.io/gce-pd",
-+					Parameters: map[string]string{
-+						"type": "pd-ssd",
-+						"zone": getRandomClusterZone(c),
-+					},
-+					ClaimSize:    "1.5Gi",
-+					ExpectedSize: "2Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkGCEPD(volume, "pd-ssd")
-+						framework.ExpectNoError(err, "checkGCEPD pd-ssd")
-+					},
-+				},
-+				{
-+					Name:           "HDD PD on GCE/GKE",
-+					CloudProviders: []string{"gce", "gke"},
-+					Provisioner:    "kubernetes.io/gce-pd",
-+					Parameters: map[string]string{
-+						"type": "pd-standard",
-+					},
-+					ClaimSize:    "1.5Gi",
-+					ExpectedSize: "2Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkGCEPD(volume, "pd-standard")
-+						framework.ExpectNoError(err, "checkGCEPD pd-standard")
-+					},
-+				},
-+				// AWS
-+				{
-+					Name:           "gp2 EBS on AWS",
-+					CloudProviders: []string{"aws"},
-+					Provisioner:    "kubernetes.io/aws-ebs",
-+					Parameters: map[string]string{
-+						"type": "gp2",
-+						"zone": getRandomClusterZone(c),
-+					},
-+					ClaimSize:    "1.5Gi",
-+					ExpectedSize: "2Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkAWSEBS(volume, "gp2", false)
-+						framework.ExpectNoError(err, "checkAWSEBS gp2")
-+					},
-+				},
-+				{
-+					Name:           "io1 EBS on AWS",
-+					CloudProviders: []string{"aws"},
-+					Provisioner:    "kubernetes.io/aws-ebs",
-+					Parameters: map[string]string{
-+						"type":      "io1",
-+						"iopsPerGB": "50",
-+					},
-+					ClaimSize:    "3.5Gi",
-+					ExpectedSize: "4Gi", // 4 GiB is minimum for io1
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkAWSEBS(volume, "io1", false)
-+						framework.ExpectNoError(err, "checkAWSEBS io1")
-+					},
-+				},
-+				{
-+					Name:           "sc1 EBS on AWS",
-+					CloudProviders: []string{"aws"},
-+					Provisioner:    "kubernetes.io/aws-ebs",
-+					Parameters: map[string]string{
-+						"type": "sc1",
-+					},
-+					ClaimSize:    "500Gi", // minimum for sc1
-+					ExpectedSize: "500Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkAWSEBS(volume, "sc1", false)
-+						framework.ExpectNoError(err, "checkAWSEBS sc1")
-+					},
-+				},
-+				{
-+					Name:           "st1 EBS on AWS",
-+					CloudProviders: []string{"aws"},
-+					Provisioner:    "kubernetes.io/aws-ebs",
-+					Parameters: map[string]string{
-+						"type": "st1",
-+					},
-+					ClaimSize:    "500Gi", // minimum for st1
-+					ExpectedSize: "500Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkAWSEBS(volume, "st1", false)
-+						framework.ExpectNoError(err, "checkAWSEBS st1")
-+					},
-+				},
-+				{
-+					Name:           "encrypted EBS on AWS",
-+					CloudProviders: []string{"aws"},
-+					Provisioner:    "kubernetes.io/aws-ebs",
-+					Parameters: map[string]string{
-+						"encrypted": "true",
-+					},
-+					ClaimSize:    "1Gi",
-+					ExpectedSize: "1Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+						gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+						err := checkAWSEBS(volume, "gp2", true)
-+						framework.ExpectNoError(err, "checkAWSEBS gp2 encrypted")
-+					},
-+				},
-+				// OpenStack generic tests (works on all OpenStack deployments)
-+				{
-+					Name:           "generic Cinder volume on OpenStack",
-+					CloudProviders: []string{"openstack"},
-+					Provisioner:    "kubernetes.io/cinder",
-+					Parameters:     map[string]string{},
-+					ClaimSize:      "1.5Gi",
-+					ExpectedSize:   "2Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+					},
-+				},
-+				{
-+					Name:           "Cinder volume with empty volume type and zone on OpenStack",
-+					CloudProviders: []string{"openstack"},
-+					Provisioner:    "kubernetes.io/cinder",
-+					Parameters: map[string]string{
-+						"type":         "",
-+						"availability": "",
-+					},
-+					ClaimSize:    "1.5Gi",
-+					ExpectedSize: "2Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+					},
-+				},
-+				// vSphere generic test
-+				{
-+					Name:           "generic vSphere volume",
-+					CloudProviders: []string{"vsphere"},
-+					Provisioner:    "kubernetes.io/vsphere-volume",
-+					Parameters:     map[string]string{},
-+					ClaimSize:      "1.5Gi",
-+					ExpectedSize:   "1.5Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+					},
-+				},
-+				// Azure
-+				{
-+					Name:           "Azure disk volume with empty sku and location",
-+					CloudProviders: []string{"azure"},
-+					Provisioner:    "kubernetes.io/azure-disk",
-+					Parameters:     map[string]string{},
-+					ClaimSize:      "1Gi",
-+					ExpectedSize:   "1Gi",
-+					PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+						testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+					},
-+				},
-+			}
-+
-+			var betaTest *testsuites.StorageClassTest
-+			for i, t := range tests {
-+				// Beware of clojure, use local variables instead of those from
-+				// outer scope
-+				test := t
-+
-+				if !framework.ProviderIs(test.CloudProviders...) {
-+					framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
-+					continue
-+				}
-+
-+				// Remember the last supported test for subsequent test of beta API
-+				betaTest = &test
-+
-+				ginkgo.By("Testing " + test.Name)
-+				suffix := fmt.Sprintf("%d", i)
-+				test.Client = c
-+				test.Class = newStorageClass(test, ns, suffix)
-+				test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-+					ClaimSize:        test.ClaimSize,
-+					StorageClassName: &test.Class.Name,
-+					VolumeMode:       &test.VolumeMode,
-+				}, ns)
-+				test.TestDynamicProvisioning()
-+			}
-+
-+			// Run the last test with storage.k8s.io/v1beta1 on pvc
-+			if betaTest != nil {
-+				ginkgo.By("Testing " + betaTest.Name + " with beta volume provisioning")
-+				class := newBetaStorageClass(*betaTest, "beta")
-+				// we need to create the class manually, testDynamicProvisioning does not accept beta class
-+				class, err := c.StorageV1beta1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
-+				framework.ExpectNoError(err)
-+				defer deleteStorageClass(c, class.Name)
-+
-+				betaTest.Client = c
-+				betaTest.Class = nil
-+				betaTest.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-+					ClaimSize:        betaTest.ClaimSize,
-+					StorageClassName: &class.Name,
-+					VolumeMode:       &betaTest.VolumeMode,
-+				}, ns)
-+				betaTest.Claim.Spec.StorageClassName = &(class.Name)
-+				(*betaTest).TestDynamicProvisioning()
-+			}
-+		})
-+
-+		ginkgo.It("should provision storage with non-default reclaim policy Retain", func() {
-+			e2eskipper.SkipUnlessProviderIs("gce", "gke")
-+
-+			test := testsuites.StorageClassTest{
-+				Client:         c,
-+				Name:           "HDD PD on GCE/GKE",
-+				CloudProviders: []string{"gce", "gke"},
-+				Provisioner:    "kubernetes.io/gce-pd",
-+				Parameters: map[string]string{
-+					"type": "pd-standard",
-+				},
-+				ClaimSize:    "1Gi",
-+				ExpectedSize: "1Gi",
-+				PvCheck: func(claim *v1.PersistentVolumeClaim) {
-+					volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
-+					gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
-+
-+					err := checkGCEPD(volume, "pd-standard")
-+					framework.ExpectNoError(err, "checkGCEPD")
-+				},
-+			}
-+			test.Class = newStorageClass(test, ns, "reclaimpolicy")
-+			retain := v1.PersistentVolumeReclaimRetain
-+			test.Class.ReclaimPolicy = &retain
-+			test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-+				ClaimSize:        test.ClaimSize,
-+				StorageClassName: &test.Class.Name,
-+				VolumeMode:       &test.VolumeMode,
-+			}, ns)
-+			pv := test.TestDynamicProvisioning()
-+
-+			ginkgo.By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
-+			framework.ExpectNoError(e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
-+
-+			ginkgo.By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
-+			framework.ExpectNoError(e2epv.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName))
-+
-+			ginkgo.By(fmt.Sprintf("deleting the PV %q", pv.Name))
-+			framework.ExpectNoError(e2epv.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
-+			framework.ExpectNoError(e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second))
-+		})
-+
-+		ginkgo.It("should not provision a volume in an unmanaged GCE zone.", func() {
-+			e2eskipper.SkipUnlessProviderIs("gce", "gke")
-+			var suffix string = "unmananged"
-+
-+			ginkgo.By("Discovering an unmanaged zone")
-+			allZones := sets.NewString() // all zones in the project
-+
-+			gceCloud, err := gce.GetGCECloud()
-+			framework.ExpectNoError(err)
-+
-+			// Get all k8s managed zones (same as zones with nodes in them for test)
-+			managedZones, err := gceCloud.GetAllZonesFromCloudProvider()
-+			framework.ExpectNoError(err)
-+
-+			// Get a list of all zones in the project
-+			zones, err := gceCloud.ComputeServices().GA.Zones.List(framework.TestContext.CloudConfig.ProjectID).Do()
-+			framework.ExpectNoError(err)
-+			for _, z := range zones.Items {
-+				allZones.Insert(z.Name)
-+			}
-+
-+			// Get the subset of zones not managed by k8s
-+			var unmanagedZone string
-+			var popped bool
-+			unmanagedZones := allZones.Difference(managedZones)
-+			// And select one of them at random.
-+			if unmanagedZone, popped = unmanagedZones.PopAny(); !popped {
-+				e2eskipper.Skipf("No unmanaged zones found.")
-+			}
-+
-+			ginkgo.By("Creating a StorageClass for the unmanaged zone")
-+			test := testsuites.StorageClassTest{
-+				Name:        "unmanaged_zone",
-+				Provisioner: "kubernetes.io/gce-pd",
-+				Parameters:  map[string]string{"zone": unmanagedZone},
-+				ClaimSize:   "1Gi",
-+			}
-+			sc := newStorageClass(test, ns, suffix)
-+			sc, err = c.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
-+			framework.ExpectNoError(err)
-+			defer deleteStorageClass(c, sc.Name)
-+
-+			ginkgo.By("Creating a claim and expecting it to timeout")
-+			pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-+				ClaimSize:        test.ClaimSize,
-+				StorageClassName: &sc.Name,
-+				VolumeMode:       &test.VolumeMode,
-+			}, ns)
-+			pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{})
-+			framework.ExpectNoError(err)
-+			defer func() {
-+				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
-+			}()
-+
-+			// The claim should timeout phase:Pending
-+			err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
-+			framework.ExpectError(err)
-+			framework.Logf(err.Error())
-+		})
-+
-+		ginkgo.It("should test that deleting a claim before the volume is provisioned deletes the volume.", func() {
-+			// This case tests for the regressions of a bug fixed by PR #21268
-+			// REGRESSION: Deleting the PVC before the PV is provisioned can result in the PV
-+			// not being deleted.
-+			// NOTE:  Polls until no PVs are detected, times out at 5 minutes.
-+
-+			e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
-+
-+			const raceAttempts int = 100
-+			var residualPVs []*v1.PersistentVolume
-+			ginkgo.By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
-+			test := testsuites.StorageClassTest{
-+				Name:        "deletion race",
-+				Provisioner: "", // Use a native one based on current cloud provider
-+				ClaimSize:   "1Gi",
-+			}
-+
-+			class := newStorageClass(test, ns, "race")
-+			class, err := c.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
-+			framework.ExpectNoError(err)
-+			defer deleteStorageClass(c, class.Name)
-+
-+			// To increase chance of detection, attempt multiple iterations
-+			for i := 0; i < raceAttempts; i++ {
-+				prefix := fmt.Sprintf("race-%d", i)
-+				claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
-+					NamePrefix:       prefix,
-+					ClaimSize:        test.ClaimSize,
-+					StorageClassName: &class.Name,
-+					VolumeMode:       &test.VolumeMode,
-+				}, ns)
-+				tmpClaim, err := e2epv.CreatePVC(c, ns, claim)
-+				framework.ExpectNoError(err)
-+				framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns))
-+			}
-+
-+			ginkgo.By(fmt.Sprintf("Checking for residual PersistentVolumes associated with StorageClass %s", class.Name))
-+			residualPVs, err = waitForProvisionedVolumesDeleted(c, class.Name)
-+			// Cleanup the test resources before breaking
-+			defer deleteProvisionedVolumesAndDisks(c, residualPVs)
-+			framework.ExpectNoError(err, "PersistentVolumes were not deleted as expected. %d remain", len(residualPVs))
-+
-+			framework.Logf("0 PersistentVolumes remain.")
-+		})
-+
-+		ginkgo.It("deletion should be idempotent", func() {
-+			// This test ensures that deletion of a volume is idempotent.
-+			// It creates a PV with Retain policy, deletes underlying AWS / GCE
-+			// volume and changes the reclaim policy to Delete.
-+			// PV controller should delete the PV even though the underlying volume
-+			// is already deleted.
-+			e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
-+			ginkgo.By("creating PD")
-+			diskName, err := e2epv.CreatePDWithRetry()
-+			framework.ExpectNoError(err)
-+
-+			ginkgo.By("creating PV")
-+			pv := e2epv.MakePersistentVolume(e2epv.PersistentVolumeConfig{
-+				NamePrefix: "volume-idempotent-delete-",
-+				// Use Retain to keep the PV, the test will change it to Delete
-+				// when the time comes.
-+				ReclaimPolicy: v1.PersistentVolumeReclaimRetain,
-+				AccessModes: []v1.PersistentVolumeAccessMode{
-+					v1.ReadWriteOnce,
-+				},
-+				Capacity: "1Gi",
-+				// PV is bound to non-existing PVC, so it's reclaim policy is
-+				// executed immediately
-+				Prebind: &v1.PersistentVolumeClaim{
-+					ObjectMeta: metav1.ObjectMeta{
-+						Name:      "dummy-claim-name",
-+						Namespace: ns,
-+						UID:       types.UID("01234567890"),
-+					},
-+				},
-+			})
-+			switch framework.TestContext.Provider {
-+			case "aws":
-+				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
-+					AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
-+						VolumeID: diskName,
-+					},
-+				}
-+			case "gce", "gke":
-+				pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
-+					GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
-+						PDName: diskName,
-+					},
-+				}
-+			}
-+			pv, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
-+			framework.ExpectNoError(err)
-+
-+			ginkgo.By("waiting for the PV to get Released")
-+			err = e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 2*time.Second, e2epv.PVReclaimingTimeout)
-+			framework.ExpectNoError(err)
-+
-+			ginkgo.By("deleting the PD")
-+			err = e2epv.DeletePVSource(&pv.Spec.PersistentVolumeSource)
-+			framework.ExpectNoError(err)
-+
-+			ginkgo.By("changing the PV reclaim policy")
-+			pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
-+			framework.ExpectNoError(err)
-+			pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
-+			pv, err = c.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{})
-+			framework.ExpectNoError(err)
-+
-+			ginkgo.By("waiting for the PV to get deleted")
-+			err = e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 5*time.Second, e2epv.PVDeletingTimeout)
-+			framework.ExpectNoError(err)
-+		})
-+	})
-+})
-diff --git a/test/e2e/upgrades/nvidia-gpu.go b/test/e2e/upgrades/nvidia-gpu.go
-index cf3b8c0cda3..30515197ef7 100644
---- a/test/e2e/upgrades/nvidia-gpu.go
-+++ b/test/e2e/upgrades/nvidia-gpu.go
-@@ -1,3 +1,5 @@
-+// +build !providerless
-+
- /*
- Copyright 2018 The Kubernetes Authors.
- 
--- 
-2.26.2
-
diff --git a/third_party/go/patches/k8s-fix-paths.patch b/third_party/go/patches/k8s-fix-paths.patch
deleted file mode 100644
index ba39a43..0000000
--- a/third_party/go/patches/k8s-fix-paths.patch
+++ /dev/null
@@ -1,56 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From db9ab831cd17f9be540917a77bbb3e0551f4fb4f Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@brun.one>
-Date: Mon, 25 Jan 2021 15:08:20 +0100
-Subject: [PATCH] Make DeviceManager socket relative to RootDir
-
----
- pkg/kubelet/cm/container_manager_linux.go | 2 +-
- pkg/kubelet/cm/devicemanager/manager.go   | 4 ++--
- 2 files changed, 3 insertions(+), 3 deletions(-)
-
-diff --git a/pkg/kubelet/cm/container_manager_linux.go b/pkg/kubelet/cm/container_manager_linux.go
-index 57110ed7745..15cf010074e 100644
---- a/pkg/kubelet/cm/container_manager_linux.go
-+++ b/pkg/kubelet/cm/container_manager_linux.go
-@@ -315,7 +315,7 @@ func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.I
- 
- 	klog.Infof("Creating device plugin manager: %t", devicePluginEnabled)
- 	if devicePluginEnabled {
--		cm.deviceManager, err = devicemanager.NewManagerImpl(numaNodeInfo, cm.topologyManager)
-+		cm.deviceManager, err = devicemanager.NewManagerImpl(numaNodeInfo, cm.topologyManager, nodeConfig.KubeletRootDir)
- 		cm.topologyManager.AddHintProvider(cm.deviceManager)
- 	} else {
- 		cm.deviceManager, err = devicemanager.NewManagerStub()
-diff --git a/pkg/kubelet/cm/devicemanager/manager.go b/pkg/kubelet/cm/devicemanager/manager.go
-index 5d1925f9458..bfff3c50fcc 100644
---- a/pkg/kubelet/cm/devicemanager/manager.go
-+++ b/pkg/kubelet/cm/devicemanager/manager.go
-@@ -124,8 +124,8 @@ func (s *sourcesReadyStub) AddSource(source string) {}
- func (s *sourcesReadyStub) AllReady() bool          { return true }
- 
- // NewManagerImpl creates a new manager.
--func NewManagerImpl(numaNodeInfo cputopology.NUMANodeInfo, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
--	return newManagerImpl(pluginapi.KubeletSocket, numaNodeInfo, topologyAffinityStore)
-+func NewManagerImpl(numaNodeInfo cputopology.NUMANodeInfo, topologyAffinityStore topologymanager.Store, kubeletRootDir string) (*ManagerImpl, error) {
-+	return newManagerImpl(filepath.Join(kubeletRootDir, "device-plugins/kubelet.sock"), numaNodeInfo, topologyAffinityStore)
- }
- 
- func newManagerImpl(socketPath string, numaNodeInfo cputopology.NUMANodeInfo, topologyAffinityStore topologymanager.Store) (*ManagerImpl, error) {
--- 
-2.25.1
-
diff --git a/third_party/go/patches/k8s-infra-bzl4-compat.patch b/third_party/go/patches/k8s-infra-bzl4-compat.patch
deleted file mode 100644
index 4be0fe8..0000000
--- a/third_party/go/patches/k8s-infra-bzl4-compat.patch
+++ /dev/null
@@ -1,82 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 6313aef65ed37aa971737058af391f5be1ae976c Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@nexantic.com>
-Date: Wed, 3 Feb 2021 18:11:03 +0100
-Subject: [PATCH] Don't use run_shell with list as cmd
-
-Going upstream at https://github.com/kubernetes/repo-infra/pull/225
----
- defs/go.bzl | 26 ++++++++++++++------------
- 1 file changed, 14 insertions(+), 12 deletions(-)
-
-diff --git a/defs/go.bzl b/defs/go.bzl
-index 21cffdd..f4617e1 100644
---- a/defs/go.bzl
-+++ b/defs/go.bzl
-@@ -63,16 +63,7 @@ def _go_genrule_impl(ctx):
- 
-     srcs = [src for srcs in ctx.attr.srcs for src in srcs.files.to_list()]
- 
--    deps = depset(
--        gopath_files + srcs,
--        transitive =
--            # tools
--            [dep.files for dep in ctx.attr.tools] +
--            # go toolchain
--            [depset(go.sdk.libs + go.sdk.srcs + go.sdk.tools + [go.sdk.go])],
--    )
--
--    _, cmd, _ = ctx.resolve_command(
-+    inputs, cmd, input_manifests = ctx.resolve_command(
-         command = ctx.attr.cmd,
-         attribute = "cmd",
-         expand_locations = True,
-@@ -83,6 +74,15 @@ def _go_genrule_impl(ctx):
-         tools = ctx.attr.tools,
-     )
- 
-+    deps = depset(
-+        gopath_files + srcs + inputs,
-+        transitive =
-+            # tools
-+            [dep.files for dep in ctx.attr.tools] +
-+            # go toolchain
-+            [depset(go.sdk.libs + go.sdk.srcs + go.sdk.tools + [go.sdk.go])],
-+    )
-+
-     env = dict()
-     env.update(ctx.configuration.default_shell_env)
-     env.update(go.env)
-@@ -92,11 +92,13 @@ def _go_genrule_impl(ctx):
-         "GOROOT": paths.dirname(go.sdk.root_file.path),
-     })
- 
--    ctx.actions.run_shell(
-+    ctx.actions.run(
-         inputs = deps,
-         outputs = ctx.outputs.outs,
-         env = env,
--        command = cmd,
-+        executable = cmd[0],
-+        arguments = cmd[1:],
-+        input_manifests = input_manifests,
-         progress_message = "%s %s" % (ctx.attr.message, ctx),
-         mnemonic = "GoGenrule",
-     )
--- 
-2.25.1
-
diff --git a/third_party/go/patches/k8s-infra-fix-go116.patch b/third_party/go/patches/k8s-infra-fix-go116.patch
deleted file mode 100644
index bf35938..0000000
--- a/third_party/go/patches/k8s-infra-fix-go116.patch
+++ /dev/null
@@ -1,23 +0,0 @@
-From 5cf776eb8f872f78cf7325c39fbfe1777f702407 Mon Sep 17 00:00:00 2001
-From: Mike Danese <mikedanese@google.com>
-Date: Fri, 26 Feb 2021 13:39:16 -0800
-Subject: [PATCH] fix go_genrule for 1.16
-
----
- defs/go.bzl | 3 +++
- 1 file changed, 3 insertions(+)
-
-diff --git a/defs/go.bzl b/defs/go.bzl
-index f4617e1..8873253 100644
---- a/defs/go.bzl
-+++ b/defs/go.bzl
-@@ -90,6 +90,9 @@ def _go_genrule_impl(ctx):
-         "PATH": ctx.configuration.host_path_separator.join(["/usr/local/bin", "/bin", "/usr/bin"]),
-         "GOPATH": paths.dirname(gopath_placeholder.path),
-         "GOROOT": paths.dirname(go.sdk.root_file.path),
-+        # hack to tie us over until we fix this to use modules or stop using
-+        # it.
-+        "GO111MODULE": "off",
-     })
- 
-     ctx.actions.run(
diff --git a/third_party/go/patches/k8s-jose-semver-fix.patch b/third_party/go/patches/k8s-jose-semver-fix.patch
new file mode 100644
index 0000000..423485a
--- /dev/null
+++ b/third_party/go/patches/k8s-jose-semver-fix.patch
@@ -0,0 +1,34 @@
+From b0b42e86e834a1d02fe83f7be3663d19f6a1ee80 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Thu, 17 Mar 2022 16:56:29 +0100
+Subject: [PATCH] Fix for semver breakage in go-jose
+
+---
+ pkg/serviceaccount/claims.go | 4 ++--
+ 1 file changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/pkg/serviceaccount/claims.go b/pkg/serviceaccount/claims.go
+index 1e1475c779f..06620f7a420 100644
+--- a/pkg/serviceaccount/claims.go
++++ b/pkg/serviceaccount/claims.go
+@@ -50,7 +50,7 @@ type kubernetes struct {
+ 	Svcacct   ref             `json:"serviceaccount,omitempty"`
+ 	Pod       *ref            `json:"pod,omitempty"`
+ 	Secret    *ref            `json:"secret,omitempty"`
+-	WarnAfter jwt.NumericDate `json:"warnafter,omitempty"`
++	WarnAfter *jwt.NumericDate `json:"warnafter,omitempty"`
+ }
+ 
+ type ref struct {
+@@ -198,7 +198,7 @@ func (v *validator) Validate(ctx context.Context, _ string, public *jwt.Claims,
+ 
+ 	// Check special 'warnafter' field for projected service account token transition.
+ 	warnafter := private.Kubernetes.WarnAfter
+-	if warnafter != 0 {
++	if warnafter != nil {
+ 		if nowTime.After(warnafter.Time()) {
+ 			secondsAfterWarn := nowTime.Unix() - warnafter.Time().Unix()
+ 			auditInfo := fmt.Sprintf("subject: %s, seconds after warning threshold: %d", public.Subject, secondsAfterWarn)
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/k8s-kubernetes-build.patch b/third_party/go/patches/k8s-kubernetes-build.patch
deleted file mode 100644
index 7debfda..0000000
--- a/third_party/go/patches/k8s-kubernetes-build.patch
+++ /dev/null
@@ -1,108 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-This patch updates BUILD files to reflect changes in Go sources. This only needs to be applied because Gazelle applies patches after BUILDfile generation.
-
-diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go io_k8s_kubernetes/cmd/kubelet/app/plugins.go
---- io_k8s_kubernetes.orig/cmd/kubelet/app/BUILD 13:43:57.827669732 +0200
-+++ io_k8s_kubernetes/cmd/kubelet/app/BUILD 15:12:22.682316924 +0200
-@@ -45,8 +45,6 @@
-         "//pkg/capabilities:go_default_library",
-         "//pkg/cloudprovider/providers:go_default_library",
-         "//pkg/credentialprovider:go_default_library",
--        "//pkg/credentialprovider/aws:go_default_library",
--        "//pkg/credentialprovider/azure:go_default_library",
-         "//pkg/credentialprovider/gcp:go_default_library",
-         "//pkg/features:go_default_library",
-         "//pkg/kubelet:go_default_library",
-@@ -78,25 +76,13 @@
-         "//pkg/util/oom:go_default_library",
-         "//pkg/util/rlimit:go_default_library",
-         "//pkg/volume:go_default_library",
--        "//pkg/volume/cephfs:go_default_library",
-         "//pkg/volume/configmap:go_default_library",
-         "//pkg/volume/csi:go_default_library",
-         "//pkg/volume/downwardapi:go_default_library",
-         "//pkg/volume/emptydir:go_default_library",
--        "//pkg/volume/fc:go_default_library",
-         "//pkg/volume/flexvolume:go_default_library",
--        "//pkg/volume/flocker:go_default_library",
--        "//pkg/volume/git_repo:go_default_library",
--        "//pkg/volume/glusterfs:go_default_library",
-         "//pkg/volume/hostpath:go_default_library",
--        "//pkg/volume/iscsi:go_default_library",
-         "//pkg/volume/local:go_default_library",
--        "//pkg/volume/nfs:go_default_library",
--        "//pkg/volume/portworx:go_default_library",
-         "//pkg/volume/projected:go_default_library",
--        "//pkg/volume/quobyte:go_default_library",
--        "//pkg/volume/rbd:go_default_library",
--        "//pkg/volume/scaleio:go_default_library",
-         "//pkg/volume/secret:go_default_library",
--        "//pkg/volume/storageos:go_default_library",
-         "//pkg/volume/util/hostutil:go_default_library",
---- io_k8s_kubernetes.orig/cmd/kubelet/app/options/BUILD 13:43:57.827669732 +0200
-+++ io_k8s_kubernetes/cmd/kubelet/app/options/BUILD 15:12:22.682316924 +0200
-@@ -20,8 +20,6 @@
-     importpath = "k8s.io/kubernetes/cmd/kubelet/app/options",
-     deps = [
-         "//pkg/apis/core:go_default_library",
--        "//pkg/credentialprovider/azure:go_default_library",
--        "//pkg/credentialprovider/gcp:go_default_library",
-         "//pkg/features:go_default_library",
-         "//pkg/kubelet/apis:go_default_library",
-         "//pkg/kubelet/apis/config:go_default_library",
---- io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/BUILD 13:43:57.827669732 +0200
-+++ io_k8s_kubernetes/pkg/kubelet/cadvisor/BUILD 15:12:22.682316924 +0200
-@@ -37,8 +37,6 @@
-             "@com_github_google_cadvisor//container/systemd/install:go_default_library",
-             "@com_github_google_cadvisor//fs:go_default_library",
-             "@com_github_google_cadvisor//manager:go_default_library",
--            "@com_github_google_cadvisor//utils/cloudinfo/aws:go_default_library",
--            "@com_github_google_cadvisor//utils/cloudinfo/azure:go_default_library",
-             "@com_github_google_cadvisor//utils/cloudinfo/gce:go_default_library",
-             "@com_github_google_cadvisor//utils/sysfs:go_default_library",
-             "@io_k8s_klog//:go_default_library",
-@@ -52,8 +50,6 @@
-             "@com_github_google_cadvisor//container/systemd/install:go_default_library",
-             "@com_github_google_cadvisor//fs:go_default_library",
-             "@com_github_google_cadvisor//manager:go_default_library",
--            "@com_github_google_cadvisor//utils/cloudinfo/aws:go_default_library",
--            "@com_github_google_cadvisor//utils/cloudinfo/azure:go_default_library",
-             "@com_github_google_cadvisor//utils/cloudinfo/gce:go_default_library",
-             "@com_github_google_cadvisor//utils/sysfs:go_default_library",
-             "@io_k8s_klog//:go_default_library",
---- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/BUILD 13:43:57.827669732 +0200
-+++ io_k8s_kubernetes/cmd/kube-controller-manager/app/BUILD 15:12:22.682316924 +0200
-@@ -90,19 +90,9 @@
-         "//pkg/volume:go_default_library",
-         "//pkg/volume/csi:go_default_library",
-         "//pkg/volume/csimigration:go_default_library",
--        "//pkg/volume/fc:go_default_library",
-         "//pkg/volume/flexvolume:go_default_library",
--        "//pkg/volume/flocker:go_default_library",
--        "//pkg/volume/glusterfs:go_default_library",
-         "//pkg/volume/hostpath:go_default_library",
--        "//pkg/volume/iscsi:go_default_library",
-         "//pkg/volume/local:go_default_library",
--        "//pkg/volume/nfs:go_default_library",
--        "//pkg/volume/portworx:go_default_library",
--        "//pkg/volume/quobyte:go_default_library",
--        "//pkg/volume/rbd:go_default_library",
--        "//pkg/volume/scaleio:go_default_library",
--        "//pkg/volume/storageos:go_default_library",
-         "//pkg/volume/util:go_default_library",
-         "@com_github_spf13_cobra//:go_default_library",
-         "@io_k8s_api//core/v1:go_default_library",
diff --git a/third_party/go/patches/k8s-kubernetes.patch b/third_party/go/patches/k8s-kubernetes.patch
deleted file mode 100644
index 0efaa37..0000000
--- a/third_party/go/patches/k8s-kubernetes.patch
+++ /dev/null
@@ -1,399 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-This fixes OpenAPI codegen for when included from the Monogon workspace. It basically undoes vendorification.
-
-diff -ur io_k8s_kubernetes.orig/build/code_generation.bzl io_k8s_kubernetes/build/code_generation.bzl
---- io_k8s_kubernetes.orig/build/code_generation.bzl	2021-01-26 12:10:52.593725692 +0100
-+++ io_k8s_kubernetes/build/code_generation.bzl	2021-01-26 12:11:04.571786562 +0100
-@@ -27,6 +27,12 @@
-         ...
-     )
-     """
-+    if pkg.startswith('staging/src/k8s.io/'):
-+        parts = pkg.split('/', 4)
-+        project = parts[3]
-+        project = project.replace('-', '_')
-+        path = parts[4]
-+        return "@io_k8s_%s//%s:go_default_library" % (project, path)
-     return "//%s:go_default_library" % pkg
- 
- def go_pkg(pkg):
-@@ -42,6 +48,8 @@
-         ...
-     )
-     """
-+    if pkg.startswith('staging/src/'):
-+        return pkg[len('staging/src/'):]
-     for prefix in ["staging/src", "vendor"]:
-         if pkg.startswith(prefix):
-             return paths.relativize(pkg, prefix)
-@@ -49,8 +57,8 @@
- 
- def openapi_deps():
-     deps = [
--        "//vendor/github.com/go-openapi/spec:go_default_library",
--        "//vendor/k8s.io/kube-openapi/pkg/common:go_default_library",
-+        "@com_github_go_openapi_spec//:go_default_library",
-+        "@io_k8s_kube_openapi//pkg/common:go_default_library",
-     ]
-     deps.extend([bazel_go_library(pkg) for pkg in tags_values_pkgs["openapi-gen"]["true"]])
-     return deps
-@@ -76,7 +84,7 @@
-         # the generator must run from the repo root inside the generated GOPATH.
-         # All of bazel's $(location)s are relative to the original working directory, however.
-         cmd = " ".join([
--            "$(location //vendor/k8s.io/kube-openapi/cmd/openapi-gen)",
-+            "$(location @io_k8s_kube_openapi//cmd/openapi-gen)",
-             "--v 1",
-             "--logtostderr",
-             "--go-header-file $(location //" + openapi_vendor_prefix + "hack/boilerplate:boilerplate.generatego.txt)",
-@@ -88,6 +96,6 @@
-             "&& rm tmp_api_violations.report",
-         ]),
-         go_deps = openapi_deps(),
--        tools = ["//vendor/k8s.io/kube-openapi/cmd/openapi-gen"],
-+        tools = ["@io_k8s_kube_openapi//cmd/openapi-gen"],
-         message = "GenOpenAPI",
-     )
-diff -ur io_k8s_kubernetes.orig/cmd/kube-apiserver/app/options/globalflags.go io_k8s_kubernetes/cmd/kube-apiserver/app/options/globalflags.go
---- io_k8s_kubernetes.orig/cmd/kube-apiserver/app/options/globalflags.go	2021-01-26 12:10:52.605725751 +0100
-+++ io_k8s_kubernetes/cmd/kube-apiserver/app/options/globalflags.go	2021-01-26 12:11:04.572786567 +0100
-@@ -32,9 +32,6 @@
- func AddCustomGlobalFlags(fs *pflag.FlagSet) {
- 	// Lookup flags in global flag set and re-register the values with our flagset.
- 
--	// Adds flags from k8s.io/kubernetes/pkg/cloudprovider/providers.
--	registerLegacyGlobalFlags(fs)
--
- 	// Adds flags from k8s.io/apiserver/pkg/admission.
- 	globalflag.Register(fs, "default-not-ready-toleration-seconds")
- 	globalflag.Register(fs, "default-unreachable-toleration-seconds")
-diff -ur io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/controllermanager.go io_k8s_kubernetes/cmd/kube-controller-manager/app/controllermanager.go
---- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/controllermanager.go	2021-01-26 12:10:52.605725751 +0100
-+++ io_k8s_kubernetes/cmd/kube-controller-manager/app/controllermanager.go	2021-01-26 12:11:04.572786567 +0100
-@@ -140,7 +140,6 @@
- 	namedFlagSets := s.Flags(KnownControllers(), ControllersDisabledByDefault.List())
- 	verflag.AddFlags(namedFlagSets.FlagSet("global"))
- 	globalflag.AddGlobalFlags(namedFlagSets.FlagSet("global"), cmd.Name())
--	registerLegacyGlobalFlags(namedFlagSets)
- 	for _, f := range namedFlagSets.FlagSets {
- 		fs.AddFlagSet(f)
- 	}
-Only in io_k8s_kubernetes/cmd/kube-controller-manager/app: controllermanager.go.orig
-diff -ur io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/plugins.go io_k8s_kubernetes/cmd/kube-controller-manager/app/plugins.go
---- io_k8s_kubernetes.orig/cmd/kube-controller-manager/app/plugins.go	2021-01-26 12:10:52.606725757 +0100
-+++ io_k8s_kubernetes/cmd/kube-controller-manager/app/plugins.go	2021-01-26 12:11:04.572786567 +0100
-@@ -32,19 +32,9 @@
- 	// Volume plugins
- 	"k8s.io/kubernetes/pkg/volume"
- 	"k8s.io/kubernetes/pkg/volume/csi"
--	"k8s.io/kubernetes/pkg/volume/fc"
- 	"k8s.io/kubernetes/pkg/volume/flexvolume"
--	"k8s.io/kubernetes/pkg/volume/flocker"
--	"k8s.io/kubernetes/pkg/volume/glusterfs"
- 	"k8s.io/kubernetes/pkg/volume/hostpath"
--	"k8s.io/kubernetes/pkg/volume/iscsi"
- 	"k8s.io/kubernetes/pkg/volume/local"
--	"k8s.io/kubernetes/pkg/volume/nfs"
--	"k8s.io/kubernetes/pkg/volume/portworx"
--	"k8s.io/kubernetes/pkg/volume/quobyte"
--	"k8s.io/kubernetes/pkg/volume/rbd"
--	"k8s.io/kubernetes/pkg/volume/scaleio"
--	"k8s.io/kubernetes/pkg/volume/storageos"
- 	volumeutil "k8s.io/kubernetes/pkg/volume/util"
- 
- 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-@@ -58,18 +48,7 @@
- // The list of plugins is manually compiled. This code and the plugin
- // initialization code for kubelet really, really need a through refactor.
- func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) {
--	var err error
- 	allPlugins := []volume.VolumePlugin{}
--	allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
--	if err != nil {
--		return allPlugins, err
--	}
--	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
- 	return allPlugins, nil
- }
-@@ -83,18 +62,7 @@
- 
- // ProbeExpandableVolumePlugins returns volume plugins which are expandable
- func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
--	var err error
- 	allPlugins := []volume.VolumePlugin{}
--	allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
--	if err != nil {
--		return allPlugins, err
--	}
--	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
- 	return allPlugins, nil
- }
- 
-@@ -124,30 +92,7 @@
- 	}
- 	allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(hostPathConfig)...)
- 
--	nfsConfig := volume.VolumeConfig{
--		RecyclerMinimumTimeout:   int(config.PersistentVolumeRecyclerConfiguration.MinimumTimeoutNFS),
--		RecyclerTimeoutIncrement: int(config.PersistentVolumeRecyclerConfiguration.IncrementTimeoutNFS),
--		RecyclerPodTemplate:      volume.NewPersistentVolumeRecyclerPodTemplate(),
--	}
--	if err := AttemptToLoadRecycler(config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, &nfsConfig); err != nil {
--		klog.Fatalf("Could not create NFS recycler pod from file %s: %+v", config.PersistentVolumeRecyclerConfiguration.PodTemplateFilePathNFS, err)
--	}
--	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(nfsConfig)...)
--	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
--	// add rbd provisioner
--	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
--	var err error
--	allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
--	if err != nil {
--		return allPlugins, err
--	}
--
--	allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
- 
- 	if utilfeature.DefaultFeatureGate.Enabled(features.CSIInlineVolume) {
- 		allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
-diff -ur io_k8s_kubernetes.orig/cmd/kubectl/BUILD io_k8s_kubernetes/cmd/kubectl/BUILD
---- io_k8s_kubernetes.orig/cmd/kubectl/BUILD	2021-01-26 12:10:52.616725807 +0100
-+++ io_k8s_kubernetes/cmd/kubectl/BUILD	2021-01-26 12:11:04.572786567 +0100
-@@ -3,7 +3,7 @@
-     "go_binary",
-     "go_library",
- )
--load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
-+load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
- 
- go_binary(
-     name = "kubectl",
-diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/options/globalflags.go io_k8s_kubernetes/cmd/kubelet/app/options/globalflags.go
---- io_k8s_kubernetes.orig/cmd/kubelet/app/options/globalflags.go	2021-01-26 12:10:52.617725812 +0100
-+++ io_k8s_kubernetes/cmd/kubelet/app/options/globalflags.go	2021-01-26 12:12:03.724087183 +0100
-@@ -28,10 +28,6 @@
- 	"k8s.io/component-base/logs"
- 	"k8s.io/component-base/version/verflag"
- 	"k8s.io/klog/v2"
--
--	// ensure libs have a chance to globally register their flags
--	_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
--	_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
- )
- 
- // AddGlobalFlags explicitly registers flags that libraries (glog, verflag, etc.) register
-@@ -80,12 +76,8 @@
- 
- // addCredentialProviderFlags adds flags from k8s.io/kubernetes/pkg/credentialprovider
- func addCredentialProviderFlags(fs *pflag.FlagSet) {
--	// lookup flags in global flag set and re-register the values with our flagset
--	global := pflag.CommandLine
- 	local := pflag.NewFlagSet(os.Args[0], pflag.ExitOnError)
- 
--	addLegacyCloudProviderCredentialProviderFlags(global, local)
--
- 	fs.AddFlagSet(local)
- }
- 
-diff -ur io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go io_k8s_kubernetes/cmd/kubelet/app/plugins.go
---- io_k8s_kubernetes.orig/cmd/kubelet/app/plugins.go	2021-01-26 12:10:52.617725812 +0100
-+++ io_k8s_kubernetes/cmd/kubelet/app/plugins.go	2021-01-26 12:11:04.573786572 +0100
-@@ -19,8 +19,6 @@
- // This file exists to force the desired plugin implementations to be linked.
- import (
- 	// Credential providers
--	_ "k8s.io/kubernetes/pkg/credentialprovider/aws"
--	_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
- 	_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
- 
- 	"k8s.io/component-base/featuregate"
-@@ -28,27 +26,15 @@
- 
- 	// Volume plugins
- 	"k8s.io/kubernetes/pkg/volume"
--	"k8s.io/kubernetes/pkg/volume/cephfs"
- 	"k8s.io/kubernetes/pkg/volume/configmap"
- 	"k8s.io/kubernetes/pkg/volume/csi"
- 	"k8s.io/kubernetes/pkg/volume/downwardapi"
- 	"k8s.io/kubernetes/pkg/volume/emptydir"
--	"k8s.io/kubernetes/pkg/volume/fc"
- 	"k8s.io/kubernetes/pkg/volume/flexvolume"
--	"k8s.io/kubernetes/pkg/volume/flocker"
--	"k8s.io/kubernetes/pkg/volume/git_repo"
--	"k8s.io/kubernetes/pkg/volume/glusterfs"
- 	"k8s.io/kubernetes/pkg/volume/hostpath"
--	"k8s.io/kubernetes/pkg/volume/iscsi"
- 	"k8s.io/kubernetes/pkg/volume/local"
--	"k8s.io/kubernetes/pkg/volume/nfs"
--	"k8s.io/kubernetes/pkg/volume/portworx"
- 	"k8s.io/kubernetes/pkg/volume/projected"
--	"k8s.io/kubernetes/pkg/volume/quobyte"
--	"k8s.io/kubernetes/pkg/volume/rbd"
--	"k8s.io/kubernetes/pkg/volume/scaleio"
- 	"k8s.io/kubernetes/pkg/volume/secret"
--	"k8s.io/kubernetes/pkg/volume/storageos"
- 
- 	// Cloud providers
- 	_ "k8s.io/kubernetes/pkg/cloudprovider/providers"
-@@ -64,30 +50,13 @@
- 	//
- 	// Kubelet does not currently need to configure volume plugins.
- 	// If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig
--	var err error
--	allPlugins, err = appendLegacyProviderVolumes(allPlugins, featureGate)
--	if err != nil {
--		return allPlugins, err
--	}
- 	allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(volume.VolumeConfig{})...)
--	allPlugins = append(allPlugins, nfs.ProbeVolumePlugins(volume.VolumeConfig{})...)
- 	allPlugins = append(allPlugins, secret.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, cephfs.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, downwardapi.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, configmap.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, projected.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
--	allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
- 	allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
- 	return allPlugins, nil
- }
-diff -ur io_k8s_kubernetes.orig/cmd/kubelet/BUILD io_k8s_kubernetes/cmd/kubelet/BUILD
---- io_k8s_kubernetes.orig/cmd/kubelet/BUILD	2021-01-26 12:10:52.616725807 +0100
-+++ io_k8s_kubernetes/cmd/kubelet/BUILD	2021-01-26 12:11:04.573786572 +0100
-@@ -5,7 +5,7 @@
-     go_binary = "go_binary_conditional_pure",
- )
- load("@io_bazel_rules_go//go:def.bzl", "go_library")
--load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
-+load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
- 
- go_binary(
-     name = "kubelet",
-diff -ur io_k8s_kubernetes.orig/pkg/controller/nodeipam/ipam/cidr_allocator.go io_k8s_kubernetes/pkg/controller/nodeipam/ipam/cidr_allocator.go
---- io_k8s_kubernetes.orig/pkg/controller/nodeipam/ipam/cidr_allocator.go	2021-01-26 12:10:52.664726051 +0100
-+++ io_k8s_kubernetes/pkg/controller/nodeipam/ipam/cidr_allocator.go	2021-01-26 12:11:04.573786572 +0100
-@@ -112,8 +112,6 @@
- 	switch allocatorType {
- 	case RangeAllocatorType:
- 		return NewCIDRRangeAllocator(kubeClient, nodeInformer, allocatorParams, nodeList)
--	case CloudAllocatorType:
--		return NewCloudCIDRAllocator(kubeClient, cloud, nodeInformer)
- 	default:
- 		return nil, fmt.Errorf("invalid CIDR allocator type: %v", allocatorType)
- 	}
-Only in io_k8s_kubernetes/pkg/controller/nodeipam/ipam: cidr_allocator.go.orig
-diff -ur io_k8s_kubernetes.orig/pkg/controller/nodeipam/nolegacyprovider.go io_k8s_kubernetes/pkg/controller/nodeipam/nolegacyprovider.go
---- io_k8s_kubernetes.orig/pkg/controller/nodeipam/nolegacyprovider.go	2021-01-26 12:10:52.665726056 +0100
-+++ io_k8s_kubernetes/pkg/controller/nodeipam/nolegacyprovider.go	2021-01-26 12:11:04.573786572 +0100
-@@ -1,5 +1,3 @@
--// +build providerless
--
- /*
- Copyright 2019 The Kubernetes Authors.
- 
-diff -ur io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/cadvisor_linux.go io_k8s_kubernetes/pkg/kubelet/cadvisor/cadvisor_linux.go
---- io_k8s_kubernetes.orig/pkg/kubelet/cadvisor/cadvisor_linux.go	2021-01-26 12:10:52.676726112 +0100
-+++ io_k8s_kubernetes/pkg/kubelet/cadvisor/cadvisor_linux.go	2021-01-26 12:11:04.573786572 +0100
-@@ -33,8 +33,6 @@
- 
- 	// Register cloud info providers.
- 	// TODO(#68522): Remove this in 1.20+ once the cAdvisor endpoints are removed.
--	_ "github.com/google/cadvisor/utils/cloudinfo/aws"
--	_ "github.com/google/cadvisor/utils/cloudinfo/azure"
- 	_ "github.com/google/cadvisor/utils/cloudinfo/gce"
- 
- 	"github.com/google/cadvisor/cache/memory"
-Only in io_k8s_kubernetes/pkg/kubelet/cadvisor: cadvisor_linux.go.orig
-diff -ur io_k8s_kubernetes.orig/test/e2e/BUILD io_k8s_kubernetes/test/e2e/BUILD
---- io_k8s_kubernetes.orig/test/e2e/BUILD	2021-01-26 12:10:52.736726417 +0100
-+++ io_k8s_kubernetes/test/e2e/BUILD	2021-01-26 12:11:04.573786572 +0100
-@@ -5,7 +5,7 @@
-     go_test = "go_test_conditional_pure",
- )
- load("@io_bazel_rules_go//go:def.bzl", "go_library")
--load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
-+load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
- 
- go_test(
-     name = "go_default_test",
-diff -ur io_k8s_kubernetes.orig/test/e2e/generated/BUILD io_k8s_kubernetes/test/e2e/generated/BUILD
---- io_k8s_kubernetes.orig/test/e2e/generated/BUILD	2021-01-26 12:10:52.743726453 +0100
-+++ io_k8s_kubernetes/test/e2e/generated/BUILD	2021-01-26 12:11:04.573786572 +0100
-@@ -4,23 +4,24 @@
-     "@io_bazel_rules_go//go:def.bzl",
-     "go_library",
- )
--load("//build:bindata.bzl", "go_bindata")
-+load("@dev_source_monogon//build/bindata:bindata.bzl", "bindata")
- 
- go_library(
-     name = "go_default_library",
-     srcs = [
--        "bindata.go",
-         "gobindata_util.go",
-         "main.go",
-     ],
-+    embed = [
-+        ":bindata",
-+    ],
-     importpath = "k8s.io/kubernetes/test/e2e/generated",
-     deps = [
-         "@io_k8s_klog_v2//:go_default_library",
-     ],
- )
- 
--# IMPORTANT: if you make any changes here, you must also update hack/generate-bindata.sh.
--go_bindata(
-+bindata(
-     name = "bindata",
-     srcs = [
-         "//test/conformance/testdata:all-srcs",
-@@ -29,9 +30,7 @@
-         "//test/fixtures:all-srcs",
-         "//test/images:all-srcs",
-     ],
--    outs = ["bindata.go"],
--    compress = True,
--    include_metadata = False,
-+    package = "generated",
- )
- 
- filegroup(
diff --git a/third_party/go/patches/k8s-native-metrics.patch b/third_party/go/patches/k8s-native-metrics.patch
index 859ee74..2edc60d 100644
--- a/third_party/go/patches/k8s-native-metrics.patch
+++ b/third_party/go/patches/k8s-native-metrics.patch
@@ -1,59 +1,15 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From b16e57cc52a437465bbd12c24fb05fe5790afe1d Mon Sep 17 00:00:00 2001
+From e3b5a31bff00c89fc95f85212bf0943d46692616 Mon Sep 17 00:00:00 2001
 From: Lorenz Brun <lorenz@brun.one>
 Date: Tue, 17 Mar 2020 21:41:08 +0100
-Subject: [PATCH 2/3] Add a native volume metrics implementation
+Subject: [PATCH 2/2] Add a native volume metrics implementation
 
 ---
- pkg/volume/BUILD                  |   3 +
  pkg/volume/metrics_native.go      | 101 +++++++++++++++++++++++++++++
  pkg/volume/metrics_native_test.go | 102 ++++++++++++++++++++++++++++++
- 3 files changed, 206 insertions(+)
+ 2 files changed, 203 insertions(+)
  create mode 100644 pkg/volume/metrics_native.go
  create mode 100644 pkg/volume/metrics_native_test.go
 
-diff --git a/pkg/volume/BUILD b/pkg/volume/BUILD
-index 720b13406dc..b6e4b7e6d6f 100644
---- a/pkg/volume/BUILD
-+++ b/pkg/volume/BUILD
-@@ -7,6 +7,7 @@ go_library(
-         "metrics_cached.go",
-         "metrics_du.go",
-         "metrics_errors.go",
-+        "metrics_native.go",
-         "metrics_nil.go",
-         "metrics_statfs.go",
-         "noop_expandable_plugin.go",
-@@ -35,6 +36,7 @@ go_library(
-         "@io_k8s_client_go//tools/cache:go_default_library",
-         "@io_k8s_client_go//tools/record:go_default_library",
-         "@io_k8s_cloud_provider//:go_default_library",
-+        "@org_golang_x_sys//unix:go_default_library",
-         "@io_k8s_klog_v2//:go_default_library",
-         "@io_k8s_utils//exec:go_default_library",
-         "@io_k8s_utils//mount:go_default_library",
-@@ -55,6 +57,7 @@ go_test(
-     name = "go_default_test",
-     srcs = [
-         "metrics_du_test.go",
-+        "metrics_native_test.go",
-         "metrics_nil_test.go",
-         "metrics_statfs_test.go",
-         "plugins_test.go",
 diff --git a/pkg/volume/metrics_native.go b/pkg/volume/metrics_native.go
 new file mode 100644
 index 00000000000..3934b946f2e
diff --git a/third_party/go/patches/k8s-native-mounter.patch b/third_party/go/patches/k8s-native-mounter.patch
index 2f754b6..a5ced4d 100644
--- a/third_party/go/patches/k8s-native-mounter.patch
+++ b/third_party/go/patches/k8s-native-mounter.patch
@@ -1,69 +1,33 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 8335005ed1983ca5ac036af15dd04b8717898c35 Mon Sep 17 00:00:00 2001
+From 6c346b4fbfd800af47ffa2ec24456f9f58a1b0f2 Mon Sep 17 00:00:00 2001
 From: Lorenz Brun <lorenz@brun.one>
 Date: Mon, 16 Mar 2020 22:13:08 +0100
-Subject: [PATCH 1/3] Provide native mounter implementation for Linux
+Subject: [PATCH 1/7] Provide native mounter implementation for Linux
 
 ---
- BUILD.bazel          |   2 +
- mount/mount_linux.go | 141 ++++++++++++++++++++++-
- 2 files changed, 139 insertions(+), 4 deletions(-)
+ mount_linux.go | 148 +++++++++++++++++-
+ 1 file changed, 144 insertions(+), 4 deletions(-)
 
-diff --git a/mount/BUILD b/mount/BUILD.bazel
-index bef3ec2cf55..6f997103dac 100644
---- a/mount/BUILD.bazel
-+++ b/mount/BUILD.bazel
-@@ -21,6 +21,7 @@ go_library(
-         "//exec:go_default_library",
-     ] + select({
-         "@io_bazel_rules_go//go/platform:android": [
-+            "@org_golang_x_sys//unix:go_default_library",
-             "//io:go_default_library",
-         ],
-         "@io_bazel_rules_go//go/platform:darwin": [
-@@ -36,6 +37,7 @@ go_library(
-             "//io:go_default_library",
-         ],
-         "@io_bazel_rules_go//go/platform:linux": [
-+            "@org_golang_x_sys//unix:go_default_library",
-             "//io:go_default_library",
-         ],
-         "@io_bazel_rules_go//go/platform:nacl": [
-diff --git a/mount/mount_linux.go b/mount/mount_linux.go
-index 41f69efe3f0..01182684653 100644
---- a/mount/mount_linux.go
-+++ b/mount/mount_linux.go
-@@ -20,6 +20,7 @@ package mount
- 
+diff --git a/mount_linux.go b/mount_linux.go
+index aaa592161d4..517bf0b2541 100644
+--- a/mount_linux.go
++++ b/mount_linux.go
+@@ -22,6 +22,7 @@ package mount
  import (
+ 	"context"
  	"fmt"
 +	"io/ioutil"
  	"os"
  	"os/exec"
  	"path/filepath"
-@@ -27,6 +28,7 @@ import (
- 	"strings"
+@@ -30,6 +31,7 @@ import (
  	"syscall"
+ 	"time"
  
 +	"golang.org/x/sys/unix"
  	"k8s.io/klog/v2"
  	utilexec "k8s.io/utils/exec"
  	utilio "k8s.io/utils/io"
-@@ -49,8 +51,10 @@ const (
+@@ -54,8 +56,10 @@ const (
  // for the linux platform.  This implementation assumes that the
  // kubelet is running in the host's root mount namespace.
  type Mounter struct {
@@ -75,8 +39,8 @@
 +	nativeSupportedFstypes map[string]struct{}
  }
  
- // New returns a mount.Interface for the current system.
-@@ -58,8 +62,10 @@ type Mounter struct {
+ var _ MounterForceUnmounter = &Mounter{}
+@@ -65,11 +69,36 @@ var _ MounterForceUnmounter = &Mounter{}
  // mounterPath allows using an alternative to `/bin/mount` for mounting.
  func New(mounterPath string) Interface {
  	return &Mounter{
@@ -89,38 +53,55 @@
  	}
  }
  
-@@ -78,6 +84,29 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
++func (mounter *Mounter) mountNative(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
++	flags, pflags, fsoptions := parseMountOptions(options)
++	if len(pflags) > 0 {
++		return fmt.Errorf("the native mounter is active and does not support mount propagation at the moment")
++	}
++
++	if !mounter.nativeSupportsFstype(fstype) && flags&unix.MS_BIND == 0 {
++		return fmt.Errorf("the native mounter is active and cannot mount filesystems of type \"%v\"", fstype)
++	}
++
++	if flags&unix.MS_BIND != 0 && flags & ^uintptr(unix.MS_BIND) != 0 {
++		if err := unix.Mount(source, target, "", unix.MS_BIND, ""); err != nil {
++			return fmt.Errorf("bind pre-mount failed: %w", err)
++		}
++		flags |= unix.MS_REMOUNT
++	}
++
++	if err := unix.Mount(source, target, fstype, flags, fsoptions); err != nil {
++		return fmt.Errorf("failed to mount filesystem: %w", err)
++	}
++	return nil
++}
++
+ // Mount mounts source to target as fstype with given options. 'source' and 'fstype' must
+ // be an empty string in case it's not required, e.g. for remount, or for auto filesystem
+ // type, where kernel handles fstype for you. The mount 'options' is a list of options,
+@@ -85,6 +114,10 @@ func (mounter *Mounter) Mount(source string, target string, fstype string, optio
  // method should be used by callers that pass sensitive material (like
  // passwords) as mount options.
  func (mounter *Mounter) MountSensitive(source string, target string, fstype string, options []string, sensitiveOptions []string) error {
 +	if !mounter.withLinuxUtils {
-+		flags, pflags, fsoptions := parseMountOptions(options)
-+		if len(pflags) > 0 {
-+			return fmt.Errorf("the native mounter is active and does not support mount propagation at the moment")
-+		}
-+
-+		if !mounter.nativeSupportsFstype(fstype) && flags&unix.MS_BIND == 0 {
-+			return fmt.Errorf("the native mounter is active and cannot mount filesystems of type \"%v\"", fstype)
-+		}
-+
-+		if flags&unix.MS_BIND != 0 && flags & ^uintptr(unix.MS_BIND) != 0 {
-+			if err := unix.Mount(source, target, "", unix.MS_BIND, ""); err != nil {
-+				return fmt.Errorf("bind pre-mount failed: %w", err)
-+			}
-+			flags |= unix.MS_REMOUNT
-+		}
-+
-+		if err := unix.Mount(source, target, fstype, flags, fsoptions); err != nil {
-+			return fmt.Errorf("failed to mount filesystem: %w", err)
-+		}
-+		return nil
++		return mounter.mountNative(source, target, fstype, options, sensitiveOptions)
 +	}
 +
  	// Path to mounter binary if containerized mounter is needed. Otherwise, it is set to empty.
  	// All Linux distros are expected to be shipped with a mount utility that a support bind mounts.
  	mounterPath := ""
-@@ -102,6 +131,80 @@ func (mounter *Mounter) MountSensitive(source string, target string, fstype stri
- 	return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions)
+@@ -116,6 +149,9 @@ func (mounter *Mounter) MountSensitiveWithoutSystemd(source string, target strin
+ 
+ // MountSensitiveWithoutSystemdWithMountFlags is the same as MountSensitiveWithoutSystemd with additional mount flags.
+ func (mounter *Mounter) MountSensitiveWithoutSystemdWithMountFlags(source string, target string, fstype string, options []string, sensitiveOptions []string, mountFlags []string) error {
++	if !mounter.withLinuxUtils {
++		return mounter.mountNative(source, target, fstype, options, sensitiveOptions)
++	}
+ 	mounterPath := ""
+ 	bind, bindOpts, bindRemountOpts, bindRemountOptsSensitive := MakeBindOptsSensitive(options, sensitiveOptions)
+ 	if bind {
+@@ -138,6 +174,80 @@ func (mounter *Mounter) MountSensitiveWithoutSystemdWithMountFlags(source string
+ 	return mounter.doMount(mounterPath, defaultMountCommand, source, target, fstype, options, sensitiveOptions, mountFlags, false)
  }
  
 +// nativeSupportsFstype checks if the native mounter can mount the given fstype
@@ -198,9 +179,9 @@
 +}
 +
  // doMount runs the mount command. mounterPath is the path to mounter binary if containerized mounter is used.
- // sensitiveOptions is an extention of options except they will not be logged (because they may contain sensitive material)
- func (mounter *Mounter) doMount(mounterPath string, mountCmd string, source string, target string, fstype string, options []string, sensitiveOptions []string) error {
-@@ -179,6 +282,30 @@ func detectSystemd() bool {
+ // sensitiveOptions is an extension of options except they will not be logged (because they may contain sensitive material)
+ // systemdMountRequired is an extension of option to decide whether uses systemd mount.
+@@ -223,6 +333,30 @@ func detectSystemd() bool {
  	return true
  }
  
@@ -231,7 +212,7 @@
  // MakeMountArgs makes the arguments to the mount(8) command.
  // options MUST not contain sensitive material (like passwords).
  func MakeMountArgs(source, target, fstype string, options []string) (mountArgs []string) {
-@@ -236,6 +363,12 @@ func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []
+@@ -292,6 +426,12 @@ func AddSystemdScopeSensitive(systemdRunPath, mountName, command string, args []
  // Unmount unmounts the target.
  func (mounter *Mounter) Unmount(target string) error {
  	klog.V(4).Infof("Unmounting %s", target)
diff --git a/third_party/go/patches/k8s-revert-seccomp-runtime-default.patch b/third_party/go/patches/k8s-revert-seccomp-runtime-default.patch
deleted file mode 100644
index d8377b5..0000000
--- a/third_party/go/patches/k8s-revert-seccomp-runtime-default.patch
+++ /dev/null
@@ -1,48 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From 2adf4ec9796839014a708761b8fb6ad815306def Mon Sep 17 00:00:00 2001
-From: Serge Bazanski <serge@nexantic.com>
-Date: Tue, 26 Jan 2021 11:37:01 +0100
-Subject: [PATCH] Manually revert 
- https://github.com/kubernetes/kubernetes/pull/90949
-
-This reverts PR 90494 which breaks runc within Metropolis. See T916.
-
----
- pkg/kubelet/kuberuntime/kuberuntime_sandbox.go | 7 ++-----
- 1 file changed, 2 insertions(+), 5 deletions(-)
-
-diff --git a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go
-index 0978044f753..c46436f2a41 100644
---- a/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go
-+++ b/pkg/kubelet/kuberuntime/kuberuntime_sandbox.go
-@@ -148,11 +148,8 @@ func (m *kubeGenericRuntimeManager) generatePodSandboxLinuxConfig(pod *v1.Pod) (
- 	lc := &runtimeapi.LinuxPodSandboxConfig{
- 		CgroupParent: cgroupParent,
- 		SecurityContext: &runtimeapi.LinuxSandboxSecurityContext{
--			Privileged: kubecontainer.HasPrivilegedContainer(pod),
--
--			// Forcing sandbox to run as `runtime/default` allow users to
--			// use least privileged seccomp profiles at pod level. Issue #84623
--			SeccompProfilePath: v1.SeccompProfileRuntimeDefault,
-+			Privileged:         kubecontainer.HasPrivilegedContainer(pod),
-+			SeccompProfilePath: m.getSeccompProfile(pod.Annotations, "", pod.Spec.SecurityContext, nil),
- 		},
- 	}
- 
--- 
-2.26.2
-
diff --git a/third_party/go/patches/k8s-use-native.patch b/third_party/go/patches/k8s-use-native.patch
deleted file mode 100644
index 61001da..0000000
--- a/third_party/go/patches/k8s-use-native.patch
+++ /dev/null
@@ -1,153 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-From ee4a7df588550ee5cbc3b8419e1ce185a8abb302 Mon Sep 17 00:00:00 2001
-From: Lorenz Brun <lorenz@brun.one>
-Date: Tue, 17 Mar 2020 22:07:24 +0100
-Subject: [PATCH 3/3] Use StatFS and Native volume metrics instead of du
-
----
- pkg/kubelet/stats/log_metrics_provider.go | 2 +-
- pkg/volume/configmap/configmap.go         | 4 ++--
- pkg/volume/downwardapi/downwardapi.go     | 4 ++--
- pkg/volume/emptydir/empty_dir.go          | 4 ++--
- pkg/volume/projected/projected.go         | 4 ++--
- pkg/volume/secret/secret.go               | 4 ++--
- 6 files changed, 11 insertions(+), 11 deletions(-)
-
-diff --git a/pkg/kubelet/stats/log_metrics_provider.go b/pkg/kubelet/stats/log_metrics_provider.go
-index 4a53eef74a3..ff87fec5ec3 100644
---- a/pkg/kubelet/stats/log_metrics_provider.go
-+++ b/pkg/kubelet/stats/log_metrics_provider.go
-@@ -33,5 +33,5 @@ func NewLogMetricsService() LogMetricsService {
- }
- 
- func (l logMetrics) createLogMetricsProvider(path string) volume.MetricsProvider {
--	return volume.NewMetricsDu(path)
-+	return volume.NewMetricsNative(path)
- }
-diff --git a/pkg/volume/configmap/configmap.go b/pkg/volume/configmap/configmap.go
-index 0e74dd0a1d8..430d739aab7 100644
---- a/pkg/volume/configmap/configmap.go
-+++ b/pkg/volume/configmap/configmap.go
-@@ -97,7 +97,7 @@ func (plugin *configMapPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts v
- 			pod.UID,
- 			plugin,
- 			plugin.host.GetMounter(plugin.GetPluginName()),
--			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
-+			volume.NewCachedMetrics(volume.NewMetricsNative(getPath(pod.UID, spec.Name(), plugin.host))),
- 		},
- 		source:       *spec.Volume.ConfigMap,
- 		pod:          *pod,
-@@ -113,7 +113,7 @@ func (plugin *configMapPlugin) NewUnmounter(volName string, podUID types.UID) (v
- 			podUID,
- 			plugin,
- 			plugin.host.GetMounter(plugin.GetPluginName()),
--			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
-+			volume.NewCachedMetrics(volume.NewMetricsNative(getPath(podUID, volName, plugin.host))),
- 		},
- 	}, nil
- }
-diff --git a/pkg/volume/downwardapi/downwardapi.go b/pkg/volume/downwardapi/downwardapi.go
-index a1779c0dac9..f0a0f99b318 100644
---- a/pkg/volume/downwardapi/downwardapi.go
-+++ b/pkg/volume/downwardapi/downwardapi.go
-@@ -99,7 +99,7 @@ func (plugin *downwardAPIPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts
- 		pod:             pod,
- 		podUID:          pod.UID,
- 		plugin:          plugin,
--		MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
-+		MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
- 	}
- 	return &downwardAPIVolumeMounter{
- 		downwardAPIVolume: v,
-@@ -114,7 +114,7 @@ func (plugin *downwardAPIPlugin) NewUnmounter(volName string, podUID types.UID)
- 			volName:         volName,
- 			podUID:          podUID,
- 			plugin:          plugin,
--			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
-+			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
- 		},
- 	}, nil
- }
-diff --git a/pkg/volume/emptydir/empty_dir.go b/pkg/volume/emptydir/empty_dir.go
-index 0a25d2b684c..5dc83b90c5b 100644
---- a/pkg/volume/emptydir/empty_dir.go
-+++ b/pkg/volume/emptydir/empty_dir.go
-@@ -121,7 +121,7 @@ func (plugin *emptyDirPlugin) newMounterInternal(spec *volume.Spec, pod *v1.Pod,
- 		mounter:         mounter,
- 		mountDetector:   mountDetector,
- 		plugin:          plugin,
--		MetricsProvider: volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host)),
-+		MetricsProvider: volume.NewMetricsNative(getPath(pod.UID, spec.Name(), plugin.host)),
- 	}, nil
- }
- 
-@@ -138,7 +138,7 @@ func (plugin *emptyDirPlugin) newUnmounterInternal(volName string, podUID types.
- 		mounter:         mounter,
- 		mountDetector:   mountDetector,
- 		plugin:          plugin,
--		MetricsProvider: volume.NewMetricsDu(getPath(podUID, volName, plugin.host)),
-+		MetricsProvider: volume.NewMetricsNative(getPath(podUID, volName, plugin.host)),
- 	}
- 	return ed, nil
- }
-diff --git a/pkg/volume/projected/projected.go b/pkg/volume/projected/projected.go
-index 0f65a97610c..890f9c1c7bc 100644
---- a/pkg/volume/projected/projected.go
-+++ b/pkg/volume/projected/projected.go
-@@ -114,7 +114,7 @@ func (plugin *projectedPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts v
- 			sources:         spec.Volume.Projected.Sources,
- 			podUID:          pod.UID,
- 			plugin:          plugin,
--			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
-+			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
- 		},
- 		source: *spec.Volume.Projected,
- 		pod:    pod,
-@@ -128,7 +128,7 @@ func (plugin *projectedPlugin) NewUnmounter(volName string, podUID types.UID) (v
- 			volName:         volName,
- 			podUID:          podUID,
- 			plugin:          plugin,
--			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
-+			MetricsProvider: volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
- 		},
- 	}, nil
- }
-diff --git a/pkg/volume/secret/secret.go b/pkg/volume/secret/secret.go
-index a195c59ddd8..4c290cb8f24 100644
---- a/pkg/volume/secret/secret.go
-+++ b/pkg/volume/secret/secret.go
-@@ -100,7 +100,7 @@ func (plugin *secretPlugin) NewMounter(spec *volume.Spec, pod *v1.Pod, opts volu
- 			pod.UID,
- 			plugin,
- 			plugin.host.GetMounter(plugin.GetPluginName()),
--			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(pod.UID, spec.Name(), plugin.host))),
-+			volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(pod.UID, spec.Name(), plugin.host))),
- 		},
- 		source:    *spec.Volume.Secret,
- 		pod:       *pod,
-@@ -116,7 +116,7 @@ func (plugin *secretPlugin) NewUnmounter(volName string, podUID types.UID) (volu
- 			podUID,
- 			plugin,
- 			plugin.host.GetMounter(plugin.GetPluginName()),
--			volume.NewCachedMetrics(volume.NewMetricsDu(getPath(podUID, volName, plugin.host))),
-+			volume.NewCachedMetrics(volume.NewMetricsStatFS(getPath(podUID, volName, plugin.host))),
- 		},
- 	}, nil
- }
--- 
-2.25.1
-
diff --git a/third_party/go/patches/kubelet-fix-path.patch b/third_party/go/patches/kubelet-fix-path.patch
new file mode 100644
index 0000000..069dece
--- /dev/null
+++ b/third_party/go/patches/kubelet-fix-path.patch
@@ -0,0 +1,25 @@
+From 500aace7a0bf5904c740e14ad18cd073df080d37 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@monogon.tech>
+Date: Wed, 16 Mar 2022 18:05:54 +0100
+Subject: [PATCH] Change device plugin path
+
+---
+ pkg/apis/deviceplugin/v1beta1/constants.go | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/pkg/apis/deviceplugin/v1beta1/constants.go b/pkg/apis/deviceplugin/v1beta1/constants.go
+index 3bed214..30f06fe 100644
+--- a/pkg/apis/deviceplugin/v1beta1/constants.go
++++ b/pkg/apis/deviceplugin/v1beta1/constants.go
+@@ -27,7 +27,7 @@ const (
+ 	// DevicePluginPath is the folder the Device Plugin is expecting sockets to be on
+ 	// Only privileged pods have access to this path
+ 	// Note: Placeholder until we find a "standard path"
+-	DevicePluginPath = "/var/lib/kubelet/device-plugins/"
++	DevicePluginPath = "/data/kubernetes/kubelet/device-plugins/"
+ 	// KubeletSocket is the path of the Kubelet registry socket
+ 	KubeletSocket = DevicePluginPath + "kubelet.sock"
+ 
+-- 
+2.25.1
+
diff --git a/third_party/go/patches/libseccomp.patch b/third_party/go/patches/libseccomp.patch
new file mode 100644
index 0000000..0778f33
--- /dev/null
+++ b/third_party/go/patches/libseccomp.patch
@@ -0,0 +1,12 @@
+--- a/BUILD.bazel  2022-03-21 17:04:56.160536936 +0100
++++ b/BUILD.bazel   2022-03-21 17:05:52.439616250 +0100
+@@ -6,6 +6,9 @@
+         "seccomp.go",
+         "seccomp_internal.go",
+     ],
++    cdeps = [
++        "@seccomp",
++    ],
+     cgo = True,
+     importpath = "github.com/seccomp/libseccomp-golang",
+     visibility = ["//visibility:public"],
diff --git a/third_party/go/patches/nfproxy.patch b/third_party/go/patches/nfproxy.patch
deleted file mode 100644
index 307d3d4..0000000
--- a/third_party/go/patches/nfproxy.patch
+++ /dev/null
@@ -1,36 +0,0 @@
-Copyright 2020 The Monogon Project Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-  http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-
-
-Fix nfproxy compatibility with our version of Kubernetes/utilproxy
---- com_github_sbezverk_nfproxy.orig/pkg/proxy/proxy_service.go	2020-07-16 14:24:06.901176302 +0200
-+++ com_github_sbezverk_nfproxy/pkg/proxy/proxy_service.go	2020-07-16 14:08:34.118927035 +0200
-@@ -22,7 +22,6 @@
- 	utilnftables "github.com/google/nftables"
- 	"github.com/sbezverk/nfproxy/pkg/nftables"
- 	v1 "k8s.io/api/core/v1"
--	"k8s.io/apimachinery/pkg/types"
- 	"k8s.io/klog"
- 	utilproxy "k8s.io/kubernetes/pkg/proxy/util"
- 	utilnet "k8s.io/utils/net"
-@@ -44,8 +43,7 @@
- 		stickySeconds := int(*svc.Spec.SessionAffinityConfig.ClientIP.TimeoutSeconds)
- 		klog.V(5).Infof("Service %s/%s has SessionAffinity set for %d seconds", svc.Namespace, svc.Name, stickySeconds)
- 	}
--	svcName := types.NamespacedName{Namespace: svc.Namespace, Name: svc.Name}
--	if utilproxy.ShouldSkipService(svcName, svc) {
-+	if utilproxy.ShouldSkipService(svc) {
- 		return
- 	}
- 	for i := range svc.Spec.Ports {
diff --git a/third_party/go/patches/runc-add-cdeps.patch b/third_party/go/patches/runc-add-cdeps.patch
new file mode 100644
index 0000000..fd15c15
--- /dev/null
+++ b/third_party/go/patches/runc-add-cdeps.patch
@@ -0,0 +1,10 @@
+--- a/libcontainer/seccomp/patchbpf/BUILD.bazel	2022-03-24 20:12:34.325421847 +0100
++++ b/libcontainer/seccomp/patchbpf/BUILD.bazel	2022-03-24 20:12:59.777602881 +0100
+@@ -7,6 +7,7 @@
+         "enosys_unsupported.go",
+     ],
+     cgo = True,
++    cdeps = ["@seccomp"],
+     importpath = "github.com/opencontainers/runc/libcontainer/seccomp/patchbpf",
+     visibility = ["//visibility:public"],
+     deps = select({