Add Kubernetes CTS
This adds patches and build specifications for the Kubernetes Conformance Test Suite. This involves
gating various cloud-specific tests behind the providerless flag (otherwise we'd gain a ton of additional dependencies)
and an additional 60MiB in test binary size.
Since the CTS for weird reasons requires kubectl to be available in the path we first build a kubectl go_image and then
stack the CTS on top of it. The output bundle is then preseeded for use.
Test Plan: `bazel run //core/tests/e2e/k8s_cts`
Bug: T836
X-Origin-Diff: phab/D615
GitOrigin-RevId: 7d2cd780a3ffb63b217591c5854b4aec4031d83d
diff --git a/third_party/go/patches/k8s-e2e-tests-providerless.patch b/third_party/go/patches/k8s-e2e-tests-providerless.patch
new file mode 100644
index 0000000..e4f7dab
--- /dev/null
+++ b/third_party/go/patches/k8s-e2e-tests-providerless.patch
@@ -0,0 +1,5085 @@
+Copyright 2020 The Monogon Project Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+
+From 43752b6c34f49080de3a66b79cbcd92b214c1f01 Mon Sep 17 00:00:00 2001
+From: Lorenz Brun <lorenz@nexantic.com>
+Date: Mon, 20 Jul 2020 16:50:56 +0200
+Subject: [PATCH] POC Make e2e test suite support providerless
+
+---
+ .../custom_metrics_stackdriver_autoscaling.go | 2 +
+ test/e2e/cloud/imports.go | 2 +
+ test/e2e/e2e.go | 10 -
+ test/e2e/e2e_providers.go | 32 +
+ .../framework/providers/gce/firewall_test.go | 2 +
+ .../instrumentation/monitoring/accelerator.go | 2 +
+ .../monitoring/custom_metrics_deployments.go | 2 +
+ .../monitoring/custom_metrics_stackdriver.go | 4 +-
+ .../instrumentation/monitoring/stackdriver.go | 2 +
+ .../monitoring/stackdriver_metadata_agent.go | 4 +-
+ test/e2e/network/firewall.go | 2 +
+ test/e2e/network/ingress.go | 2 +
+ test/e2e/network/ingress_scale.go | 2 +
+ test/e2e/network/network_tiers.go | 2 +
+ test/e2e/network/scale/ingress.go | 2 +
+ .../network/scale/localrun/ingress_scale.go | 2 +-
+ test/e2e/network/service.go | 912 ----------------
+ test/e2e/network/service_providers.go | 980 ++++++++++++++++++
+ test/e2e/node/recreate_node.go | 2 +
+ test/e2e/scheduling/nvidia-gpus.go | 2 +
+ test/e2e/scheduling/ubernetes_lite_volumes.go | 2 +
+ test/e2e/storage/drivers/in_tree.go | 732 -------------
+ test/e2e/storage/drivers/in_tree_providers.go | 751 ++++++++++++++
+ test/e2e/storage/in_tree_volumes.go | 5 -
+ test/e2e/storage/in_tree_volumes_providers.go | 46 +
+ .../nfs_persistent_volume-disruptive.go | 2 +-
+ test/e2e/storage/pd.go | 2 +
+ test/e2e/storage/persistent_volumes-gce.go | 3 +
+ test/e2e/storage/regional_pd.go | 3 +
+ test/e2e/storage/volume_provisioning.go | 527 ----------
+ .../storage/volume_provisioning_providers.go | 577 +++++++++++
+ test/e2e/upgrades/nvidia-gpu.go | 2 +
+ 32 files changed, 2432 insertions(+), 2190 deletions(-)
+ create mode 100644 test/e2e/e2e_providers.go
+ create mode 100644 test/e2e/network/service_providers.go
+ create mode 100644 test/e2e/storage/drivers/in_tree_providers.go
+ create mode 100644 test/e2e/storage/in_tree_volumes_providers.go
+ create mode 100644 test/e2e/storage/volume_provisioning_providers.go
+
+diff --git a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
+index d3a7862d338..8bacec7fe1d 100644
+--- a/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
++++ b/test/e2e/autoscaling/custom_metrics_stackdriver_autoscaling.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/cloud/imports.go b/test/e2e/cloud/imports.go
+index 5aa1def97d1..382cb1a2264 100644
+--- a/test/e2e/cloud/imports.go
++++ b/test/e2e/cloud/imports.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2019 The Kubernetes Authors.
+
+diff --git a/test/e2e/e2e.go b/test/e2e/e2e.go
+index d1e23325d69..f5717e417e7 100644
+--- a/test/e2e/e2e.go
++++ b/test/e2e/e2e.go
+@@ -53,16 +53,6 @@ import (
+ utilnet "k8s.io/utils/net"
+
+ clientset "k8s.io/client-go/kubernetes"
+- // ensure auth plugins are loaded
+- _ "k8s.io/client-go/plugin/pkg/client/auth"
+-
+- // ensure that cloud providers are loaded
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/openstack"
+- _ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere"
+ )
+
+ const (
+diff --git a/test/e2e/e2e_providers.go b/test/e2e/e2e_providers.go
+new file mode 100644
+index 00000000000..cf96642b110
+--- /dev/null
++++ b/test/e2e/e2e_providers.go
+@@ -0,0 +1,32 @@
++// +build !providerless
++
++/*
++Copyright 2020 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package e2e
++
++import (
++ // ensure auth plugins are loaded
++ _ "k8s.io/client-go/plugin/pkg/client/auth"
++
++ // ensure that cloud providers are loaded
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/aws"
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/azure"
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/kubemark"
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/openstack"
++ _ "k8s.io/kubernetes/test/e2e/framework/providers/vsphere"
++)
+diff --git a/test/e2e/framework/providers/gce/firewall_test.go b/test/e2e/framework/providers/gce/firewall_test.go
+index 647441dc962..2a92543a5a7 100644
+--- a/test/e2e/framework/providers/gce/firewall_test.go
++++ b/test/e2e/framework/providers/gce/firewall_test.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2018 The Kubernetes Authors.
+
+diff --git a/test/e2e/instrumentation/monitoring/accelerator.go b/test/e2e/instrumentation/monitoring/accelerator.go
+index 90047e46ea1..6fa094e6a18 100644
+--- a/test/e2e/instrumentation/monitoring/accelerator.go
++++ b/test/e2e/instrumentation/monitoring/accelerator.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
+index de80b129315..8d96b93bf11 100644
+--- a/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
++++ b/test/e2e/instrumentation/monitoring/custom_metrics_deployments.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
+index 277b5a0ab24..ddbc3f20802 100644
+--- a/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
++++ b/test/e2e/instrumentation/monitoring/custom_metrics_stackdriver.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+@@ -21,7 +23,7 @@ import (
+ "time"
+
+ gcm "google.golang.org/api/monitoring/v3"
+- "k8s.io/api/core/v1"
++ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/labels"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+diff --git a/test/e2e/instrumentation/monitoring/stackdriver.go b/test/e2e/instrumentation/monitoring/stackdriver.go
+index dbc5e51c20d..3db0120900b 100644
+--- a/test/e2e/instrumentation/monitoring/stackdriver.go
++++ b/test/e2e/instrumentation/monitoring/stackdriver.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
+index 321591344db..bad9be5b5bf 100644
+--- a/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
++++ b/test/e2e/instrumentation/monitoring/stackdriver_metadata_agent.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+@@ -24,7 +26,7 @@ import (
+ "reflect"
+ "time"
+
+- "k8s.io/api/core/v1"
++ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ clientset "k8s.io/client-go/kubernetes"
+ "k8s.io/kubernetes/test/e2e/framework"
+diff --git a/test/e2e/network/firewall.go b/test/e2e/network/firewall.go
+index f4200f5a30c..f8612ed75a9 100644
+--- a/test/e2e/network/firewall.go
++++ b/test/e2e/network/firewall.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2016 The Kubernetes Authors.
+
+diff --git a/test/e2e/network/ingress.go b/test/e2e/network/ingress.go
+index f1bce918e8f..1a37badbd7b 100644
+--- a/test/e2e/network/ingress.go
++++ b/test/e2e/network/ingress.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2015 The Kubernetes Authors.
+
+diff --git a/test/e2e/network/ingress_scale.go b/test/e2e/network/ingress_scale.go
+index 6cc8585b7b2..867c834868c 100644
+--- a/test/e2e/network/ingress_scale.go
++++ b/test/e2e/network/ingress_scale.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2018 The Kubernetes Authors.
+
+diff --git a/test/e2e/network/network_tiers.go b/test/e2e/network/network_tiers.go
+index 5ae68a5a1ee..f3ea1f72a6b 100644
+--- a/test/e2e/network/network_tiers.go
++++ b/test/e2e/network/network_tiers.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/network/scale/ingress.go b/test/e2e/network/scale/ingress.go
+index 954296beb52..43ad9c9b618 100644
+--- a/test/e2e/network/scale/ingress.go
++++ b/test/e2e/network/scale/ingress.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2018 The Kubernetes Authors.
+
+diff --git a/test/e2e/network/scale/localrun/ingress_scale.go b/test/e2e/network/scale/localrun/ingress_scale.go
+index 2e2c39884da..5a27f5f4cb2 100644
+--- a/test/e2e/network/scale/localrun/ingress_scale.go
++++ b/test/e2e/network/scale/localrun/ingress_scale.go
+@@ -27,7 +27,7 @@ import (
+
+ "k8s.io/klog/v2"
+
+- "k8s.io/api/core/v1"
++ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ clientset "k8s.io/client-go/kubernetes"
+ "k8s.io/client-go/tools/clientcmd"
+diff --git a/test/e2e/network/service.go b/test/e2e/network/service.go
+index cce449b5e8e..db7328efbe8 100644
+--- a/test/e2e/network/service.go
++++ b/test/e2e/network/service.go
+@@ -17,7 +17,6 @@ limitations under the License.
+ package network
+
+ import (
+- "bytes"
+ "context"
+ "encoding/json"
+ "errors"
+@@ -32,8 +31,6 @@ import (
+
+ utilnet "k8s.io/apimachinery/pkg/util/net"
+
+- compute "google.golang.org/api/compute/v1"
+-
+ appsv1 "k8s.io/api/apps/v1"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+@@ -47,11 +44,9 @@ import (
+ "k8s.io/kubernetes/test/e2e/framework"
+ e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
+ e2eendpoints "k8s.io/kubernetes/test/e2e/framework/endpoints"
+- e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem"
+ e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
+ e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+ e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+- "k8s.io/kubernetes/test/e2e/framework/providers/gce"
+ e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
+ e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
+ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+@@ -59,7 +54,6 @@ import (
+ "k8s.io/kubernetes/test/e2e/storage/utils"
+ testutils "k8s.io/kubernetes/test/utils"
+ imageutils "k8s.io/kubernetes/test/utils/image"
+- gcecloud "k8s.io/legacy-cloud-providers/gce"
+
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+@@ -1191,375 +1185,6 @@ var _ = SIGDescribe("Services", func() {
+ framework.ExpectNoError(err)
+ })
+
+- // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed.
+- ginkgo.It("should be able to change the type and ports of a service [Slow] [DisabledForLargeClusters]", func() {
+- // requires cloud load-balancer support
+- e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
+-
+- loadBalancerSupportsUDP := !framework.ProviderIs("aws")
+-
+- loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault
+- if framework.ProviderIs("aws") {
+- loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS
+- }
+- loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
+-
+- // This test is more monolithic than we'd like because LB turnup can be
+- // very slow, so we lumped all the tests into one LB lifecycle.
+-
+- serviceName := "mutability-test"
+- ns1 := f.Namespace.Name // LB1 in ns1 on TCP
+- framework.Logf("namespace for TCP test: %s", ns1)
+-
+- ginkgo.By("creating a second namespace")
+- namespacePtr, err := f.CreateNamespace("services", nil)
+- framework.ExpectNoError(err, "failed to create namespace")
+- ns2 := namespacePtr.Name // LB2 in ns2 on UDP
+- framework.Logf("namespace for UDP test: %s", ns2)
+-
+- nodeIP, err := e2enode.PickIP(cs) // for later
+- framework.ExpectNoError(err)
+-
+- // Test TCP and UDP Services. Services with the same name in different
+- // namespaces should get different node ports and load balancers.
+-
+- ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
+- tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
+- tcpService, err := tcpJig.CreateTCPService(nil)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2)
+- udpJig := e2eservice.NewTestJig(cs, ns2, serviceName)
+- udpService, err := udpJig.CreateUDPService(nil)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("verifying that TCP and UDP use the same port")
+- if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port {
+- framework.Failf("expected to use the same port for TCP and UDP")
+- }
+- svcPort := int(tcpService.Spec.Ports[0].Port)
+- framework.Logf("service port (TCP and UDP): %d", svcPort)
+-
+- ginkgo.By("creating a pod to be part of the TCP service " + serviceName)
+- _, err = tcpJig.Run(nil)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("creating a pod to be part of the UDP service " + serviceName)
+- _, err = udpJig.Run(nil)
+- framework.ExpectNoError(err)
+-
+- // Change the services to NodePort.
+-
+- ginkgo.By("changing the TCP service to type=NodePort")
+- tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Type = v1.ServiceTypeNodePort
+- })
+- framework.ExpectNoError(err)
+- tcpNodePort := int(tcpService.Spec.Ports[0].NodePort)
+- framework.Logf("TCP node port: %d", tcpNodePort)
+-
+- ginkgo.By("changing the UDP service to type=NodePort")
+- udpService, err = udpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Type = v1.ServiceTypeNodePort
+- })
+- framework.ExpectNoError(err)
+- udpNodePort := int(udpService.Spec.Ports[0].NodePort)
+- framework.Logf("UDP node port: %d", udpNodePort)
+-
+- ginkgo.By("hitting the TCP service's NodePort")
+- e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the UDP service's NodePort")
+- testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- // Change the services to LoadBalancer.
+-
+- // Here we test that LoadBalancers can receive static IP addresses. This isn't
+- // necessary, but is an additional feature this monolithic test checks.
+- requestedIP := ""
+- staticIPName := ""
+- if framework.ProviderIs("gce", "gke") {
+- ginkgo.By("creating a static load balancer IP")
+- staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID)
+- gceCloud, err := gce.GetGCECloud()
+- framework.ExpectNoError(err, "failed to get GCE cloud provider")
+-
+- err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region())
+- defer func() {
+- if staticIPName != "" {
+- // Release GCE static IP - this is not kube-managed and will not be automatically released.
+- if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
+- framework.Logf("failed to release static IP %s: %v", staticIPName, err)
+- }
+- }
+- }()
+- framework.ExpectNoError(err, "failed to create region address: %s", staticIPName)
+- reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region())
+- framework.ExpectNoError(err, "failed to get region address: %s", staticIPName)
+-
+- requestedIP = reservedAddr.Address
+- framework.Logf("Allocated static load balancer IP: %s", requestedIP)
+- }
+-
+- ginkgo.By("changing the TCP service to type=LoadBalancer")
+- tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable
+- s.Spec.Type = v1.ServiceTypeLoadBalancer
+- })
+- framework.ExpectNoError(err)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("changing the UDP service to type=LoadBalancer")
+- udpService, err = udpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Type = v1.ServiceTypeLoadBalancer
+- })
+- framework.ExpectNoError(err)
+- }
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService))
+- if loadBalancerSupportsUDP {
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService))
+- }
+-
+- ginkgo.By("waiting for the TCP service to have a load balancer")
+- // Wait for the load balancer to be created asynchronously
+- tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
+- framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort)
+- }
+- if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP {
+- framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
+- }
+- tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
+- framework.Logf("TCP load balancer: %s", tcpIngressIP)
+-
+- if framework.ProviderIs("gce", "gke") {
+- // Do this as early as possible, which overrides the `defer` above.
+- // This is mostly out of fear of leaking the IP in a timeout case
+- // (as of this writing we're not 100% sure where the leaks are
+- // coming from, so this is first-aid rather than surgery).
+- ginkgo.By("demoting the static IP to ephemeral")
+- if staticIPName != "" {
+- gceCloud, err := gce.GetGCECloud()
+- framework.ExpectNoError(err, "failed to get GCE cloud provider")
+- // Deleting it after it is attached "demotes" it to an
+- // ephemeral IP, which can be auto-released.
+- if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
+- framework.Failf("failed to release static IP %s: %v", staticIPName, err)
+- }
+- staticIPName = ""
+- }
+- }
+-
+- var udpIngressIP string
+- if loadBalancerSupportsUDP {
+- ginkgo.By("waiting for the UDP service to have a load balancer")
+- // 2nd one should be faster since they ran in parallel.
+- udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
+- framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort)
+- }
+- udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])
+- framework.Logf("UDP load balancer: %s", udpIngressIP)
+-
+- ginkgo.By("verifying that TCP and UDP use different load balancers")
+- if tcpIngressIP == udpIngressIP {
+- framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
+- }
+- }
+-
+- ginkgo.By("hitting the TCP service's NodePort")
+- e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the UDP service's NodePort")
+- testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the TCP service's LoadBalancer")
+- e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("hitting the UDP service's LoadBalancer")
+- testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
+- }
+-
+- // Change the services' node ports.
+-
+- ginkgo.By("changing the TCP service's NodePort")
+- tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort)
+- framework.ExpectNoError(err)
+- tcpNodePortOld := tcpNodePort
+- tcpNodePort = int(tcpService.Spec.Ports[0].NodePort)
+- if tcpNodePort == tcpNodePortOld {
+- framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort)
+- }
+- if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
+- framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
+- }
+- framework.Logf("TCP node port: %d", tcpNodePort)
+-
+- ginkgo.By("changing the UDP service's NodePort")
+- udpService, err = udpJig.ChangeServiceNodePort(udpNodePort)
+- framework.ExpectNoError(err)
+- udpNodePortOld := udpNodePort
+- udpNodePort = int(udpService.Spec.Ports[0].NodePort)
+- if udpNodePort == udpNodePortOld {
+- framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort)
+- }
+- if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
+- framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
+- }
+- framework.Logf("UDP node port: %d", udpNodePort)
+-
+- ginkgo.By("hitting the TCP service's new NodePort")
+- e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the UDP service's new NodePort")
+- testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("checking the old TCP NodePort is closed")
+- testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("checking the old UDP NodePort is closed")
+- testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the TCP service's LoadBalancer")
+- e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("hitting the UDP service's LoadBalancer")
+- testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
+- }
+-
+- // Change the services' main ports.
+-
+- ginkgo.By("changing the TCP service's port")
+- tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Ports[0].Port++
+- })
+- framework.ExpectNoError(err)
+- svcPortOld := svcPort
+- svcPort = int(tcpService.Spec.Ports[0].Port)
+- if svcPort == svcPortOld {
+- framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort)
+- }
+- if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
+- framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort)
+- }
+- if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
+- framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
+- }
+-
+- ginkgo.By("changing the UDP service's port")
+- udpService, err = udpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Ports[0].Port++
+- })
+- framework.ExpectNoError(err)
+- if int(udpService.Spec.Ports[0].Port) != svcPort {
+- framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port)
+- }
+- if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
+- framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort)
+- }
+- if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
+- framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
+- }
+-
+- framework.Logf("service port (TCP and UDP): %d", svcPort)
+-
+- ginkgo.By("hitting the TCP service's NodePort")
+- e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the UDP service's NodePort")
+- testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the TCP service's LoadBalancer")
+- e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("hitting the UDP service's LoadBalancer")
+- testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
+- }
+-
+- ginkgo.By("Scaling the pods to 0")
+- err = tcpJig.Scale(0)
+- framework.ExpectNoError(err)
+- err = udpJig.Scale(0)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort")
+- testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort")
+- testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer")
+- testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer")
+- testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
+- }
+-
+- ginkgo.By("Scaling the pods to 1")
+- err = tcpJig.Scale(1)
+- framework.ExpectNoError(err)
+- err = udpJig.Scale(1)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("hitting the TCP service's NodePort")
+- e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the UDP service's NodePort")
+- testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("hitting the TCP service's LoadBalancer")
+- e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("hitting the UDP service's LoadBalancer")
+- testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
+- }
+-
+- // Change the services back to ClusterIP.
+-
+- ginkgo.By("changing TCP service back to type=ClusterIP")
+- _, err = tcpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Type = v1.ServiceTypeClusterIP
+- s.Spec.Ports[0].NodePort = 0
+- })
+- framework.ExpectNoError(err)
+- // Wait for the load balancer to be destroyed asynchronously
+- _, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("changing UDP service back to type=ClusterIP")
+- _, err = udpJig.UpdateService(func(s *v1.Service) {
+- s.Spec.Type = v1.ServiceTypeClusterIP
+- s.Spec.Ports[0].NodePort = 0
+- })
+- framework.ExpectNoError(err)
+- if loadBalancerSupportsUDP {
+- // Wait for the load balancer to be destroyed asynchronously
+- _, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- }
+-
+- ginkgo.By("checking the TCP NodePort is closed")
+- testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("checking the UDP NodePort is closed")
+- testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
+-
+- ginkgo.By("checking the TCP LoadBalancer is closed")
+- testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
+-
+- if loadBalancerSupportsUDP {
+- ginkgo.By("checking the UDP LoadBalancer is closed")
+- testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
+- }
+- })
+-
+ /*
+ Testname: Service, update NodePort, same port different protocol
+ Description: Create a service to accept TCP requests. By default, created service MUST be of type ClusterIP and an ClusterIP MUST be assigned to the service.
+@@ -2202,199 +1827,6 @@ var _ = SIGDescribe("Services", func() {
+ checkReachabilityFromPod(true, normalReachabilityTimeout, namespace, dropPod.Name, svcIP)
+ })
+
+- ginkgo.It("should be able to create an internal type load balancer [Slow]", func() {
+- e2eskipper.SkipUnlessProviderIs("azure", "gke", "gce")
+-
+- createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
+- pollInterval := framework.Poll * 10
+-
+- namespace := f.Namespace.Name
+- serviceName := "lb-internal"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- ginkgo.By("creating pod to be part of service " + serviceName)
+- _, err := jig.Run(nil)
+- framework.ExpectNoError(err)
+-
+- enableILB, disableILB := enableAndDisableInternalLB()
+-
+- isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool {
+- ingressEndpoint := e2eservice.GetIngressPoint(lbIngress)
+- // Needs update for providers using hostname as endpoint.
+- return strings.HasPrefix(ingressEndpoint, "10.")
+- }
+-
+- ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled")
+- svc, err := jig.CreateTCPService(func(svc *v1.Service) {
+- svc.Spec.Type = v1.ServiceTypeLoadBalancer
+- enableILB(svc)
+- })
+- framework.ExpectNoError(err)
+-
+- defer func() {
+- ginkgo.By("Clean up loadbalancer service")
+- e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
+- }()
+-
+- svc, err = jig.WaitForLoadBalancer(createTimeout)
+- framework.ExpectNoError(err)
+- lbIngress := &svc.Status.LoadBalancer.Ingress[0]
+- svcPort := int(svc.Spec.Ports[0].Port)
+- // should have an internal IP.
+- framework.ExpectEqual(isInternalEndpoint(lbIngress), true)
+-
+- // ILBs are not accessible from the test orchestrator, so it's necessary to use
+- // a pod to test the service.
+- ginkgo.By("hitting the internal load balancer from pod")
+- framework.Logf("creating pod with host network")
+- hostExec := launchHostExecPod(f.ClientSet, f.Namespace.Name, "ilb-host-exec")
+-
+- framework.Logf("Waiting up to %v for service %q's internal LB to respond to requests", createTimeout, serviceName)
+- tcpIngressIP := e2eservice.GetIngressPoint(lbIngress)
+- if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
+- cmd := fmt.Sprintf(`curl -m 5 'http://%v:%v/echo?msg=hello'`, tcpIngressIP, svcPort)
+- stdout, err := framework.RunHostCmd(hostExec.Namespace, hostExec.Name, cmd)
+- if err != nil {
+- framework.Logf("error curling; stdout: %v. err: %v", stdout, err)
+- return false, nil
+- }
+-
+- if !strings.Contains(stdout, "hello") {
+- framework.Logf("Expected output to contain 'hello', got %q; retrying...", stdout)
+- return false, nil
+- }
+-
+- framework.Logf("Successful curl; stdout: %v", stdout)
+- return true, nil
+- }); pollErr != nil {
+- framework.Failf("ginkgo.Failed to hit ILB IP, err: %v", pollErr)
+- }
+-
+- ginkgo.By("switching to external type LoadBalancer")
+- svc, err = jig.UpdateService(func(svc *v1.Service) {
+- disableILB(svc)
+- })
+- framework.ExpectNoError(err)
+- framework.Logf("Waiting up to %v for service %q to have an external LoadBalancer", createTimeout, serviceName)
+- if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
+- svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
+- if err != nil {
+- return false, err
+- }
+- lbIngress = &svc.Status.LoadBalancer.Ingress[0]
+- return !isInternalEndpoint(lbIngress), nil
+- }); pollErr != nil {
+- framework.Failf("Loadbalancer IP not changed to external.")
+- }
+- // should have an external IP.
+- gomega.Expect(isInternalEndpoint(lbIngress)).To(gomega.BeFalse())
+-
+- ginkgo.By("hitting the external load balancer")
+- framework.Logf("Waiting up to %v for service %q's external LB to respond to requests", createTimeout, serviceName)
+- tcpIngressIP = e2eservice.GetIngressPoint(lbIngress)
+- e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
+-
+- // GCE cannot test a specific IP because the test may not own it. This cloud specific condition
+- // will be removed when GCP supports similar functionality.
+- if framework.ProviderIs("azure") {
+- ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.")
+- internalStaticIP := "10.240.11.11"
+- svc, err = jig.UpdateService(func(svc *v1.Service) {
+- svc.Spec.LoadBalancerIP = internalStaticIP
+- enableILB(svc)
+- })
+- framework.ExpectNoError(err)
+- framework.Logf("Waiting up to %v for service %q to have an internal LoadBalancer", createTimeout, serviceName)
+- if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
+- svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
+- if err != nil {
+- return false, err
+- }
+- lbIngress = &svc.Status.LoadBalancer.Ingress[0]
+- return isInternalEndpoint(lbIngress), nil
+- }); pollErr != nil {
+- framework.Failf("Loadbalancer IP not changed to internal.")
+- }
+- // should have the given static internal IP.
+- framework.ExpectEqual(e2eservice.GetIngressPoint(lbIngress), internalStaticIP)
+- }
+- })
+-
+- // This test creates a load balancer, make sure its health check interval
+- // equals to gceHcCheckIntervalSeconds. Then the interval is manipulated
+- // to be something else, see if the interval will be reconciled.
+- ginkgo.It("should reconcile LB health check interval [Slow][Serial]", func() {
+- const gceHcCheckIntervalSeconds = int64(8)
+- // This test is for clusters on GCE.
+- // (It restarts kube-controller-manager, which we don't support on GKE)
+- e2eskipper.SkipUnlessProviderIs("gce")
+- e2eskipper.SkipUnlessSSHKeyPresent()
+-
+- clusterID, err := gce.GetClusterID(cs)
+- if err != nil {
+- framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err)
+- }
+- gceCloud, err := gce.GetGCECloud()
+- if err != nil {
+- framework.Failf("framework.GetGCECloud() = _, %v; want nil", err)
+- }
+-
+- namespace := f.Namespace.Name
+- serviceName := "lb-hc-int"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- ginkgo.By("create load balancer service")
+- // Create loadbalancer service with source range from node[0] and podAccept
+- svc, err := jig.CreateTCPService(func(svc *v1.Service) {
+- svc.Spec.Type = v1.ServiceTypeLoadBalancer
+- })
+- framework.ExpectNoError(err)
+-
+- defer func() {
+- ginkgo.By("Clean up loadbalancer service")
+- e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
+- }()
+-
+- svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
+- framework.ExpectNoError(err)
+-
+- hcName := gcecloud.MakeNodesHealthCheckName(clusterID)
+- hc, err := gceCloud.GetHTTPHealthCheck(hcName)
+- if err != nil {
+- framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err)
+- }
+- framework.ExpectEqual(hc.CheckIntervalSec, gceHcCheckIntervalSeconds)
+-
+- ginkgo.By("modify the health check interval")
+- hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1
+- if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil {
+- framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err)
+- }
+-
+- ginkgo.By("restart kube-controller-manager")
+- if err := e2ekubesystem.RestartControllerManager(); err != nil {
+- framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err)
+- }
+- if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil {
+- framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err)
+- }
+-
+- ginkgo.By("health check should be reconciled")
+- pollInterval := framework.Poll * 10
+- loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
+- if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) {
+- hc, err := gceCloud.GetHTTPHealthCheck(hcName)
+- if err != nil {
+- framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err)
+- return false, err
+- }
+- framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec)
+- return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil
+- }); pollErr != nil {
+- framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds)
+- }
+- })
+-
+ /*
+ Release: v1.19
+ Testname: Service, ClusterIP type, session affinity to ClientIP
+@@ -2880,350 +2312,6 @@ var _ = SIGDescribe("Services", func() {
+ })
+ })
+
+-var _ = SIGDescribe("ESIPP [Slow]", func() {
+- f := framework.NewDefaultFramework("esipp")
+- var loadBalancerCreateTimeout time.Duration
+-
+- var cs clientset.Interface
+- serviceLBNames := []string{}
+-
+- ginkgo.BeforeEach(func() {
+- // requires cloud load-balancer support - this feature currently supported only on GCE/GKE
+- e2eskipper.SkipUnlessProviderIs("gce", "gke")
+-
+- cs = f.ClientSet
+- loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
+- })
+-
+- ginkgo.AfterEach(func() {
+- if ginkgo.CurrentGinkgoTestDescription().Failed {
+- DescribeSvc(f.Namespace.Name)
+- }
+- for _, lb := range serviceLBNames {
+- framework.Logf("cleaning load balancer resource for %s", lb)
+- e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
+- }
+- //reset serviceLBNames
+- serviceLBNames = []string{}
+- })
+-
+- ginkgo.It("should work for type=LoadBalancer", func() {
+- namespace := f.Namespace.Name
+- serviceName := "external-local-lb"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
+- framework.ExpectNoError(err)
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
+- healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
+- if healthCheckNodePort == 0 {
+- framework.Failf("Service HealthCheck NodePort was not allocated")
+- }
+- defer func() {
+- err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+-
+- // Make sure we didn't leak the health check node port.
+- threshold := 2
+- nodes, err := jig.GetEndpointNodes()
+- framework.ExpectNoError(err)
+- for _, ips := range nodes {
+- err := TestHTTPHealthCheckNodePort(ips[0], healthCheckNodePort, "/healthz", e2eservice.KubeProxyEndpointLagTimeout, false, threshold)
+- framework.ExpectNoError(err)
+- }
+- err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err)
+- }()
+-
+- svcTCPPort := int(svc.Spec.Ports[0].Port)
+- ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
+-
+- ginkgo.By("reading clientIP using the TCP service's service port via its external VIP")
+- content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
+- clientIP := content.String()
+- framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP)
+-
+- ginkgo.By("checking if Source IP is preserved")
+- if strings.HasPrefix(clientIP, "10.") {
+- framework.Failf("Source IP was NOT preserved")
+- }
+- })
+-
+- ginkgo.It("should work for type=NodePort", func() {
+- namespace := f.Namespace.Name
+- serviceName := "external-local-nodeport"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- svc, err := jig.CreateOnlyLocalNodePortService(true)
+- framework.ExpectNoError(err)
+- defer func() {
+- err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err)
+- }()
+-
+- tcpNodePort := int(svc.Spec.Ports[0].NodePort)
+- endpointsNodeMap, err := jig.GetEndpointNodes()
+- framework.ExpectNoError(err)
+- path := "/clientip"
+-
+- for nodeName, nodeIPs := range endpointsNodeMap {
+- nodeIP := nodeIPs[0]
+- ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v%v%v", nodeName, nodeIP, tcpNodePort, path))
+- content := GetHTTPContent(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, path)
+- clientIP := content.String()
+- framework.Logf("ClientIP detected by target pod using NodePort is %s", clientIP)
+- if strings.HasPrefix(clientIP, "10.") {
+- framework.Failf("Source IP was NOT preserved")
+- }
+- }
+- })
+-
+- ginkgo.It("should only target nodes with endpoints", func() {
+- namespace := f.Namespace.Name
+- serviceName := "external-local-nodes"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+- nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
+- framework.ExpectNoError(err)
+-
+- svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false,
+- func(svc *v1.Service) {
+- // Change service port to avoid collision with opened hostPorts
+- // in other tests that run in parallel.
+- if len(svc.Spec.Ports) != 0 {
+- svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port))
+- svc.Spec.Ports[0].Port = 8081
+- }
+-
+- })
+- framework.ExpectNoError(err)
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
+- defer func() {
+- err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err)
+- }()
+-
+- healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
+- if healthCheckNodePort == 0 {
+- framework.Failf("Service HealthCheck NodePort was not allocated")
+- }
+-
+- ips := e2enode.CollectAddresses(nodes, v1.NodeExternalIP)
+-
+- ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
+- svcTCPPort := int(svc.Spec.Ports[0].Port)
+-
+- threshold := 2
+- path := "/healthz"
+- for i := 0; i < len(nodes.Items); i++ {
+- endpointNodeName := nodes.Items[i].Name
+-
+- ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName)
+- _, err = jig.Run(func(rc *v1.ReplicationController) {
+- rc.Name = serviceName
+- if endpointNodeName != "" {
+- rc.Spec.Template.Spec.NodeName = endpointNodeName
+- }
+- })
+- framework.ExpectNoError(err)
+-
+- ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName))
+- err = jig.WaitForEndpointOnNode(endpointNodeName)
+- framework.ExpectNoError(err)
+-
+- // HealthCheck should pass only on the node where num(endpoints) > 0
+- // All other nodes should fail the healthcheck on the service healthCheckNodePort
+- for n, publicIP := range ips {
+- // Make sure the loadbalancer picked up the health check change.
+- // Confirm traffic can reach backend through LB before checking healthcheck nodeport.
+- e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout)
+- expectedSuccess := nodes.Items[n].Name == endpointNodeName
+- port := strconv.Itoa(healthCheckNodePort)
+- ipPort := net.JoinHostPort(publicIP, port)
+- framework.Logf("Health checking %s, http://%s%s, expectedSuccess %v", nodes.Items[n].Name, ipPort, path, expectedSuccess)
+- err := TestHTTPHealthCheckNodePort(publicIP, healthCheckNodePort, path, e2eservice.KubeProxyEndpointLagTimeout, expectedSuccess, threshold)
+- framework.ExpectNoError(err)
+- }
+- framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName))
+- }
+- })
+-
+- ginkgo.It("should work from pods", func() {
+- var err error
+- namespace := f.Namespace.Name
+- serviceName := "external-local-pods"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
+- framework.ExpectNoError(err)
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
+- defer func() {
+- err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err)
+- }()
+-
+- ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
+- port := strconv.Itoa(int(svc.Spec.Ports[0].Port))
+- ipPort := net.JoinHostPort(ingressIP, port)
+- path := fmt.Sprintf("%s/clientip", ipPort)
+-
+- ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state")
+- deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1)
+- framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment")
+-
+- defer func() {
+- framework.Logf("Deleting deployment")
+- err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name)
+- }()
+-
+- deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
+- framework.ExpectNoError(err, "Error in retrieving pause pod deployment")
+- labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
+- framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment")
+-
+- pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()})
+- framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments")
+-
+- pausePod := pausePods.Items[0]
+- framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path)
+- cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path)
+-
+- var srcIP string
+- loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
+- ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName))
+- if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) {
+- stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd)
+- if err != nil {
+- framework.Logf("got err: %v, retry until timeout", err)
+- return false, nil
+- }
+- srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0])
+- return srcIP == pausePod.Status.PodIP, nil
+- }); pollErr != nil {
+- framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP)
+- }
+- })
+-
+- // TODO: Get rid of [DisabledForLargeClusters] tag when issue #90047 is fixed.
+- ginkgo.It("should handle updates to ExternalTrafficPolicy field [DisabledForLargeClusters]", func() {
+- namespace := f.Namespace.Name
+- serviceName := "external-local-update"
+- jig := e2eservice.NewTestJig(cs, namespace, serviceName)
+-
+- nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
+- framework.ExpectNoError(err)
+- if len(nodes.Items) < 2 {
+- framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint")
+- }
+-
+- svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
+- framework.ExpectNoError(err)
+- serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
+- defer func() {
+- err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
+- framework.ExpectNoError(err)
+- err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
+- framework.ExpectNoError(err)
+- }()
+-
+- // save the health check node port because it disappears when ESIPP is turned off.
+- healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
+-
+- ginkgo.By("turning ESIPP off")
+- svc, err = jig.UpdateService(func(svc *v1.Service) {
+- svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
+- })
+- framework.ExpectNoError(err)
+- if svc.Spec.HealthCheckNodePort > 0 {
+- framework.Failf("Service HealthCheck NodePort still present")
+- }
+-
+- endpointNodeMap, err := jig.GetEndpointNodes()
+- framework.ExpectNoError(err)
+- noEndpointNodeMap := map[string][]string{}
+- for _, n := range nodes.Items {
+- if _, ok := endpointNodeMap[n.Name]; ok {
+- continue
+- }
+- noEndpointNodeMap[n.Name] = e2enode.GetAddresses(&n, v1.NodeExternalIP)
+- }
+-
+- svcTCPPort := int(svc.Spec.Ports[0].Port)
+- svcNodePort := int(svc.Spec.Ports[0].NodePort)
+- ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
+- path := "/clientip"
+-
+- ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap))
+- for nodeName, nodeIPs := range noEndpointNodeMap {
+- ginkgo.By(fmt.Sprintf("Checking %v (%v:%v%v) proxies to endpoints on another node", nodeName, nodeIPs[0], svcNodePort, path))
+- GetHTTPContent(nodeIPs[0], svcNodePort, e2eservice.KubeProxyLagTimeout, path)
+- }
+-
+- for nodeName, nodeIPs := range endpointNodeMap {
+- ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIPs[0]))
+- var body bytes.Buffer
+- pollfn := func() (bool, error) {
+- result := e2enetwork.PokeHTTP(nodeIPs[0], healthCheckNodePort, "/healthz", nil)
+- if result.Code == 0 {
+- return true, nil
+- }
+- body.Reset()
+- body.Write(result.Body)
+- return false, nil
+- }
+- if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollfn); pollErr != nil {
+- framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s",
+- nodeName, healthCheckNodePort, body.String())
+- }
+- }
+-
+- // Poll till kube-proxy re-adds the MASQUERADE rule on the node.
+- ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP))
+- var clientIP string
+- pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
+- content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
+- clientIP = content.String()
+- if strings.HasPrefix(clientIP, "10.") {
+- return true, nil
+- }
+- return false, nil
+- })
+- if pollErr != nil {
+- framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP)
+- }
+-
+- // TODO: We need to attempt to create another service with the previously
+- // allocated healthcheck nodePort. If the health check nodePort has been
+- // freed, the new service creation will succeed, upon which we cleanup.
+- // If the health check nodePort has NOT been freed, the new service
+- // creation will fail.
+-
+- ginkgo.By("setting ExternalTraffic field back to OnlyLocal")
+- svc, err = jig.UpdateService(func(svc *v1.Service) {
+- svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
+- // Request the same healthCheckNodePort as before, to test the user-requested allocation path
+- svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
+- })
+- framework.ExpectNoError(err)
+- pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
+- content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path)
+- clientIP = content.String()
+- ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP))
+- if !strings.HasPrefix(clientIP, "10.") {
+- return true, nil
+- }
+- return false, nil
+- })
+- if pollErr != nil {
+- framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP)
+- }
+- })
+-})
+-
+ func execSourceipTest(pausePod v1.Pod, serviceAddress string) (string, string) {
+ var err error
+ var stdout string
+diff --git a/test/e2e/network/service_providers.go b/test/e2e/network/service_providers.go
+new file mode 100644
+index 00000000000..b7eae6feb2c
+--- /dev/null
++++ b/test/e2e/network/service_providers.go
+@@ -0,0 +1,980 @@
++// +build !providerless
++
++/*
++Copyright 2020 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package network
++
++import (
++ "bytes"
++ "context"
++ "fmt"
++ "net"
++ "strconv"
++ "strings"
++ "time"
++
++ compute "google.golang.org/api/compute/v1"
++ v1 "k8s.io/api/core/v1"
++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
++ "k8s.io/apimachinery/pkg/util/intstr"
++ "k8s.io/apimachinery/pkg/util/wait"
++ clientset "k8s.io/client-go/kubernetes"
++ cloudprovider "k8s.io/cloud-provider"
++ "k8s.io/kubernetes/test/e2e/framework"
++ e2edeployment "k8s.io/kubernetes/test/e2e/framework/deployment"
++ e2ekubesystem "k8s.io/kubernetes/test/e2e/framework/kubesystem"
++ e2enetwork "k8s.io/kubernetes/test/e2e/framework/network"
++ e2enode "k8s.io/kubernetes/test/e2e/framework/node"
++ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
++ e2erc "k8s.io/kubernetes/test/e2e/framework/rc"
++ e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
++ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
++ gcecloud "k8s.io/legacy-cloud-providers/gce"
++
++ "github.com/onsi/ginkgo"
++ "github.com/onsi/gomega"
++)
++
++var _ = SIGDescribe("Services with Cloud LoadBalancers", func() {
++
++ f := framework.NewDefaultFramework("services")
++
++ var cs clientset.Interface
++ serviceLBNames := []string{}
++
++ ginkgo.BeforeEach(func() {
++ cs = f.ClientSet
++ })
++
++ ginkgo.AfterEach(func() {
++ if ginkgo.CurrentGinkgoTestDescription().Failed {
++ DescribeSvc(f.Namespace.Name)
++ }
++ for _, lb := range serviceLBNames {
++ framework.Logf("cleaning load balancer resource for %s", lb)
++ e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
++ }
++ //reset serviceLBNames
++ serviceLBNames = []string{}
++ })
++
++ // TODO: Get rid of [DisabledForLargeClusters] tag when issue #56138 is fixed
++ ginkgo.It("should be able to change the type and ports of a service [Slow] [DisabledForLargeClusters]", func() {
++ // requires cloud load-balancer support
++ e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
++
++ loadBalancerSupportsUDP := !framework.ProviderIs("aws")
++
++ loadBalancerLagTimeout := e2eservice.LoadBalancerLagTimeoutDefault
++ if framework.ProviderIs("aws") {
++ loadBalancerLagTimeout = e2eservice.LoadBalancerLagTimeoutAWS
++ }
++ loadBalancerCreateTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
++
++ // This test is more monolithic than we'd like because LB turnup can be
++ // very slow, so we lumped all the tests into one LB lifecycle.
++
++ serviceName := "mutability-test"
++ ns1 := f.Namespace.Name // LB1 in ns1 on TCP
++ framework.Logf("namespace for TCP test: %s", ns1)
++
++ ginkgo.By("creating a second namespace")
++ namespacePtr, err := f.CreateNamespace("services", nil)
++ framework.ExpectNoError(err, "failed to create namespace")
++ ns2 := namespacePtr.Name // LB2 in ns2 on UDP
++ framework.Logf("namespace for UDP test: %s", ns2)
++
++ nodeIP, err := e2enode.PickIP(cs) // for later
++ framework.ExpectNoError(err)
++
++ // Test TCP and UDP Services. Services with the same name in different
++ // namespaces should get different node ports and load balancers.
++
++ ginkgo.By("creating a TCP service " + serviceName + " with type=ClusterIP in namespace " + ns1)
++ tcpJig := e2eservice.NewTestJig(cs, ns1, serviceName)
++ tcpService, err := tcpJig.CreateTCPService(nil)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("creating a UDP service " + serviceName + " with type=ClusterIP in namespace " + ns2)
++ udpJig := e2eservice.NewTestJig(cs, ns2, serviceName)
++ udpService, err := udpJig.CreateUDPService(nil)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("verifying that TCP and UDP use the same port")
++ if tcpService.Spec.Ports[0].Port != udpService.Spec.Ports[0].Port {
++ framework.Failf("expected to use the same port for TCP and UDP")
++ }
++ svcPort := int(tcpService.Spec.Ports[0].Port)
++ framework.Logf("service port (TCP and UDP): %d", svcPort)
++
++ ginkgo.By("creating a pod to be part of the TCP service " + serviceName)
++ _, err = tcpJig.Run(nil)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("creating a pod to be part of the UDP service " + serviceName)
++ _, err = udpJig.Run(nil)
++ framework.ExpectNoError(err)
++
++ // Change the services to NodePort.
++
++ ginkgo.By("changing the TCP service to type=NodePort")
++ tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Type = v1.ServiceTypeNodePort
++ })
++ framework.ExpectNoError(err)
++ tcpNodePort := int(tcpService.Spec.Ports[0].NodePort)
++ framework.Logf("TCP node port: %d", tcpNodePort)
++
++ ginkgo.By("changing the UDP service to type=NodePort")
++ udpService, err = udpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Type = v1.ServiceTypeNodePort
++ })
++ framework.ExpectNoError(err)
++ udpNodePort := int(udpService.Spec.Ports[0].NodePort)
++ framework.Logf("UDP node port: %d", udpNodePort)
++
++ ginkgo.By("hitting the TCP service's NodePort")
++ e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the UDP service's NodePort")
++ testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ // Change the services to LoadBalancer.
++
++ // Here we test that LoadBalancers can receive static IP addresses. This isn't
++ // necessary, but is an additional feature this monolithic test checks.
++ requestedIP := ""
++ staticIPName := ""
++ if framework.ProviderIs("gce", "gke") {
++ ginkgo.By("creating a static load balancer IP")
++ staticIPName = fmt.Sprintf("e2e-external-lb-test-%s", framework.RunID)
++ gceCloud, err := gce.GetGCECloud()
++ framework.ExpectNoError(err, "failed to get GCE cloud provider")
++
++ err = gceCloud.ReserveRegionAddress(&compute.Address{Name: staticIPName}, gceCloud.Region())
++ defer func() {
++ if staticIPName != "" {
++ // Release GCE static IP - this is not kube-managed and will not be automatically released.
++ if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
++ framework.Logf("failed to release static IP %s: %v", staticIPName, err)
++ }
++ }
++ }()
++ framework.ExpectNoError(err, "failed to create region address: %s", staticIPName)
++ reservedAddr, err := gceCloud.GetRegionAddress(staticIPName, gceCloud.Region())
++ framework.ExpectNoError(err, "failed to get region address: %s", staticIPName)
++
++ requestedIP = reservedAddr.Address
++ framework.Logf("Allocated static load balancer IP: %s", requestedIP)
++ }
++
++ ginkgo.By("changing the TCP service to type=LoadBalancer")
++ tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.LoadBalancerIP = requestedIP // will be "" if not applicable
++ s.Spec.Type = v1.ServiceTypeLoadBalancer
++ })
++ framework.ExpectNoError(err)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("changing the UDP service to type=LoadBalancer")
++ udpService, err = udpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Type = v1.ServiceTypeLoadBalancer
++ })
++ framework.ExpectNoError(err)
++ }
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(tcpService))
++ if loadBalancerSupportsUDP {
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(udpService))
++ }
++
++ ginkgo.By("waiting for the TCP service to have a load balancer")
++ // Wait for the load balancer to be created asynchronously
++ tcpService, err = tcpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
++ framework.Failf("TCP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", tcpNodePort, tcpService.Spec.Ports[0].NodePort)
++ }
++ if requestedIP != "" && e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != requestedIP {
++ framework.Failf("unexpected TCP Status.LoadBalancer.Ingress (expected %s, got %s)", requestedIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
++ }
++ tcpIngressIP := e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0])
++ framework.Logf("TCP load balancer: %s", tcpIngressIP)
++
++ if framework.ProviderIs("gce", "gke") {
++ // Do this as early as possible, which overrides the `defer` above.
++ // This is mostly out of fear of leaking the IP in a timeout case
++ // (as of this writing we're not 100% sure where the leaks are
++ // coming from, so this is first-aid rather than surgery).
++ ginkgo.By("demoting the static IP to ephemeral")
++ if staticIPName != "" {
++ gceCloud, err := gce.GetGCECloud()
++ framework.ExpectNoError(err, "failed to get GCE cloud provider")
++ // Deleting it after it is attached "demotes" it to an
++ // ephemeral IP, which can be auto-released.
++ if err := gceCloud.DeleteRegionAddress(staticIPName, gceCloud.Region()); err != nil {
++ framework.Failf("failed to release static IP %s: %v", staticIPName, err)
++ }
++ staticIPName = ""
++ }
++ }
++
++ var udpIngressIP string
++ if loadBalancerSupportsUDP {
++ ginkgo.By("waiting for the UDP service to have a load balancer")
++ // 2nd one should be faster since they ran in parallel.
++ udpService, err = udpJig.WaitForLoadBalancer(loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
++ framework.Failf("UDP Spec.Ports[0].NodePort changed (%d -> %d) when not expected", udpNodePort, udpService.Spec.Ports[0].NodePort)
++ }
++ udpIngressIP = e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0])
++ framework.Logf("UDP load balancer: %s", udpIngressIP)
++
++ ginkgo.By("verifying that TCP and UDP use different load balancers")
++ if tcpIngressIP == udpIngressIP {
++ framework.Failf("Load balancers are not different: %s", e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
++ }
++ }
++
++ ginkgo.By("hitting the TCP service's NodePort")
++ e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the UDP service's NodePort")
++ testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the TCP service's LoadBalancer")
++ e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("hitting the UDP service's LoadBalancer")
++ testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
++ }
++
++ // Change the services' node ports.
++
++ ginkgo.By("changing the TCP service's NodePort")
++ tcpService, err = tcpJig.ChangeServiceNodePort(tcpNodePort)
++ framework.ExpectNoError(err)
++ tcpNodePortOld := tcpNodePort
++ tcpNodePort = int(tcpService.Spec.Ports[0].NodePort)
++ if tcpNodePort == tcpNodePortOld {
++ framework.Failf("TCP Spec.Ports[0].NodePort (%d) did not change", tcpNodePort)
++ }
++ if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
++ framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
++ }
++ framework.Logf("TCP node port: %d", tcpNodePort)
++
++ ginkgo.By("changing the UDP service's NodePort")
++ udpService, err = udpJig.ChangeServiceNodePort(udpNodePort)
++ framework.ExpectNoError(err)
++ udpNodePortOld := udpNodePort
++ udpNodePort = int(udpService.Spec.Ports[0].NodePort)
++ if udpNodePort == udpNodePortOld {
++ framework.Failf("UDP Spec.Ports[0].NodePort (%d) did not change", udpNodePort)
++ }
++ if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
++ framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
++ }
++ framework.Logf("UDP node port: %d", udpNodePort)
++
++ ginkgo.By("hitting the TCP service's new NodePort")
++ e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the UDP service's new NodePort")
++ testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("checking the old TCP NodePort is closed")
++ testNotReachableHTTP(nodeIP, tcpNodePortOld, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("checking the old UDP NodePort is closed")
++ testNotReachableUDP(nodeIP, udpNodePortOld, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the TCP service's LoadBalancer")
++ e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("hitting the UDP service's LoadBalancer")
++ testReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
++ }
++
++ // Change the services' main ports.
++
++ ginkgo.By("changing the TCP service's port")
++ tcpService, err = tcpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Ports[0].Port++
++ })
++ framework.ExpectNoError(err)
++ svcPortOld := svcPort
++ svcPort = int(tcpService.Spec.Ports[0].Port)
++ if svcPort == svcPortOld {
++ framework.Failf("TCP Spec.Ports[0].Port (%d) did not change", svcPort)
++ }
++ if int(tcpService.Spec.Ports[0].NodePort) != tcpNodePort {
++ framework.Failf("TCP Spec.Ports[0].NodePort (%d) changed", tcpService.Spec.Ports[0].NodePort)
++ }
++ if e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]) != tcpIngressIP {
++ framework.Failf("TCP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", tcpIngressIP, e2eservice.GetIngressPoint(&tcpService.Status.LoadBalancer.Ingress[0]))
++ }
++
++ ginkgo.By("changing the UDP service's port")
++ udpService, err = udpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Ports[0].Port++
++ })
++ framework.ExpectNoError(err)
++ if int(udpService.Spec.Ports[0].Port) != svcPort {
++ framework.Failf("UDP Spec.Ports[0].Port (%d) did not change", udpService.Spec.Ports[0].Port)
++ }
++ if int(udpService.Spec.Ports[0].NodePort) != udpNodePort {
++ framework.Failf("UDP Spec.Ports[0].NodePort (%d) changed", udpService.Spec.Ports[0].NodePort)
++ }
++ if loadBalancerSupportsUDP && e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]) != udpIngressIP {
++ framework.Failf("UDP Status.LoadBalancer.Ingress changed (%s -> %s) when not expected", udpIngressIP, e2eservice.GetIngressPoint(&udpService.Status.LoadBalancer.Ingress[0]))
++ }
++
++ framework.Logf("service port (TCP and UDP): %d", svcPort)
++
++ ginkgo.By("hitting the TCP service's NodePort")
++ e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the UDP service's NodePort")
++ testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the TCP service's LoadBalancer")
++ e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("hitting the UDP service's LoadBalancer")
++ testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
++ }
++
++ ginkgo.By("Scaling the pods to 0")
++ err = tcpJig.Scale(0)
++ framework.ExpectNoError(err)
++ err = udpJig.Scale(0)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("looking for ICMP REJECT on the TCP service's NodePort")
++ testRejectedHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("looking for ICMP REJECT on the UDP service's NodePort")
++ testRejectedUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("looking for ICMP REJECT on the TCP service's LoadBalancer")
++ testRejectedHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("looking for ICMP REJECT on the UDP service's LoadBalancer")
++ testRejectedUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
++ }
++
++ ginkgo.By("Scaling the pods to 1")
++ err = tcpJig.Scale(1)
++ framework.ExpectNoError(err)
++ err = udpJig.Scale(1)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("hitting the TCP service's NodePort")
++ e2eservice.TestReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the UDP service's NodePort")
++ testReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("hitting the TCP service's LoadBalancer")
++ e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("hitting the UDP service's LoadBalancer")
++ testReachableUDP(udpIngressIP, svcPort, loadBalancerCreateTimeout)
++ }
++
++ // Change the services back to ClusterIP.
++
++ ginkgo.By("changing TCP service back to type=ClusterIP")
++ _, err = tcpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Type = v1.ServiceTypeClusterIP
++ s.Spec.Ports[0].NodePort = 0
++ })
++ framework.ExpectNoError(err)
++ // Wait for the load balancer to be destroyed asynchronously
++ _, err = tcpJig.WaitForLoadBalancerDestroy(tcpIngressIP, svcPort, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("changing UDP service back to type=ClusterIP")
++ _, err = udpJig.UpdateService(func(s *v1.Service) {
++ s.Spec.Type = v1.ServiceTypeClusterIP
++ s.Spec.Ports[0].NodePort = 0
++ })
++ framework.ExpectNoError(err)
++ if loadBalancerSupportsUDP {
++ // Wait for the load balancer to be destroyed asynchronously
++ _, err = udpJig.WaitForLoadBalancerDestroy(udpIngressIP, svcPort, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ }
++
++ ginkgo.By("checking the TCP NodePort is closed")
++ testNotReachableHTTP(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("checking the UDP NodePort is closed")
++ testNotReachableUDP(nodeIP, udpNodePort, e2eservice.KubeProxyLagTimeout)
++
++ ginkgo.By("checking the TCP LoadBalancer is closed")
++ testNotReachableHTTP(tcpIngressIP, svcPort, loadBalancerLagTimeout)
++
++ if loadBalancerSupportsUDP {
++ ginkgo.By("checking the UDP LoadBalancer is closed")
++ testNotReachableUDP(udpIngressIP, svcPort, loadBalancerLagTimeout)
++ }
++ })
++
++ ginkgo.It("should be able to create an internal type load balancer [Slow]", func() {
++ e2eskipper.SkipUnlessProviderIs("azure", "gke", "gce")
++
++ createTimeout := e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
++ pollInterval := framework.Poll * 10
++
++ namespace := f.Namespace.Name
++ serviceName := "lb-internal"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ ginkgo.By("creating pod to be part of service " + serviceName)
++ _, err := jig.Run(nil)
++ framework.ExpectNoError(err)
++
++ enableILB, disableILB := enableAndDisableInternalLB()
++
++ isInternalEndpoint := func(lbIngress *v1.LoadBalancerIngress) bool {
++ ingressEndpoint := e2eservice.GetIngressPoint(lbIngress)
++ // Needs update for providers using hostname as endpoint.
++ return strings.HasPrefix(ingressEndpoint, "10.")
++ }
++
++ ginkgo.By("creating a service with type LoadBalancer and cloud specific Internal-LB annotation enabled")
++ svc, err := jig.CreateTCPService(func(svc *v1.Service) {
++ svc.Spec.Type = v1.ServiceTypeLoadBalancer
++ enableILB(svc)
++ })
++ framework.ExpectNoError(err)
++
++ defer func() {
++ ginkgo.By("Clean up loadbalancer service")
++ e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
++ }()
++
++ svc, err = jig.WaitForLoadBalancer(createTimeout)
++ framework.ExpectNoError(err)
++ lbIngress := &svc.Status.LoadBalancer.Ingress[0]
++ svcPort := int(svc.Spec.Ports[0].Port)
++ // should have an internal IP.
++ framework.ExpectEqual(isInternalEndpoint(lbIngress), true)
++
++ // ILBs are not accessible from the test orchestrator, so it's necessary to use
++ // a pod to test the service.
++ ginkgo.By("hitting the internal load balancer from pod")
++ framework.Logf("creating pod with host network")
++ hostExec := launchHostExecPod(f.ClientSet, f.Namespace.Name, "ilb-host-exec")
++
++ framework.Logf("Waiting up to %v for service %q's internal LB to respond to requests", createTimeout, serviceName)
++ tcpIngressIP := e2eservice.GetIngressPoint(lbIngress)
++ if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
++ cmd := fmt.Sprintf(`curl -m 5 'http://%v:%v/echo?msg=hello'`, tcpIngressIP, svcPort)
++ stdout, err := framework.RunHostCmd(hostExec.Namespace, hostExec.Name, cmd)
++ if err != nil {
++ framework.Logf("error curling; stdout: %v. err: %v", stdout, err)
++ return false, nil
++ }
++
++ if !strings.Contains(stdout, "hello") {
++ framework.Logf("Expected output to contain 'hello', got %q; retrying...", stdout)
++ return false, nil
++ }
++
++ framework.Logf("Successful curl; stdout: %v", stdout)
++ return true, nil
++ }); pollErr != nil {
++ framework.Failf("ginkgo.Failed to hit ILB IP, err: %v", pollErr)
++ }
++
++ ginkgo.By("switching to external type LoadBalancer")
++ svc, err = jig.UpdateService(func(svc *v1.Service) {
++ disableILB(svc)
++ })
++ framework.ExpectNoError(err)
++ framework.Logf("Waiting up to %v for service %q to have an external LoadBalancer", createTimeout, serviceName)
++ if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
++ svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
++ if err != nil {
++ return false, err
++ }
++ lbIngress = &svc.Status.LoadBalancer.Ingress[0]
++ return !isInternalEndpoint(lbIngress), nil
++ }); pollErr != nil {
++ framework.Failf("Loadbalancer IP not changed to external.")
++ }
++ // should have an external IP.
++ gomega.Expect(isInternalEndpoint(lbIngress)).To(gomega.BeFalse())
++
++ ginkgo.By("hitting the external load balancer")
++ framework.Logf("Waiting up to %v for service %q's external LB to respond to requests", createTimeout, serviceName)
++ tcpIngressIP = e2eservice.GetIngressPoint(lbIngress)
++ e2eservice.TestReachableHTTP(tcpIngressIP, svcPort, e2eservice.LoadBalancerLagTimeoutDefault)
++
++ // GCE cannot test a specific IP because the test may not own it. This cloud specific condition
++ // will be removed when GCP supports similar functionality.
++ if framework.ProviderIs("azure") {
++ ginkgo.By("switching back to interal type LoadBalancer, with static IP specified.")
++ internalStaticIP := "10.240.11.11"
++ svc, err = jig.UpdateService(func(svc *v1.Service) {
++ svc.Spec.LoadBalancerIP = internalStaticIP
++ enableILB(svc)
++ })
++ framework.ExpectNoError(err)
++ framework.Logf("Waiting up to %v for service %q to have an internal LoadBalancer", createTimeout, serviceName)
++ if pollErr := wait.PollImmediate(pollInterval, createTimeout, func() (bool, error) {
++ svc, err := cs.CoreV1().Services(namespace).Get(context.TODO(), serviceName, metav1.GetOptions{})
++ if err != nil {
++ return false, err
++ }
++ lbIngress = &svc.Status.LoadBalancer.Ingress[0]
++ return isInternalEndpoint(lbIngress), nil
++ }); pollErr != nil {
++ framework.Failf("Loadbalancer IP not changed to internal.")
++ }
++ // should have the given static internal IP.
++ framework.ExpectEqual(e2eservice.GetIngressPoint(lbIngress), internalStaticIP)
++ }
++ })
++
++ // This test creates a load balancer, make sure its health check interval
++ // equals to gceHcCheckIntervalSeconds. Then the interval is manipulated
++ // to be something else, see if the interval will be reconciled.
++ ginkgo.It("should reconcile LB health check interval [Slow][Serial]", func() {
++ const gceHcCheckIntervalSeconds = int64(8)
++ // This test is for clusters on GCE.
++ // (It restarts kube-controller-manager, which we don't support on GKE)
++ e2eskipper.SkipUnlessProviderIs("gce")
++ e2eskipper.SkipUnlessSSHKeyPresent()
++
++ clusterID, err := gce.GetClusterID(cs)
++ if err != nil {
++ framework.Failf("framework.GetClusterID(cs) = _, %v; want nil", err)
++ }
++ gceCloud, err := gce.GetGCECloud()
++ if err != nil {
++ framework.Failf("framework.GetGCECloud() = _, %v; want nil", err)
++ }
++
++ namespace := f.Namespace.Name
++ serviceName := "lb-hc-int"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ ginkgo.By("create load balancer service")
++ // Create loadbalancer service with source range from node[0] and podAccept
++ svc, err := jig.CreateTCPService(func(svc *v1.Service) {
++ svc.Spec.Type = v1.ServiceTypeLoadBalancer
++ })
++ framework.ExpectNoError(err)
++
++ defer func() {
++ ginkgo.By("Clean up loadbalancer service")
++ e2eservice.WaitForServiceDeletedWithFinalizer(cs, svc.Namespace, svc.Name)
++ }()
++
++ svc, err = jig.WaitForLoadBalancer(e2eservice.GetServiceLoadBalancerCreationTimeout(cs))
++ framework.ExpectNoError(err)
++
++ hcName := gcecloud.MakeNodesHealthCheckName(clusterID)
++ hc, err := gceCloud.GetHTTPHealthCheck(hcName)
++ if err != nil {
++ framework.Failf("gceCloud.GetHttpHealthCheck(%q) = _, %v; want nil", hcName, err)
++ }
++ framework.ExpectEqual(hc.CheckIntervalSec, gceHcCheckIntervalSeconds)
++
++ ginkgo.By("modify the health check interval")
++ hc.CheckIntervalSec = gceHcCheckIntervalSeconds - 1
++ if err = gceCloud.UpdateHTTPHealthCheck(hc); err != nil {
++ framework.Failf("gcecloud.UpdateHttpHealthCheck(%#v) = %v; want nil", hc, err)
++ }
++
++ ginkgo.By("restart kube-controller-manager")
++ if err := e2ekubesystem.RestartControllerManager(); err != nil {
++ framework.Failf("e2ekubesystem.RestartControllerManager() = %v; want nil", err)
++ }
++ if err := e2ekubesystem.WaitForControllerManagerUp(); err != nil {
++ framework.Failf("e2ekubesystem.WaitForControllerManagerUp() = %v; want nil", err)
++ }
++
++ ginkgo.By("health check should be reconciled")
++ pollInterval := framework.Poll * 10
++ loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
++ if pollErr := wait.PollImmediate(pollInterval, loadBalancerPropagationTimeout, func() (bool, error) {
++ hc, err := gceCloud.GetHTTPHealthCheck(hcName)
++ if err != nil {
++ framework.Logf("ginkgo.Failed to get HttpHealthCheck(%q): %v", hcName, err)
++ return false, err
++ }
++ framework.Logf("hc.CheckIntervalSec = %v", hc.CheckIntervalSec)
++ return hc.CheckIntervalSec == gceHcCheckIntervalSeconds, nil
++ }); pollErr != nil {
++ framework.Failf("Health check %q does not reconcile its check interval to %d.", hcName, gceHcCheckIntervalSeconds)
++ }
++ })
++
++ var _ = SIGDescribe("ESIPP [Slow]", func() {
++ f := framework.NewDefaultFramework("esipp")
++ var loadBalancerCreateTimeout time.Duration
++
++ var cs clientset.Interface
++ serviceLBNames := []string{}
++
++ ginkgo.BeforeEach(func() {
++ // requires cloud load-balancer support - this feature currently supported only on GCE/GKE
++ e2eskipper.SkipUnlessProviderIs("gce", "gke")
++
++ cs = f.ClientSet
++ loadBalancerCreateTimeout = e2eservice.GetServiceLoadBalancerCreationTimeout(cs)
++ })
++
++ ginkgo.AfterEach(func() {
++ if ginkgo.CurrentGinkgoTestDescription().Failed {
++ DescribeSvc(f.Namespace.Name)
++ }
++ for _, lb := range serviceLBNames {
++ framework.Logf("cleaning load balancer resource for %s", lb)
++ e2eservice.CleanupServiceResources(cs, lb, framework.TestContext.CloudConfig.Region, framework.TestContext.CloudConfig.Zone)
++ }
++ //reset serviceLBNames
++ serviceLBNames = []string{}
++ })
++
++ ginkgo.It("should work for type=LoadBalancer", func() {
++ namespace := f.Namespace.Name
++ serviceName := "external-local-lb"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
++ framework.ExpectNoError(err)
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
++ healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
++ if healthCheckNodePort == 0 {
++ framework.Failf("Service HealthCheck NodePort was not allocated")
++ }
++ defer func() {
++ err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++
++ // Make sure we didn't leak the health check node port.
++ threshold := 2
++ nodes, err := jig.GetEndpointNodes()
++ framework.ExpectNoError(err)
++ for _, ips := range nodes {
++ err := TestHTTPHealthCheckNodePort(ips[0], healthCheckNodePort, "/healthz", e2eservice.KubeProxyEndpointLagTimeout, false, threshold)
++ framework.ExpectNoError(err)
++ }
++ err = cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err)
++ }()
++
++ svcTCPPort := int(svc.Spec.Ports[0].Port)
++ ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
++
++ ginkgo.By("reading clientIP using the TCP service's service port via its external VIP")
++ content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
++ clientIP := content.String()
++ framework.Logf("ClientIP detected by target pod using VIP:SvcPort is %s", clientIP)
++
++ ginkgo.By("checking if Source IP is preserved")
++ if strings.HasPrefix(clientIP, "10.") {
++ framework.Failf("Source IP was NOT preserved")
++ }
++ })
++
++ ginkgo.It("should work for type=NodePort", func() {
++ namespace := f.Namespace.Name
++ serviceName := "external-local-nodeport"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ svc, err := jig.CreateOnlyLocalNodePortService(true)
++ framework.ExpectNoError(err)
++ defer func() {
++ err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err)
++ }()
++
++ tcpNodePort := int(svc.Spec.Ports[0].NodePort)
++ endpointsNodeMap, err := jig.GetEndpointNodes()
++ framework.ExpectNoError(err)
++ path := "/clientip"
++
++ for nodeName, nodeIPs := range endpointsNodeMap {
++ nodeIP := nodeIPs[0]
++ ginkgo.By(fmt.Sprintf("reading clientIP using the TCP service's NodePort, on node %v: %v%v%v", nodeName, nodeIP, tcpNodePort, path))
++ content := GetHTTPContent(nodeIP, tcpNodePort, e2eservice.KubeProxyLagTimeout, path)
++ clientIP := content.String()
++ framework.Logf("ClientIP detected by target pod using NodePort is %s", clientIP)
++ if strings.HasPrefix(clientIP, "10.") {
++ framework.Failf("Source IP was NOT preserved")
++ }
++ }
++ })
++
++ ginkgo.It("should only target nodes with endpoints", func() {
++ namespace := f.Namespace.Name
++ serviceName := "external-local-nodes"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++ nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
++ framework.ExpectNoError(err)
++
++ svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, false,
++ func(svc *v1.Service) {
++ // Change service port to avoid collision with opened hostPorts
++ // in other tests that run in parallel.
++ if len(svc.Spec.Ports) != 0 {
++ svc.Spec.Ports[0].TargetPort = intstr.FromInt(int(svc.Spec.Ports[0].Port))
++ svc.Spec.Ports[0].Port = 8081
++ }
++
++ })
++ framework.ExpectNoError(err)
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
++ defer func() {
++ err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err)
++ }()
++
++ healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
++ if healthCheckNodePort == 0 {
++ framework.Failf("Service HealthCheck NodePort was not allocated")
++ }
++
++ ips := e2enode.CollectAddresses(nodes, v1.NodeExternalIP)
++
++ ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
++ svcTCPPort := int(svc.Spec.Ports[0].Port)
++
++ threshold := 2
++ path := "/healthz"
++ for i := 0; i < len(nodes.Items); i++ {
++ endpointNodeName := nodes.Items[i].Name
++
++ ginkgo.By("creating a pod to be part of the service " + serviceName + " on node " + endpointNodeName)
++ _, err = jig.Run(func(rc *v1.ReplicationController) {
++ rc.Name = serviceName
++ if endpointNodeName != "" {
++ rc.Spec.Template.Spec.NodeName = endpointNodeName
++ }
++ })
++ framework.ExpectNoError(err)
++
++ ginkgo.By(fmt.Sprintf("waiting for service endpoint on node %v", endpointNodeName))
++ err = jig.WaitForEndpointOnNode(endpointNodeName)
++ framework.ExpectNoError(err)
++
++ // HealthCheck should pass only on the node where num(endpoints) > 0
++ // All other nodes should fail the healthcheck on the service healthCheckNodePort
++ for n, publicIP := range ips {
++ // Make sure the loadbalancer picked up the health check change.
++ // Confirm traffic can reach backend through LB before checking healthcheck nodeport.
++ e2eservice.TestReachableHTTP(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout)
++ expectedSuccess := nodes.Items[n].Name == endpointNodeName
++ port := strconv.Itoa(healthCheckNodePort)
++ ipPort := net.JoinHostPort(publicIP, port)
++ framework.Logf("Health checking %s, http://%s%s, expectedSuccess %v", nodes.Items[n].Name, ipPort, path, expectedSuccess)
++ err := TestHTTPHealthCheckNodePort(publicIP, healthCheckNodePort, path, e2eservice.KubeProxyEndpointLagTimeout, expectedSuccess, threshold)
++ framework.ExpectNoError(err)
++ }
++ framework.ExpectNoError(e2erc.DeleteRCAndWaitForGC(f.ClientSet, namespace, serviceName))
++ }
++ })
++
++ ginkgo.It("should work from pods", func() {
++ var err error
++ namespace := f.Namespace.Name
++ serviceName := "external-local-pods"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
++ framework.ExpectNoError(err)
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
++ defer func() {
++ err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err)
++ }()
++
++ ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
++ port := strconv.Itoa(int(svc.Spec.Ports[0].Port))
++ ipPort := net.JoinHostPort(ingressIP, port)
++ path := fmt.Sprintf("%s/clientip", ipPort)
++
++ ginkgo.By("Creating pause pod deployment to make sure, pausePods are in desired state")
++ deployment := createPausePodDeployment(cs, "pause-pod-deployment", namespace, 1)
++ framework.ExpectNoError(e2edeployment.WaitForDeploymentComplete(cs, deployment), "Failed to complete pause pod deployment")
++
++ defer func() {
++ framework.Logf("Deleting deployment")
++ err = cs.AppsV1().Deployments(namespace).Delete(context.TODO(), deployment.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err, "Failed to delete deployment %s", deployment.Name)
++ }()
++
++ deployment, err = cs.AppsV1().Deployments(namespace).Get(context.TODO(), deployment.Name, metav1.GetOptions{})
++ framework.ExpectNoError(err, "Error in retrieving pause pod deployment")
++ labelSelector, err := metav1.LabelSelectorAsSelector(deployment.Spec.Selector)
++ framework.ExpectNoError(err, "Error in setting LabelSelector as selector from deployment")
++
++ pausePods, err := cs.CoreV1().Pods(namespace).List(context.TODO(), metav1.ListOptions{LabelSelector: labelSelector.String()})
++ framework.ExpectNoError(err, "Error in listing pods associated with pause pod deployments")
++
++ pausePod := pausePods.Items[0]
++ framework.Logf("Waiting up to %v curl %v", e2eservice.KubeProxyLagTimeout, path)
++ cmd := fmt.Sprintf(`curl -q -s --connect-timeout 30 %v`, path)
++
++ var srcIP string
++ loadBalancerPropagationTimeout := e2eservice.GetServiceLoadBalancerPropagationTimeout(cs)
++ ginkgo.By(fmt.Sprintf("Hitting external lb %v from pod %v on node %v", ingressIP, pausePod.Name, pausePod.Spec.NodeName))
++ if pollErr := wait.PollImmediate(framework.Poll, loadBalancerPropagationTimeout, func() (bool, error) {
++ stdout, err := framework.RunHostCmd(pausePod.Namespace, pausePod.Name, cmd)
++ if err != nil {
++ framework.Logf("got err: %v, retry until timeout", err)
++ return false, nil
++ }
++ srcIP = strings.TrimSpace(strings.Split(stdout, ":")[0])
++ return srcIP == pausePod.Status.PodIP, nil
++ }); pollErr != nil {
++ framework.Failf("Source IP not preserved from %v, expected '%v' got '%v'", pausePod.Name, pausePod.Status.PodIP, srcIP)
++ }
++ })
++
++ // TODO: Get rid of [DisabledForLargeClusters] tag when issue #90047 is fixed.
++ ginkgo.It("should handle updates to ExternalTrafficPolicy field [DisabledForLargeClusters]", func() {
++ namespace := f.Namespace.Name
++ serviceName := "external-local-update"
++ jig := e2eservice.NewTestJig(cs, namespace, serviceName)
++
++ nodes, err := e2enode.GetBoundedReadySchedulableNodes(cs, e2eservice.MaxNodesForEndpointsTests)
++ framework.ExpectNoError(err)
++ if len(nodes.Items) < 2 {
++ framework.Failf("Need at least 2 nodes to verify source ip from a node without endpoint")
++ }
++
++ svc, err := jig.CreateOnlyLocalLoadBalancerService(loadBalancerCreateTimeout, true, nil)
++ framework.ExpectNoError(err)
++ serviceLBNames = append(serviceLBNames, cloudprovider.DefaultLoadBalancerName(svc))
++ defer func() {
++ err = jig.ChangeServiceType(v1.ServiceTypeClusterIP, loadBalancerCreateTimeout)
++ framework.ExpectNoError(err)
++ err := cs.CoreV1().Services(svc.Namespace).Delete(context.TODO(), svc.Name, metav1.DeleteOptions{})
++ framework.ExpectNoError(err)
++ }()
++
++ // save the health check node port because it disappears when ESIPP is turned off.
++ healthCheckNodePort := int(svc.Spec.HealthCheckNodePort)
++
++ ginkgo.By("turning ESIPP off")
++ svc, err = jig.UpdateService(func(svc *v1.Service) {
++ svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeCluster
++ })
++ framework.ExpectNoError(err)
++ if svc.Spec.HealthCheckNodePort > 0 {
++ framework.Failf("Service HealthCheck NodePort still present")
++ }
++
++ endpointNodeMap, err := jig.GetEndpointNodes()
++ framework.ExpectNoError(err)
++ noEndpointNodeMap := map[string][]string{}
++ for _, n := range nodes.Items {
++ if _, ok := endpointNodeMap[n.Name]; ok {
++ continue
++ }
++ noEndpointNodeMap[n.Name] = e2enode.GetAddresses(&n, v1.NodeExternalIP)
++ }
++
++ svcTCPPort := int(svc.Spec.Ports[0].Port)
++ svcNodePort := int(svc.Spec.Ports[0].NodePort)
++ ingressIP := e2eservice.GetIngressPoint(&svc.Status.LoadBalancer.Ingress[0])
++ path := "/clientip"
++
++ ginkgo.By(fmt.Sprintf("endpoints present on nodes %v, absent on nodes %v", endpointNodeMap, noEndpointNodeMap))
++ for nodeName, nodeIPs := range noEndpointNodeMap {
++ ginkgo.By(fmt.Sprintf("Checking %v (%v:%v%v) proxies to endpoints on another node", nodeName, nodeIPs[0], svcNodePort, path))
++ GetHTTPContent(nodeIPs[0], svcNodePort, e2eservice.KubeProxyLagTimeout, path)
++ }
++
++ for nodeName, nodeIPs := range endpointNodeMap {
++ ginkgo.By(fmt.Sprintf("checking kube-proxy health check fails on node with endpoint (%s), public IP %s", nodeName, nodeIPs[0]))
++ var body bytes.Buffer
++ pollfn := func() (bool, error) {
++ result := e2enetwork.PokeHTTP(nodeIPs[0], healthCheckNodePort, "/healthz", nil)
++ if result.Code == 0 {
++ return true, nil
++ }
++ body.Reset()
++ body.Write(result.Body)
++ return false, nil
++ }
++ if pollErr := wait.PollImmediate(framework.Poll, e2eservice.TestTimeout, pollfn); pollErr != nil {
++ framework.Failf("Kube-proxy still exposing health check on node %v:%v, after ESIPP was turned off. body %s",
++ nodeName, healthCheckNodePort, body.String())
++ }
++ }
++
++ // Poll till kube-proxy re-adds the MASQUERADE rule on the node.
++ ginkgo.By(fmt.Sprintf("checking source ip is NOT preserved through loadbalancer %v", ingressIP))
++ var clientIP string
++ pollErr := wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
++ content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, "/clientip")
++ clientIP = content.String()
++ if strings.HasPrefix(clientIP, "10.") {
++ return true, nil
++ }
++ return false, nil
++ })
++ if pollErr != nil {
++ framework.Failf("Source IP WAS preserved even after ESIPP turned off. Got %v, expected a ten-dot cluster ip.", clientIP)
++ }
++
++ // TODO: We need to attempt to create another service with the previously
++ // allocated healthcheck nodePort. If the health check nodePort has been
++ // freed, the new service creation will succeed, upon which we cleanup.
++ // If the health check nodePort has NOT been freed, the new service
++ // creation will fail.
++
++ ginkgo.By("setting ExternalTraffic field back to OnlyLocal")
++ svc, err = jig.UpdateService(func(svc *v1.Service) {
++ svc.Spec.ExternalTrafficPolicy = v1.ServiceExternalTrafficPolicyTypeLocal
++ // Request the same healthCheckNodePort as before, to test the user-requested allocation path
++ svc.Spec.HealthCheckNodePort = int32(healthCheckNodePort)
++ })
++ framework.ExpectNoError(err)
++ pollErr = wait.PollImmediate(framework.Poll, e2eservice.KubeProxyLagTimeout, func() (bool, error) {
++ content := GetHTTPContent(ingressIP, svcTCPPort, e2eservice.KubeProxyLagTimeout, path)
++ clientIP = content.String()
++ ginkgo.By(fmt.Sprintf("Endpoint %v:%v%v returned client ip %v", ingressIP, svcTCPPort, path, clientIP))
++ if !strings.HasPrefix(clientIP, "10.") {
++ return true, nil
++ }
++ return false, nil
++ })
++ if pollErr != nil {
++ framework.Failf("Source IP (%v) is not the client IP even after ESIPP turned on, expected a public IP.", clientIP)
++ }
++ })
++ })
++})
+diff --git a/test/e2e/node/recreate_node.go b/test/e2e/node/recreate_node.go
+index da3fc974485..b403fa7f737 100644
+--- a/test/e2e/node/recreate_node.go
++++ b/test/e2e/node/recreate_node.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2019 The Kubernetes Authors.
+
+diff --git a/test/e2e/scheduling/nvidia-gpus.go b/test/e2e/scheduling/nvidia-gpus.go
+index 334a6f5b9f6..59fbdee014b 100644
+--- a/test/e2e/scheduling/nvidia-gpus.go
++++ b/test/e2e/scheduling/nvidia-gpus.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/scheduling/ubernetes_lite_volumes.go b/test/e2e/scheduling/ubernetes_lite_volumes.go
+index 513ed07543f..78c0f081990 100644
+--- a/test/e2e/scheduling/ubernetes_lite_volumes.go
++++ b/test/e2e/scheduling/ubernetes_lite_volumes.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+diff --git a/test/e2e/storage/drivers/in_tree.go b/test/e2e/storage/drivers/in_tree.go
+index d5183f28081..de25e2cf007 100644
+--- a/test/e2e/storage/drivers/in_tree.go
++++ b/test/e2e/storage/drivers/in_tree.go
+@@ -38,10 +38,8 @@ package drivers
+ import (
+ "context"
+ "fmt"
+- "os/exec"
+ "strconv"
+ "strings"
+- "time"
+
+ "github.com/onsi/ginkgo"
+ v1 "k8s.io/api/core/v1"
+@@ -57,13 +55,11 @@ import (
+ e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
+ e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+ e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+- e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
+ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+ e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
+ "k8s.io/kubernetes/test/e2e/storage/testpatterns"
+ "k8s.io/kubernetes/test/e2e/storage/testsuites"
+ "k8s.io/kubernetes/test/e2e/storage/utils"
+- vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
+ imageutils "k8s.io/kubernetes/test/utils/image"
+ )
+
+@@ -1032,734 +1028,6 @@ func (e *emptydirDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTes
+ }, func() {}
+ }
+
+-// Cinder
+-// This driver assumes that OpenStack client tools are installed
+-// (/usr/bin/nova, /usr/bin/cinder and /usr/bin/keystone)
+-// and that the usual OpenStack authentication env. variables are set
+-// (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least).
+-type cinderDriver struct {
+- driverInfo testsuites.DriverInfo
+-}
+-
+-type cinderVolume struct {
+- volumeName string
+- volumeID string
+-}
+-
+-var _ testsuites.TestDriver = &cinderDriver{}
+-var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
+-var _ testsuites.InlineVolumeTestDriver = &cinderDriver{}
+-var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
+-var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
+-
+-// InitCinderDriver returns cinderDriver that implements TestDriver interface
+-func InitCinderDriver() testsuites.TestDriver {
+- return &cinderDriver{
+- driverInfo: testsuites.DriverInfo{
+- Name: "cinder",
+- InTreePluginName: "kubernetes.io/cinder",
+- MaxFileSize: testpatterns.FileSizeMedium,
+- SupportedSizeRange: e2evolume.SizeRange{
+- Min: "5Gi",
+- },
+- SupportedFsType: sets.NewString(
+- "", // Default fsType
+- "ext3",
+- ),
+- TopologyKeys: []string{v1.LabelZoneFailureDomain},
+- Capabilities: map[testsuites.Capability]bool{
+- testsuites.CapPersistence: true,
+- testsuites.CapFsGroup: true,
+- testsuites.CapExec: true,
+- testsuites.CapBlock: true,
+- // Cinder supports volume limits, but the test creates large
+- // number of volumes and times out test suites.
+- testsuites.CapVolumeLimits: false,
+- testsuites.CapTopology: true,
+- },
+- },
+- }
+-}
+-
+-func (c *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
+- return &c.driverInfo
+-}
+-
+-func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+- e2eskipper.SkipUnlessProviderIs("openstack")
+-}
+-
+-func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
+- cv, ok := e2evolume.(*cinderVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
+-
+- volSource := v1.VolumeSource{
+- Cinder: &v1.CinderVolumeSource{
+- VolumeID: cv.volumeID,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- volSource.Cinder.FSType = fsType
+- }
+- return &volSource
+-}
+-
+-func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+- cv, ok := e2evolume.(*cinderVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
+-
+- pvSource := v1.PersistentVolumeSource{
+- Cinder: &v1.CinderPersistentVolumeSource{
+- VolumeID: cv.volumeID,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- pvSource.Cinder.FSType = fsType
+- }
+- return &pvSource, nil
+-}
+-
+-func (c *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+- provisioner := "kubernetes.io/cinder"
+- parameters := map[string]string{}
+- if fsType != "" {
+- parameters["fsType"] = fsType
+- }
+- ns := config.Framework.Namespace.Name
+- suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name)
+-
+- return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
+-}
+-
+-func (c *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+- return &testsuites.PerTestConfig{
+- Driver: c,
+- Prefix: "cinder",
+- Framework: f,
+- }, func() {}
+-}
+-
+-func (c *cinderDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+- f := config.Framework
+- ns := f.Namespace
+-
+- // We assume that namespace.Name is a random string
+- volumeName := ns.Name
+- ginkgo.By("creating a test Cinder volume")
+- output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput()
+- outputString := string(output[:])
+- framework.Logf("cinder output:\n%s", outputString)
+- framework.ExpectNoError(err)
+-
+- // Parse 'id'' from stdout. Expected format:
+- // | attachments | [] |
+- // | availability_zone | nova |
+- // ...
+- // | id | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 |
+- volumeID := ""
+- for _, line := range strings.Split(outputString, "\n") {
+- fields := strings.Fields(line)
+- if len(fields) != 5 {
+- continue
+- }
+- if fields[1] != "id" {
+- continue
+- }
+- volumeID = fields[3]
+- break
+- }
+- framework.Logf("Volume ID: %s", volumeID)
+- framework.ExpectNotEqual(volumeID, "")
+- return &cinderVolume{
+- volumeName: volumeName,
+- volumeID: volumeID,
+- }
+-}
+-
+-func (v *cinderVolume) DeleteVolume() {
+- name := v.volumeName
+-
+- // Try to delete the volume for several seconds - it takes
+- // a while for the plugin to detach it.
+- var output []byte
+- var err error
+- timeout := time.Second * 120
+-
+- framework.Logf("Waiting up to %v for removal of cinder volume %s", timeout, name)
+- for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
+- output, err = exec.Command("cinder", "delete", name).CombinedOutput()
+- if err == nil {
+- framework.Logf("Cinder volume %s deleted", name)
+- return
+- }
+- framework.Logf("Failed to delete volume %s: %v", name, err)
+- }
+- framework.Logf("Giving up deleting volume %s: %v\n%s", name, err, string(output[:]))
+-}
+-
+-// GCE
+-type gcePdDriver struct {
+- driverInfo testsuites.DriverInfo
+-}
+-
+-type gcePdVolume struct {
+- volumeName string
+-}
+-
+-var _ testsuites.TestDriver = &gcePdDriver{}
+-var _ testsuites.PreprovisionedVolumeTestDriver = &gcePdDriver{}
+-var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
+-var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
+-var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
+-
+-// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
+-func InitGcePdDriver() testsuites.TestDriver {
+- // In current test structure, it first initialize the driver and then set up
+- // the new framework, so we cannot get the correct OS here. So here set to
+- // support all fs types including both linux and windows. We have code to check Node OS later
+- // during test.
+- supportedTypes := sets.NewString(
+- "", // Default fsType
+- "ext2",
+- "ext3",
+- "ext4",
+- "xfs",
+- "ntfs",
+- )
+- return &gcePdDriver{
+- driverInfo: testsuites.DriverInfo{
+- Name: "gcepd",
+- InTreePluginName: "kubernetes.io/gce-pd",
+- MaxFileSize: testpatterns.FileSizeMedium,
+- SupportedSizeRange: e2evolume.SizeRange{
+- Min: "5Gi",
+- },
+- SupportedFsType: supportedTypes,
+- SupportedMountOption: sets.NewString("debug", "nouid32"),
+- TopologyKeys: []string{v1.LabelZoneFailureDomain},
+- Capabilities: map[testsuites.Capability]bool{
+- testsuites.CapPersistence: true,
+- testsuites.CapFsGroup: true,
+- testsuites.CapBlock: true,
+- testsuites.CapExec: true,
+- testsuites.CapMultiPODs: true,
+- testsuites.CapControllerExpansion: true,
+- testsuites.CapNodeExpansion: true,
+- // GCE supports volume limits, but the test creates large
+- // number of volumes and times out test suites.
+- testsuites.CapVolumeLimits: false,
+- testsuites.CapTopology: true,
+- },
+- },
+- }
+-}
+-
+-func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
+- return &g.driverInfo
+-}
+-
+-func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+- e2eskipper.SkipUnlessProviderIs("gce", "gke")
+- if pattern.FeatureTag == "[sig-windows]" {
+- e2eskipper.SkipUnlessNodeOSDistroIs("windows")
+- }
+-}
+-
+-func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
+- gv, ok := e2evolume.(*gcePdVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
+- volSource := v1.VolumeSource{
+- GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+- PDName: gv.volumeName,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- volSource.GCEPersistentDisk.FSType = fsType
+- }
+- return &volSource
+-}
+-
+-func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+- gv, ok := e2evolume.(*gcePdVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
+- pvSource := v1.PersistentVolumeSource{
+- GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+- PDName: gv.volumeName,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- pvSource.GCEPersistentDisk.FSType = fsType
+- }
+- return &pvSource, nil
+-}
+-
+-func (g *gcePdDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+- provisioner := "kubernetes.io/gce-pd"
+- parameters := map[string]string{}
+- if fsType != "" {
+- parameters["fsType"] = fsType
+- }
+- ns := config.Framework.Namespace.Name
+- suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
+- delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
+-
+- return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
+-}
+-
+-func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+- config := &testsuites.PerTestConfig{
+- Driver: g,
+- Prefix: "gcepd",
+- Framework: f,
+- }
+-
+- if framework.NodeOSDistroIs("windows") {
+- config.ClientNodeSelection = e2epod.NodeSelection{
+- Selector: map[string]string{
+- "kubernetes.io/os": "windows",
+- },
+- }
+- }
+- return config, func() {}
+-
+-}
+-
+-func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+- zone := getInlineVolumeZone(config.Framework)
+- if volType == testpatterns.InlineVolume {
+- // PD will be created in framework.TestContext.CloudConfig.Zone zone,
+- // so pods should be also scheduled there.
+- config.ClientNodeSelection = e2epod.NodeSelection{
+- Selector: map[string]string{
+- v1.LabelZoneFailureDomain: zone,
+- },
+- }
+- }
+- ginkgo.By("creating a test gce pd volume")
+- vname, err := e2epv.CreatePDWithRetryAndZone(zone)
+- framework.ExpectNoError(err)
+- return &gcePdVolume{
+- volumeName: vname,
+- }
+-}
+-
+-func (v *gcePdVolume) DeleteVolume() {
+- e2epv.DeletePDWithRetry(v.volumeName)
+-}
+-
+-// vSphere
+-type vSphereDriver struct {
+- driverInfo testsuites.DriverInfo
+-}
+-
+-type vSphereVolume struct {
+- volumePath string
+- nodeInfo *vspheretest.NodeInfo
+-}
+-
+-var _ testsuites.TestDriver = &vSphereDriver{}
+-var _ testsuites.PreprovisionedVolumeTestDriver = &vSphereDriver{}
+-var _ testsuites.InlineVolumeTestDriver = &vSphereDriver{}
+-var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{}
+-var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
+-
+-// InitVSphereDriver returns vSphereDriver that implements TestDriver interface
+-func InitVSphereDriver() testsuites.TestDriver {
+- return &vSphereDriver{
+- driverInfo: testsuites.DriverInfo{
+- Name: "vsphere",
+- InTreePluginName: "kubernetes.io/vsphere-volume",
+- MaxFileSize: testpatterns.FileSizeMedium,
+- SupportedSizeRange: e2evolume.SizeRange{
+- Min: "5Gi",
+- },
+- SupportedFsType: sets.NewString(
+- "", // Default fsType
+- "ext4",
+- ),
+- TopologyKeys: []string{v1.LabelZoneFailureDomain},
+- Capabilities: map[testsuites.Capability]bool{
+- testsuites.CapPersistence: true,
+- testsuites.CapFsGroup: true,
+- testsuites.CapExec: true,
+- testsuites.CapMultiPODs: true,
+- testsuites.CapTopology: true,
+- },
+- },
+- }
+-}
+-func (v *vSphereDriver) GetDriverInfo() *testsuites.DriverInfo {
+- return &v.driverInfo
+-}
+-
+-func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+- e2eskipper.SkipUnlessProviderIs("vsphere")
+-}
+-
+-func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
+- vsv, ok := e2evolume.(*vSphereVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
+-
+- // vSphere driver doesn't seem to support readOnly volume
+- // TODO: check if it is correct
+- if readOnly {
+- return nil
+- }
+- volSource := v1.VolumeSource{
+- VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+- VolumePath: vsv.volumePath,
+- },
+- }
+- if fsType != "" {
+- volSource.VsphereVolume.FSType = fsType
+- }
+- return &volSource
+-}
+-
+-func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+- vsv, ok := e2evolume.(*vSphereVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
+-
+- // vSphere driver doesn't seem to support readOnly volume
+- // TODO: check if it is correct
+- if readOnly {
+- return nil, nil
+- }
+- pvSource := v1.PersistentVolumeSource{
+- VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+- VolumePath: vsv.volumePath,
+- },
+- }
+- if fsType != "" {
+- pvSource.VsphereVolume.FSType = fsType
+- }
+- return &pvSource, nil
+-}
+-
+-func (v *vSphereDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+- provisioner := "kubernetes.io/vsphere-volume"
+- parameters := map[string]string{}
+- if fsType != "" {
+- parameters["fsType"] = fsType
+- }
+- ns := config.Framework.Namespace.Name
+- suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name)
+-
+- return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
+-}
+-
+-func (v *vSphereDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+- return &testsuites.PerTestConfig{
+- Driver: v,
+- Prefix: "vsphere",
+- Framework: f,
+- }, func() {}
+-}
+-
+-func (v *vSphereDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+- f := config.Framework
+- vspheretest.Bootstrap(f)
+- nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo()
+- volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef)
+- framework.ExpectNoError(err)
+- return &vSphereVolume{
+- volumePath: volumePath,
+- nodeInfo: nodeInfo,
+- }
+-}
+-
+-func (v *vSphereVolume) DeleteVolume() {
+- v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef)
+-}
+-
+-// Azure Disk
+-type azureDiskDriver struct {
+- driverInfo testsuites.DriverInfo
+-}
+-
+-type azureDiskVolume struct {
+- volumeName string
+-}
+-
+-var _ testsuites.TestDriver = &azureDiskDriver{}
+-var _ testsuites.PreprovisionedVolumeTestDriver = &azureDiskDriver{}
+-var _ testsuites.InlineVolumeTestDriver = &azureDiskDriver{}
+-var _ testsuites.PreprovisionedPVTestDriver = &azureDiskDriver{}
+-var _ testsuites.DynamicPVTestDriver = &azureDiskDriver{}
+-
+-// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
+-func InitAzureDiskDriver() testsuites.TestDriver {
+- return &azureDiskDriver{
+- driverInfo: testsuites.DriverInfo{
+- Name: "azure-disk",
+- InTreePluginName: "kubernetes.io/azure-disk",
+- MaxFileSize: testpatterns.FileSizeMedium,
+- SupportedSizeRange: e2evolume.SizeRange{
+- Min: "5Gi",
+- },
+- SupportedFsType: sets.NewString(
+- "", // Default fsType
+- "ext3",
+- "ext4",
+- "xfs",
+- ),
+- TopologyKeys: []string{v1.LabelZoneFailureDomain},
+- Capabilities: map[testsuites.Capability]bool{
+- testsuites.CapPersistence: true,
+- testsuites.CapFsGroup: true,
+- testsuites.CapBlock: true,
+- testsuites.CapExec: true,
+- testsuites.CapMultiPODs: true,
+- // Azure supports volume limits, but the test creates large
+- // number of volumes and times out test suites.
+- testsuites.CapVolumeLimits: false,
+- testsuites.CapTopology: true,
+- },
+- },
+- }
+-}
+-
+-func (a *azureDiskDriver) GetDriverInfo() *testsuites.DriverInfo {
+- return &a.driverInfo
+-}
+-
+-func (a *azureDiskDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+- e2eskipper.SkipUnlessProviderIs("azure")
+-}
+-
+-func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
+- av, ok := e2evolume.(*azureDiskVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
+- diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
+-
+- kind := v1.AzureManagedDisk
+- volSource := v1.VolumeSource{
+- AzureDisk: &v1.AzureDiskVolumeSource{
+- DiskName: diskName,
+- DataDiskURI: av.volumeName,
+- Kind: &kind,
+- ReadOnly: &readOnly,
+- },
+- }
+- if fsType != "" {
+- volSource.AzureDisk.FSType = &fsType
+- }
+- return &volSource
+-}
+-
+-func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+- av, ok := e2evolume.(*azureDiskVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
+-
+- diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
+-
+- kind := v1.AzureManagedDisk
+- pvSource := v1.PersistentVolumeSource{
+- AzureDisk: &v1.AzureDiskVolumeSource{
+- DiskName: diskName,
+- DataDiskURI: av.volumeName,
+- Kind: &kind,
+- ReadOnly: &readOnly,
+- },
+- }
+- if fsType != "" {
+- pvSource.AzureDisk.FSType = &fsType
+- }
+- return &pvSource, nil
+-}
+-
+-func (a *azureDiskDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+- provisioner := "kubernetes.io/azure-disk"
+- parameters := map[string]string{}
+- if fsType != "" {
+- parameters["fsType"] = fsType
+- }
+- ns := config.Framework.Namespace.Name
+- suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
+- delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
+-
+- return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
+-}
+-
+-func (a *azureDiskDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+- return &testsuites.PerTestConfig{
+- Driver: a,
+- Prefix: "azure",
+- Framework: f,
+- }, func() {}
+-}
+-
+-func (a *azureDiskDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+- ginkgo.By("creating a test azure disk volume")
+- zone := getInlineVolumeZone(config.Framework)
+- if volType == testpatterns.InlineVolume {
+- // PD will be created in framework.TestContext.CloudConfig.Zone zone,
+- // so pods should be also scheduled there.
+- config.ClientNodeSelection = e2epod.NodeSelection{
+- Selector: map[string]string{
+- v1.LabelZoneFailureDomain: zone,
+- },
+- }
+- }
+- volumeName, err := e2epv.CreatePDWithRetryAndZone(zone)
+- framework.ExpectNoError(err)
+- return &azureDiskVolume{
+- volumeName: volumeName,
+- }
+-}
+-
+-func (v *azureDiskVolume) DeleteVolume() {
+- e2epv.DeletePDWithRetry(v.volumeName)
+-}
+-
+-// AWS
+-type awsDriver struct {
+- driverInfo testsuites.DriverInfo
+-}
+-
+-type awsVolume struct {
+- volumeName string
+-}
+-
+-var _ testsuites.TestDriver = &awsDriver{}
+-
+-var _ testsuites.PreprovisionedVolumeTestDriver = &awsDriver{}
+-var _ testsuites.InlineVolumeTestDriver = &awsDriver{}
+-var _ testsuites.PreprovisionedPVTestDriver = &awsDriver{}
+-var _ testsuites.DynamicPVTestDriver = &awsDriver{}
+-
+-// InitAwsDriver returns awsDriver that implements TestDriver interface
+-func InitAwsDriver() testsuites.TestDriver {
+- return &awsDriver{
+- driverInfo: testsuites.DriverInfo{
+- Name: "aws",
+- InTreePluginName: "kubernetes.io/aws-ebs",
+- MaxFileSize: testpatterns.FileSizeMedium,
+- SupportedSizeRange: e2evolume.SizeRange{
+- Min: "5Gi",
+- },
+- SupportedFsType: sets.NewString(
+- "", // Default fsType
+- "ext2",
+- "ext3",
+- "ext4",
+- "xfs",
+- "ntfs",
+- ),
+- SupportedMountOption: sets.NewString("debug", "nouid32"),
+- TopologyKeys: []string{v1.LabelZoneFailureDomain},
+- Capabilities: map[testsuites.Capability]bool{
+- testsuites.CapPersistence: true,
+- testsuites.CapFsGroup: true,
+- testsuites.CapBlock: true,
+- testsuites.CapExec: true,
+- testsuites.CapMultiPODs: true,
+- testsuites.CapControllerExpansion: true,
+- testsuites.CapNodeExpansion: true,
+- // AWS supports volume limits, but the test creates large
+- // number of volumes and times out test suites.
+- testsuites.CapVolumeLimits: false,
+- testsuites.CapTopology: true,
+- },
+- },
+- }
+-}
+-
+-func (a *awsDriver) GetDriverInfo() *testsuites.DriverInfo {
+- return &a.driverInfo
+-}
+-
+-func (a *awsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
+- e2eskipper.SkipUnlessProviderIs("aws")
+-}
+-
+-func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
+- av, ok := e2evolume.(*awsVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
+- volSource := v1.VolumeSource{
+- AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
+- VolumeID: av.volumeName,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- volSource.AWSElasticBlockStore.FSType = fsType
+- }
+- return &volSource
+-}
+-
+-func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
+- av, ok := e2evolume.(*awsVolume)
+- framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
+- pvSource := v1.PersistentVolumeSource{
+- AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
+- VolumeID: av.volumeName,
+- ReadOnly: readOnly,
+- },
+- }
+- if fsType != "" {
+- pvSource.AWSElasticBlockStore.FSType = fsType
+- }
+- return &pvSource, nil
+-}
+-
+-func (a *awsDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
+- provisioner := "kubernetes.io/aws-ebs"
+- parameters := map[string]string{}
+- if fsType != "" {
+- parameters["fsType"] = fsType
+- }
+- ns := config.Framework.Namespace.Name
+- suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
+- delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
+-
+- return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
+-}
+-
+-func (a *awsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
+- config := &testsuites.PerTestConfig{
+- Driver: a,
+- Prefix: "aws",
+- Framework: f,
+- }
+-
+- if framework.NodeOSDistroIs("windows") {
+- config.ClientNodeSelection = e2epod.NodeSelection{
+- Selector: map[string]string{
+- "kubernetes.io/os": "windows",
+- },
+- }
+- }
+- return config, func() {}
+-}
+-
+-func (a *awsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
+- zone := getInlineVolumeZone(config.Framework)
+- if volType == testpatterns.InlineVolume {
+- // PD will be created in framework.TestContext.CloudConfig.Zone zone,
+- // so pods should be also scheduled there.
+- config.ClientNodeSelection = e2epod.NodeSelection{
+- Selector: map[string]string{
+- v1.LabelZoneFailureDomain: zone,
+- },
+- }
+- }
+- ginkgo.By("creating a test aws volume")
+- vname, err := e2epv.CreatePDWithRetryAndZone(zone)
+- framework.ExpectNoError(err)
+- return &awsVolume{
+- volumeName: vname,
+- }
+-}
+-
+-func (v *awsVolume) DeleteVolume() {
+- e2epv.DeletePDWithRetry(v.volumeName)
+-}
+-
+ // local
+ type localDriver struct {
+ driverInfo testsuites.DriverInfo
+diff --git a/test/e2e/storage/drivers/in_tree_providers.go b/test/e2e/storage/drivers/in_tree_providers.go
+new file mode 100644
+index 00000000000..c7f5dd3052e
+--- /dev/null
++++ b/test/e2e/storage/drivers/in_tree_providers.go
+@@ -0,0 +1,751 @@
++// +build !providerless
++
++package drivers
++
++import (
++ "fmt"
++ "os/exec"
++ "strings"
++ "time"
++
++ "github.com/onsi/ginkgo"
++ v1 "k8s.io/api/core/v1"
++ storagev1 "k8s.io/api/storage/v1"
++ "k8s.io/apimachinery/pkg/util/sets"
++ "k8s.io/kubernetes/test/e2e/framework"
++ e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
++ e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
++ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
++ e2evolume "k8s.io/kubernetes/test/e2e/framework/volume"
++ "k8s.io/kubernetes/test/e2e/storage/testpatterns"
++ "k8s.io/kubernetes/test/e2e/storage/testsuites"
++ vspheretest "k8s.io/kubernetes/test/e2e/storage/vsphere"
++)
++
++// Cinder
++// This driver assumes that OpenStack client tools are installed
++// (/usr/bin/nova, /usr/bin/cinder and /usr/bin/keystone)
++// and that the usual OpenStack authentication env. variables are set
++// (OS_USERNAME, OS_PASSWORD, OS_TENANT_NAME at least).
++type cinderDriver struct {
++ driverInfo testsuites.DriverInfo
++}
++
++type cinderVolume struct {
++ volumeName string
++ volumeID string
++}
++
++var _ testsuites.TestDriver = &cinderDriver{}
++var _ testsuites.PreprovisionedVolumeTestDriver = &cinderDriver{}
++var _ testsuites.InlineVolumeTestDriver = &cinderDriver{}
++var _ testsuites.PreprovisionedPVTestDriver = &cinderDriver{}
++var _ testsuites.DynamicPVTestDriver = &cinderDriver{}
++
++// InitCinderDriver returns cinderDriver that implements TestDriver interface
++func InitCinderDriver() testsuites.TestDriver {
++ return &cinderDriver{
++ driverInfo: testsuites.DriverInfo{
++ Name: "cinder",
++ InTreePluginName: "kubernetes.io/cinder",
++ MaxFileSize: testpatterns.FileSizeMedium,
++ SupportedSizeRange: e2evolume.SizeRange{
++ Min: "5Gi",
++ },
++ SupportedFsType: sets.NewString(
++ "", // Default fsType
++ "ext3",
++ ),
++ TopologyKeys: []string{v1.LabelZoneFailureDomain},
++ Capabilities: map[testsuites.Capability]bool{
++ testsuites.CapPersistence: true,
++ testsuites.CapFsGroup: true,
++ testsuites.CapExec: true,
++ testsuites.CapBlock: true,
++ // Cinder supports volume limits, but the test creates large
++ // number of volumes and times out test suites.
++ testsuites.CapVolumeLimits: false,
++ testsuites.CapTopology: true,
++ },
++ },
++ }
++}
++
++func (c *cinderDriver) GetDriverInfo() *testsuites.DriverInfo {
++ return &c.driverInfo
++}
++
++func (c *cinderDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
++ e2eskipper.SkipUnlessProviderIs("openstack")
++}
++
++func (c *cinderDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
++ cv, ok := e2evolume.(*cinderVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
++
++ volSource := v1.VolumeSource{
++ Cinder: &v1.CinderVolumeSource{
++ VolumeID: cv.volumeID,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ volSource.Cinder.FSType = fsType
++ }
++ return &volSource
++}
++
++func (c *cinderDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
++ cv, ok := e2evolume.(*cinderVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to Cinder test volume")
++
++ pvSource := v1.PersistentVolumeSource{
++ Cinder: &v1.CinderPersistentVolumeSource{
++ VolumeID: cv.volumeID,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ pvSource.Cinder.FSType = fsType
++ }
++ return &pvSource, nil
++}
++
++func (c *cinderDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
++ provisioner := "kubernetes.io/cinder"
++ parameters := map[string]string{}
++ if fsType != "" {
++ parameters["fsType"] = fsType
++ }
++ ns := config.Framework.Namespace.Name
++ suffix := fmt.Sprintf("%s-sc", c.driverInfo.Name)
++
++ return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
++}
++
++func (c *cinderDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
++ return &testsuites.PerTestConfig{
++ Driver: c,
++ Prefix: "cinder",
++ Framework: f,
++ }, func() {}
++}
++
++func (c *cinderDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
++ f := config.Framework
++ ns := f.Namespace
++
++ // We assume that namespace.Name is a random string
++ volumeName := ns.Name
++ ginkgo.By("creating a test Cinder volume")
++ output, err := exec.Command("cinder", "create", "--display-name="+volumeName, "1").CombinedOutput()
++ outputString := string(output[:])
++ framework.Logf("cinder output:\n%s", outputString)
++ framework.ExpectNoError(err)
++
++ // Parse 'id'' from stdout. Expected format:
++ // | attachments | [] |
++ // | availability_zone | nova |
++ // ...
++ // | id | 1d6ff08f-5d1c-41a4-ad72-4ef872cae685 |
++ volumeID := ""
++ for _, line := range strings.Split(outputString, "\n") {
++ fields := strings.Fields(line)
++ if len(fields) != 5 {
++ continue
++ }
++ if fields[1] != "id" {
++ continue
++ }
++ volumeID = fields[3]
++ break
++ }
++ framework.Logf("Volume ID: %s", volumeID)
++ framework.ExpectNotEqual(volumeID, "")
++ return &cinderVolume{
++ volumeName: volumeName,
++ volumeID: volumeID,
++ }
++}
++
++func (v *cinderVolume) DeleteVolume() {
++ name := v.volumeName
++
++ // Try to delete the volume for several seconds - it takes
++ // a while for the plugin to detach it.
++ var output []byte
++ var err error
++ timeout := time.Second * 120
++
++ framework.Logf("Waiting up to %v for removal of cinder volume %s", timeout, name)
++ for start := time.Now(); time.Since(start) < timeout; time.Sleep(5 * time.Second) {
++ output, err = exec.Command("cinder", "delete", name).CombinedOutput()
++ if err == nil {
++ framework.Logf("Cinder volume %s deleted", name)
++ return
++ }
++ framework.Logf("Failed to delete volume %s: %v", name, err)
++ }
++ framework.Logf("Giving up deleting volume %s: %v\n%s", name, err, string(output[:]))
++}
++
++// GCE
++type gcePdDriver struct {
++ driverInfo testsuites.DriverInfo
++}
++
++type gcePdVolume struct {
++ volumeName string
++}
++
++var _ testsuites.TestDriver = &gcePdDriver{}
++var _ testsuites.PreprovisionedVolumeTestDriver = &gcePdDriver{}
++var _ testsuites.InlineVolumeTestDriver = &gcePdDriver{}
++var _ testsuites.PreprovisionedPVTestDriver = &gcePdDriver{}
++var _ testsuites.DynamicPVTestDriver = &gcePdDriver{}
++
++// InitGcePdDriver returns gcePdDriver that implements TestDriver interface
++func InitGcePdDriver() testsuites.TestDriver {
++ // In current test structure, it first initialize the driver and then set up
++ // the new framework, so we cannot get the correct OS here. So here set to
++ // support all fs types including both linux and windows. We have code to check Node OS later
++ // during test.
++ supportedTypes := sets.NewString(
++ "", // Default fsType
++ "ext2",
++ "ext3",
++ "ext4",
++ "xfs",
++ "ntfs",
++ )
++ return &gcePdDriver{
++ driverInfo: testsuites.DriverInfo{
++ Name: "gcepd",
++ InTreePluginName: "kubernetes.io/gce-pd",
++ MaxFileSize: testpatterns.FileSizeMedium,
++ SupportedSizeRange: e2evolume.SizeRange{
++ Min: "5Gi",
++ },
++ SupportedFsType: supportedTypes,
++ SupportedMountOption: sets.NewString("debug", "nouid32"),
++ TopologyKeys: []string{v1.LabelZoneFailureDomain},
++ Capabilities: map[testsuites.Capability]bool{
++ testsuites.CapPersistence: true,
++ testsuites.CapFsGroup: true,
++ testsuites.CapBlock: true,
++ testsuites.CapExec: true,
++ testsuites.CapMultiPODs: true,
++ testsuites.CapControllerExpansion: true,
++ testsuites.CapNodeExpansion: true,
++ // GCE supports volume limits, but the test creates large
++ // number of volumes and times out test suites.
++ testsuites.CapVolumeLimits: false,
++ testsuites.CapTopology: true,
++ },
++ },
++ }
++}
++
++func (g *gcePdDriver) GetDriverInfo() *testsuites.DriverInfo {
++ return &g.driverInfo
++}
++
++func (g *gcePdDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
++ e2eskipper.SkipUnlessProviderIs("gce", "gke")
++ if pattern.FeatureTag == "[sig-windows]" {
++ e2eskipper.SkipUnlessNodeOSDistroIs("windows")
++ }
++}
++
++func (g *gcePdDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
++ gv, ok := e2evolume.(*gcePdVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
++ volSource := v1.VolumeSource{
++ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
++ PDName: gv.volumeName,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ volSource.GCEPersistentDisk.FSType = fsType
++ }
++ return &volSource
++}
++
++func (g *gcePdDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
++ gv, ok := e2evolume.(*gcePdVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to GCE PD test volume")
++ pvSource := v1.PersistentVolumeSource{
++ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
++ PDName: gv.volumeName,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ pvSource.GCEPersistentDisk.FSType = fsType
++ }
++ return &pvSource, nil
++}
++
++func (g *gcePdDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
++ provisioner := "kubernetes.io/gce-pd"
++ parameters := map[string]string{}
++ if fsType != "" {
++ parameters["fsType"] = fsType
++ }
++ ns := config.Framework.Namespace.Name
++ suffix := fmt.Sprintf("%s-sc", g.driverInfo.Name)
++ delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
++
++ return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
++}
++
++func (g *gcePdDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
++ config := &testsuites.PerTestConfig{
++ Driver: g,
++ Prefix: "gcepd",
++ Framework: f,
++ }
++
++ if framework.NodeOSDistroIs("windows") {
++ config.ClientNodeSelection = e2epod.NodeSelection{
++ Selector: map[string]string{
++ "kubernetes.io/os": "windows",
++ },
++ }
++ }
++ return config, func() {}
++
++}
++
++func (g *gcePdDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
++ zone := getInlineVolumeZone(config.Framework)
++ if volType == testpatterns.InlineVolume {
++ // PD will be created in framework.TestContext.CloudConfig.Zone zone,
++ // so pods should be also scheduled there.
++ config.ClientNodeSelection = e2epod.NodeSelection{
++ Selector: map[string]string{
++ v1.LabelZoneFailureDomain: zone,
++ },
++ }
++ }
++ ginkgo.By("creating a test gce pd volume")
++ vname, err := e2epv.CreatePDWithRetryAndZone(zone)
++ framework.ExpectNoError(err)
++ return &gcePdVolume{
++ volumeName: vname,
++ }
++}
++
++func (v *gcePdVolume) DeleteVolume() {
++ e2epv.DeletePDWithRetry(v.volumeName)
++}
++
++// vSphere
++type vSphereDriver struct {
++ driverInfo testsuites.DriverInfo
++}
++
++type vSphereVolume struct {
++ volumePath string
++ nodeInfo *vspheretest.NodeInfo
++}
++
++var _ testsuites.TestDriver = &vSphereDriver{}
++var _ testsuites.PreprovisionedVolumeTestDriver = &vSphereDriver{}
++var _ testsuites.InlineVolumeTestDriver = &vSphereDriver{}
++var _ testsuites.PreprovisionedPVTestDriver = &vSphereDriver{}
++var _ testsuites.DynamicPVTestDriver = &vSphereDriver{}
++
++// InitVSphereDriver returns vSphereDriver that implements TestDriver interface
++func InitVSphereDriver() testsuites.TestDriver {
++ return &vSphereDriver{
++ driverInfo: testsuites.DriverInfo{
++ Name: "vsphere",
++ InTreePluginName: "kubernetes.io/vsphere-volume",
++ MaxFileSize: testpatterns.FileSizeMedium,
++ SupportedSizeRange: e2evolume.SizeRange{
++ Min: "5Gi",
++ },
++ SupportedFsType: sets.NewString(
++ "", // Default fsType
++ "ext4",
++ ),
++ TopologyKeys: []string{v1.LabelZoneFailureDomain},
++ Capabilities: map[testsuites.Capability]bool{
++ testsuites.CapPersistence: true,
++ testsuites.CapFsGroup: true,
++ testsuites.CapExec: true,
++ testsuites.CapMultiPODs: true,
++ testsuites.CapTopology: true,
++ },
++ },
++ }
++}
++func (v *vSphereDriver) GetDriverInfo() *testsuites.DriverInfo {
++ return &v.driverInfo
++}
++
++func (v *vSphereDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
++ e2eskipper.SkipUnlessProviderIs("vsphere")
++}
++
++func (v *vSphereDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
++ vsv, ok := e2evolume.(*vSphereVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
++
++ // vSphere driver doesn't seem to support readOnly volume
++ // TODO: check if it is correct
++ if readOnly {
++ return nil
++ }
++ volSource := v1.VolumeSource{
++ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
++ VolumePath: vsv.volumePath,
++ },
++ }
++ if fsType != "" {
++ volSource.VsphereVolume.FSType = fsType
++ }
++ return &volSource
++}
++
++func (v *vSphereDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
++ vsv, ok := e2evolume.(*vSphereVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to vSphere test volume")
++
++ // vSphere driver doesn't seem to support readOnly volume
++ // TODO: check if it is correct
++ if readOnly {
++ return nil, nil
++ }
++ pvSource := v1.PersistentVolumeSource{
++ VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
++ VolumePath: vsv.volumePath,
++ },
++ }
++ if fsType != "" {
++ pvSource.VsphereVolume.FSType = fsType
++ }
++ return &pvSource, nil
++}
++
++func (v *vSphereDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
++ provisioner := "kubernetes.io/vsphere-volume"
++ parameters := map[string]string{}
++ if fsType != "" {
++ parameters["fsType"] = fsType
++ }
++ ns := config.Framework.Namespace.Name
++ suffix := fmt.Sprintf("%s-sc", v.driverInfo.Name)
++
++ return testsuites.GetStorageClass(provisioner, parameters, nil, ns, suffix)
++}
++
++func (v *vSphereDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
++ return &testsuites.PerTestConfig{
++ Driver: v,
++ Prefix: "vsphere",
++ Framework: f,
++ }, func() {}
++}
++
++func (v *vSphereDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
++ f := config.Framework
++ vspheretest.Bootstrap(f)
++ nodeInfo := vspheretest.GetReadySchedulableRandomNodeInfo()
++ volumePath, err := nodeInfo.VSphere.CreateVolume(&vspheretest.VolumeOptions{}, nodeInfo.DataCenterRef)
++ framework.ExpectNoError(err)
++ return &vSphereVolume{
++ volumePath: volumePath,
++ nodeInfo: nodeInfo,
++ }
++}
++
++func (v *vSphereVolume) DeleteVolume() {
++ v.nodeInfo.VSphere.DeleteVolume(v.volumePath, v.nodeInfo.DataCenterRef)
++}
++
++// Azure Disk
++type azureDiskDriver struct {
++ driverInfo testsuites.DriverInfo
++}
++
++type azureDiskVolume struct {
++ volumeName string
++}
++
++var _ testsuites.TestDriver = &azureDiskDriver{}
++var _ testsuites.PreprovisionedVolumeTestDriver = &azureDiskDriver{}
++var _ testsuites.InlineVolumeTestDriver = &azureDiskDriver{}
++var _ testsuites.PreprovisionedPVTestDriver = &azureDiskDriver{}
++var _ testsuites.DynamicPVTestDriver = &azureDiskDriver{}
++
++// InitAzureDiskDriver returns azureDiskDriver that implements TestDriver interface
++func InitAzureDiskDriver() testsuites.TestDriver {
++ return &azureDiskDriver{
++ driverInfo: testsuites.DriverInfo{
++ Name: "azure-disk",
++ InTreePluginName: "kubernetes.io/azure-disk",
++ MaxFileSize: testpatterns.FileSizeMedium,
++ SupportedSizeRange: e2evolume.SizeRange{
++ Min: "5Gi",
++ },
++ SupportedFsType: sets.NewString(
++ "", // Default fsType
++ "ext3",
++ "ext4",
++ "xfs",
++ ),
++ TopologyKeys: []string{v1.LabelZoneFailureDomain},
++ Capabilities: map[testsuites.Capability]bool{
++ testsuites.CapPersistence: true,
++ testsuites.CapFsGroup: true,
++ testsuites.CapBlock: true,
++ testsuites.CapExec: true,
++ testsuites.CapMultiPODs: true,
++ // Azure supports volume limits, but the test creates large
++ // number of volumes and times out test suites.
++ testsuites.CapVolumeLimits: false,
++ testsuites.CapTopology: true,
++ },
++ },
++ }
++}
++
++func (a *azureDiskDriver) GetDriverInfo() *testsuites.DriverInfo {
++ return &a.driverInfo
++}
++
++func (a *azureDiskDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
++ e2eskipper.SkipUnlessProviderIs("azure")
++}
++
++func (a *azureDiskDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
++ av, ok := e2evolume.(*azureDiskVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
++ diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
++
++ kind := v1.AzureManagedDisk
++ volSource := v1.VolumeSource{
++ AzureDisk: &v1.AzureDiskVolumeSource{
++ DiskName: diskName,
++ DataDiskURI: av.volumeName,
++ Kind: &kind,
++ ReadOnly: &readOnly,
++ },
++ }
++ if fsType != "" {
++ volSource.AzureDisk.FSType = &fsType
++ }
++ return &volSource
++}
++
++func (a *azureDiskDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
++ av, ok := e2evolume.(*azureDiskVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to Azure test volume")
++
++ diskName := av.volumeName[(strings.LastIndex(av.volumeName, "/") + 1):]
++
++ kind := v1.AzureManagedDisk
++ pvSource := v1.PersistentVolumeSource{
++ AzureDisk: &v1.AzureDiskVolumeSource{
++ DiskName: diskName,
++ DataDiskURI: av.volumeName,
++ Kind: &kind,
++ ReadOnly: &readOnly,
++ },
++ }
++ if fsType != "" {
++ pvSource.AzureDisk.FSType = &fsType
++ }
++ return &pvSource, nil
++}
++
++func (a *azureDiskDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
++ provisioner := "kubernetes.io/azure-disk"
++ parameters := map[string]string{}
++ if fsType != "" {
++ parameters["fsType"] = fsType
++ }
++ ns := config.Framework.Namespace.Name
++ suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
++ delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
++
++ return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
++}
++
++func (a *azureDiskDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
++ return &testsuites.PerTestConfig{
++ Driver: a,
++ Prefix: "azure",
++ Framework: f,
++ }, func() {}
++}
++
++func (a *azureDiskDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
++ ginkgo.By("creating a test azure disk volume")
++ zone := getInlineVolumeZone(config.Framework)
++ if volType == testpatterns.InlineVolume {
++ // PD will be created in framework.TestContext.CloudConfig.Zone zone,
++ // so pods should be also scheduled there.
++ config.ClientNodeSelection = e2epod.NodeSelection{
++ Selector: map[string]string{
++ v1.LabelZoneFailureDomain: zone,
++ },
++ }
++ }
++ volumeName, err := e2epv.CreatePDWithRetryAndZone(zone)
++ framework.ExpectNoError(err)
++ return &azureDiskVolume{
++ volumeName: volumeName,
++ }
++}
++
++func (v *azureDiskVolume) DeleteVolume() {
++ e2epv.DeletePDWithRetry(v.volumeName)
++}
++
++// AWS
++type awsDriver struct {
++ driverInfo testsuites.DriverInfo
++}
++
++type awsVolume struct {
++ volumeName string
++}
++
++var _ testsuites.TestDriver = &awsDriver{}
++
++var _ testsuites.PreprovisionedVolumeTestDriver = &awsDriver{}
++var _ testsuites.InlineVolumeTestDriver = &awsDriver{}
++var _ testsuites.PreprovisionedPVTestDriver = &awsDriver{}
++var _ testsuites.DynamicPVTestDriver = &awsDriver{}
++
++// InitAwsDriver returns awsDriver that implements TestDriver interface
++func InitAwsDriver() testsuites.TestDriver {
++ return &awsDriver{
++ driverInfo: testsuites.DriverInfo{
++ Name: "aws",
++ InTreePluginName: "kubernetes.io/aws-ebs",
++ MaxFileSize: testpatterns.FileSizeMedium,
++ SupportedSizeRange: e2evolume.SizeRange{
++ Min: "5Gi",
++ },
++ SupportedFsType: sets.NewString(
++ "", // Default fsType
++ "ext2",
++ "ext3",
++ "ext4",
++ "xfs",
++ "ntfs",
++ ),
++ SupportedMountOption: sets.NewString("debug", "nouid32"),
++ TopologyKeys: []string{v1.LabelZoneFailureDomain},
++ Capabilities: map[testsuites.Capability]bool{
++ testsuites.CapPersistence: true,
++ testsuites.CapFsGroup: true,
++ testsuites.CapBlock: true,
++ testsuites.CapExec: true,
++ testsuites.CapMultiPODs: true,
++ testsuites.CapControllerExpansion: true,
++ testsuites.CapNodeExpansion: true,
++ // AWS supports volume limits, but the test creates large
++ // number of volumes and times out test suites.
++ testsuites.CapVolumeLimits: false,
++ testsuites.CapTopology: true,
++ },
++ },
++ }
++}
++
++func (a *awsDriver) GetDriverInfo() *testsuites.DriverInfo {
++ return &a.driverInfo
++}
++
++func (a *awsDriver) SkipUnsupportedTest(pattern testpatterns.TestPattern) {
++ e2eskipper.SkipUnlessProviderIs("aws")
++}
++
++func (a *awsDriver) GetVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) *v1.VolumeSource {
++ av, ok := e2evolume.(*awsVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
++ volSource := v1.VolumeSource{
++ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
++ VolumeID: av.volumeName,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ volSource.AWSElasticBlockStore.FSType = fsType
++ }
++ return &volSource
++}
++
++func (a *awsDriver) GetPersistentVolumeSource(readOnly bool, fsType string, e2evolume testsuites.TestVolume) (*v1.PersistentVolumeSource, *v1.VolumeNodeAffinity) {
++ av, ok := e2evolume.(*awsVolume)
++ framework.ExpectEqual(ok, true, "Failed to cast test volume to AWS test volume")
++ pvSource := v1.PersistentVolumeSource{
++ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
++ VolumeID: av.volumeName,
++ ReadOnly: readOnly,
++ },
++ }
++ if fsType != "" {
++ pvSource.AWSElasticBlockStore.FSType = fsType
++ }
++ return &pvSource, nil
++}
++
++func (a *awsDriver) GetDynamicProvisionStorageClass(config *testsuites.PerTestConfig, fsType string) *storagev1.StorageClass {
++ provisioner := "kubernetes.io/aws-ebs"
++ parameters := map[string]string{}
++ if fsType != "" {
++ parameters["fsType"] = fsType
++ }
++ ns := config.Framework.Namespace.Name
++ suffix := fmt.Sprintf("%s-sc", a.driverInfo.Name)
++ delayedBinding := storagev1.VolumeBindingWaitForFirstConsumer
++
++ return testsuites.GetStorageClass(provisioner, parameters, &delayedBinding, ns, suffix)
++}
++
++func (a *awsDriver) PrepareTest(f *framework.Framework) (*testsuites.PerTestConfig, func()) {
++ config := &testsuites.PerTestConfig{
++ Driver: a,
++ Prefix: "aws",
++ Framework: f,
++ }
++
++ if framework.NodeOSDistroIs("windows") {
++ config.ClientNodeSelection = e2epod.NodeSelection{
++ Selector: map[string]string{
++ "kubernetes.io/os": "windows",
++ },
++ }
++ }
++ return config, func() {}
++}
++
++func (a *awsDriver) CreateVolume(config *testsuites.PerTestConfig, volType testpatterns.TestVolType) testsuites.TestVolume {
++ zone := getInlineVolumeZone(config.Framework)
++ if volType == testpatterns.InlineVolume {
++ // PD will be created in framework.TestContext.CloudConfig.Zone zone,
++ // so pods should be also scheduled there.
++ config.ClientNodeSelection = e2epod.NodeSelection{
++ Selector: map[string]string{
++ v1.LabelZoneFailureDomain: zone,
++ },
++ }
++ }
++ ginkgo.By("creating a test aws volume")
++ vname, err := e2epv.CreatePDWithRetryAndZone(zone)
++ framework.ExpectNoError(err)
++ return &awsVolume{
++ volumeName: vname,
++ }
++}
++
++func (v *awsVolume) DeleteVolume() {
++ e2epv.DeletePDWithRetry(v.volumeName)
++}
+diff --git a/test/e2e/storage/in_tree_volumes.go b/test/e2e/storage/in_tree_volumes.go
+index 19372062407..8322db743cd 100644
+--- a/test/e2e/storage/in_tree_volumes.go
++++ b/test/e2e/storage/in_tree_volumes.go
+@@ -33,11 +33,6 @@ var testDrivers = []func() testsuites.TestDriver{
+ drivers.InitHostPathDriver,
+ drivers.InitHostPathSymlinkDriver,
+ drivers.InitEmptydirDriver,
+- drivers.InitCinderDriver,
+- drivers.InitGcePdDriver,
+- drivers.InitVSphereDriver,
+- drivers.InitAzureDiskDriver,
+- drivers.InitAwsDriver,
+ drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectory),
+ drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryLink),
+ drivers.InitLocalDriverWithVolumeType(utils.LocalVolumeDirectoryBindMounted),
+diff --git a/test/e2e/storage/in_tree_volumes_providers.go b/test/e2e/storage/in_tree_volumes_providers.go
+new file mode 100644
+index 00000000000..d6a5dbca191
+--- /dev/null
++++ b/test/e2e/storage/in_tree_volumes_providers.go
+@@ -0,0 +1,46 @@
++// +build !providerless
++
++/*
++Copyright 2020 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package storage
++
++import (
++ "github.com/onsi/ginkgo"
++ "k8s.io/kubernetes/test/e2e/storage/drivers"
++ "k8s.io/kubernetes/test/e2e/storage/testsuites"
++ "k8s.io/kubernetes/test/e2e/storage/utils"
++)
++
++// List of testDrivers to be executed in below loop
++var testDriversProviders = []func() testsuites.TestDriver{
++ drivers.InitCinderDriver,
++ drivers.InitGcePdDriver,
++ drivers.InitVSphereDriver,
++ drivers.InitAzureDiskDriver,
++ drivers.InitAwsDriver,
++}
++
++// This executes testSuites for in-tree volumes.
++var _ = utils.SIGDescribe("In-tree Volumes for Cloud Providers", func() {
++ for _, initDriver := range testDriversProviders {
++ curDriver := initDriver()
++
++ ginkgo.Context(testsuites.GetDriverNameWithFeatureTags(curDriver), func() {
++ testsuites.DefineTestSuite(curDriver, testsuites.BaseSuites)
++ })
++ }
++})
+diff --git a/test/e2e/storage/nfs_persistent_volume-disruptive.go b/test/e2e/storage/nfs_persistent_volume-disruptive.go
+index 5afebb5e903..b197eee99a6 100644
+--- a/test/e2e/storage/nfs_persistent_volume-disruptive.go
++++ b/test/e2e/storage/nfs_persistent_volume-disruptive.go
+@@ -91,7 +91,7 @@ var _ = utils.SIGDescribe("NFSPersistentVolumes[Disruptive][Flaky]", func() {
+
+ ginkgo.BeforeEach(func() {
+ // To protect the NFS volume pod from the kubelet restart, we isolate it on its own node.
+- e2eskipper.SkipUnlessNodeCountIsAtLeast(minNodes)
++ e2eskipper.SkipUnlessNodeCountIsAtLeast(2)
+ e2eskipper.SkipIfProviderIs("local")
+
+ c = f.ClientSet
+diff --git a/test/e2e/storage/pd.go b/test/e2e/storage/pd.go
+index b6d720406aa..86fa0cae488 100644
+--- a/test/e2e/storage/pd.go
++++ b/test/e2e/storage/pd.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2015 The Kubernetes Authors.
+
+diff --git a/test/e2e/storage/persistent_volumes-gce.go b/test/e2e/storage/persistent_volumes-gce.go
+index 8b0343e4b58..76a80042811 100644
+--- a/test/e2e/storage/persistent_volumes-gce.go
++++ b/test/e2e/storage/persistent_volumes-gce.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+@@ -18,6 +20,7 @@ package storage
+
+ import (
+ "context"
++
+ "github.com/onsi/ginkgo"
+ v1 "k8s.io/api/core/v1"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+diff --git a/test/e2e/storage/regional_pd.go b/test/e2e/storage/regional_pd.go
+index 7763afaf6b1..a042dcc9d4a 100644
+--- a/test/e2e/storage/regional_pd.go
++++ b/test/e2e/storage/regional_pd.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2016 The Kubernetes Authors.
+
+@@ -18,6 +20,7 @@ package storage
+
+ import (
+ "context"
++
+ "github.com/onsi/ginkgo"
+ "github.com/onsi/gomega"
+
+diff --git a/test/e2e/storage/volume_provisioning.go b/test/e2e/storage/volume_provisioning.go
+index a8b494eb3ac..c070a81283c 100644
+--- a/test/e2e/storage/volume_provisioning.go
++++ b/test/e2e/storage/volume_provisioning.go
+@@ -24,11 +24,6 @@ import (
+ "time"
+
+ "github.com/onsi/ginkgo"
+- "github.com/onsi/gomega"
+-
+- "github.com/aws/aws-sdk-go/aws"
+- "github.com/aws/aws-sdk-go/aws/session"
+- "github.com/aws/aws-sdk-go/service/ec2"
+
+ v1 "k8s.io/api/core/v1"
+ rbacv1 "k8s.io/api/rbac/v1"
+@@ -37,9 +32,7 @@ import (
+ apierrors "k8s.io/apimachinery/pkg/api/errors"
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+ "k8s.io/apimachinery/pkg/runtime/schema"
+- "k8s.io/apimachinery/pkg/types"
+ "k8s.io/apimachinery/pkg/util/rand"
+- "k8s.io/apimachinery/pkg/util/sets"
+ "k8s.io/apimachinery/pkg/util/wait"
+ "k8s.io/apiserver/pkg/authentication/serviceaccount"
+ clientset "k8s.io/client-go/kubernetes"
+@@ -48,7 +41,6 @@ import (
+ e2eauth "k8s.io/kubernetes/test/e2e/framework/auth"
+ e2enode "k8s.io/kubernetes/test/e2e/framework/node"
+ e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+- "k8s.io/kubernetes/test/e2e/framework/providers/gce"
+ e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
+ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
+ "k8s.io/kubernetes/test/e2e/storage/testsuites"
+@@ -61,80 +53,6 @@ const (
+ externalPluginName = "example.com/nfs"
+ )
+
+-// checkAWSEBS checks properties of an AWS EBS. Test framework does not
+-// instantiate full AWS provider, therefore we need use ec2 API directly.
+-func checkAWSEBS(volume *v1.PersistentVolume, volumeType string, encrypted bool) error {
+- diskName := volume.Spec.AWSElasticBlockStore.VolumeID
+-
+- var client *ec2.EC2
+-
+- tokens := strings.Split(diskName, "/")
+- volumeID := tokens[len(tokens)-1]
+-
+- zone := framework.TestContext.CloudConfig.Zone
+-
+- awsSession, err := session.NewSession()
+- if err != nil {
+- return fmt.Errorf("error creating session: %v", err)
+- }
+-
+- if len(zone) > 0 {
+- region := zone[:len(zone)-1]
+- cfg := aws.Config{Region: ®ion}
+- framework.Logf("using region %s", region)
+- client = ec2.New(awsSession, &cfg)
+- } else {
+- framework.Logf("no region configured")
+- client = ec2.New(awsSession)
+- }
+-
+- request := &ec2.DescribeVolumesInput{
+- VolumeIds: []*string{&volumeID},
+- }
+- info, err := client.DescribeVolumes(request)
+- if err != nil {
+- return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
+- }
+- if len(info.Volumes) == 0 {
+- return fmt.Errorf("no volumes found for volume %q", volumeID)
+- }
+- if len(info.Volumes) > 1 {
+- return fmt.Errorf("multiple volumes found for volume %q", volumeID)
+- }
+-
+- awsVolume := info.Volumes[0]
+- if awsVolume.VolumeType == nil {
+- return fmt.Errorf("expected volume type %q, got nil", volumeType)
+- }
+- if *awsVolume.VolumeType != volumeType {
+- return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
+- }
+- if encrypted && awsVolume.Encrypted == nil {
+- return fmt.Errorf("expected encrypted volume, got no encryption")
+- }
+- if encrypted && !*awsVolume.Encrypted {
+- return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
+- }
+- return nil
+-}
+-
+-func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
+- cloud, err := gce.GetGCECloud()
+- if err != nil {
+- return err
+- }
+- diskName := volume.Spec.GCEPersistentDisk.PDName
+- disk, err := cloud.GetDiskByNameUnknownZone(diskName)
+- if err != nil {
+- return err
+- }
+-
+- if !strings.HasSuffix(disk.Type, volumeType) {
+- return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
+- }
+- return nil
+-}
+-
+ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
+ f := framework.NewDefaultFramework("volume-provisioning")
+
+@@ -147,451 +65,6 @@ var _ = utils.SIGDescribe("Dynamic Provisioning", func() {
+ ns = f.Namespace.Name
+ })
+
+- ginkgo.Describe("DynamicProvisioner [Slow]", func() {
+- ginkgo.It("should provision storage with different parameters", func() {
+-
+- // This test checks that dynamic provisioning can provision a volume
+- // that can be used to persist data among pods.
+- tests := []testsuites.StorageClassTest{
+- // GCE/GKE
+- {
+- Name: "SSD PD on GCE/GKE",
+- CloudProviders: []string{"gce", "gke"},
+- Provisioner: "kubernetes.io/gce-pd",
+- Parameters: map[string]string{
+- "type": "pd-ssd",
+- "zone": getRandomClusterZone(c),
+- },
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "2Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkGCEPD(volume, "pd-ssd")
+- framework.ExpectNoError(err, "checkGCEPD pd-ssd")
+- },
+- },
+- {
+- Name: "HDD PD on GCE/GKE",
+- CloudProviders: []string{"gce", "gke"},
+- Provisioner: "kubernetes.io/gce-pd",
+- Parameters: map[string]string{
+- "type": "pd-standard",
+- },
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "2Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkGCEPD(volume, "pd-standard")
+- framework.ExpectNoError(err, "checkGCEPD pd-standard")
+- },
+- },
+- // AWS
+- {
+- Name: "gp2 EBS on AWS",
+- CloudProviders: []string{"aws"},
+- Provisioner: "kubernetes.io/aws-ebs",
+- Parameters: map[string]string{
+- "type": "gp2",
+- "zone": getRandomClusterZone(c),
+- },
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "2Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkAWSEBS(volume, "gp2", false)
+- framework.ExpectNoError(err, "checkAWSEBS gp2")
+- },
+- },
+- {
+- Name: "io1 EBS on AWS",
+- CloudProviders: []string{"aws"},
+- Provisioner: "kubernetes.io/aws-ebs",
+- Parameters: map[string]string{
+- "type": "io1",
+- "iopsPerGB": "50",
+- },
+- ClaimSize: "3.5Gi",
+- ExpectedSize: "4Gi", // 4 GiB is minimum for io1
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkAWSEBS(volume, "io1", false)
+- framework.ExpectNoError(err, "checkAWSEBS io1")
+- },
+- },
+- {
+- Name: "sc1 EBS on AWS",
+- CloudProviders: []string{"aws"},
+- Provisioner: "kubernetes.io/aws-ebs",
+- Parameters: map[string]string{
+- "type": "sc1",
+- },
+- ClaimSize: "500Gi", // minimum for sc1
+- ExpectedSize: "500Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkAWSEBS(volume, "sc1", false)
+- framework.ExpectNoError(err, "checkAWSEBS sc1")
+- },
+- },
+- {
+- Name: "st1 EBS on AWS",
+- CloudProviders: []string{"aws"},
+- Provisioner: "kubernetes.io/aws-ebs",
+- Parameters: map[string]string{
+- "type": "st1",
+- },
+- ClaimSize: "500Gi", // minimum for st1
+- ExpectedSize: "500Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkAWSEBS(volume, "st1", false)
+- framework.ExpectNoError(err, "checkAWSEBS st1")
+- },
+- },
+- {
+- Name: "encrypted EBS on AWS",
+- CloudProviders: []string{"aws"},
+- Provisioner: "kubernetes.io/aws-ebs",
+- Parameters: map[string]string{
+- "encrypted": "true",
+- },
+- ClaimSize: "1Gi",
+- ExpectedSize: "1Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkAWSEBS(volume, "gp2", true)
+- framework.ExpectNoError(err, "checkAWSEBS gp2 encrypted")
+- },
+- },
+- // OpenStack generic tests (works on all OpenStack deployments)
+- {
+- Name: "generic Cinder volume on OpenStack",
+- CloudProviders: []string{"openstack"},
+- Provisioner: "kubernetes.io/cinder",
+- Parameters: map[string]string{},
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "2Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- },
+- },
+- {
+- Name: "Cinder volume with empty volume type and zone on OpenStack",
+- CloudProviders: []string{"openstack"},
+- Provisioner: "kubernetes.io/cinder",
+- Parameters: map[string]string{
+- "type": "",
+- "availability": "",
+- },
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "2Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- },
+- },
+- // vSphere generic test
+- {
+- Name: "generic vSphere volume",
+- CloudProviders: []string{"vsphere"},
+- Provisioner: "kubernetes.io/vsphere-volume",
+- Parameters: map[string]string{},
+- ClaimSize: "1.5Gi",
+- ExpectedSize: "1.5Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- },
+- },
+- // Azure
+- {
+- Name: "Azure disk volume with empty sku and location",
+- CloudProviders: []string{"azure"},
+- Provisioner: "kubernetes.io/azure-disk",
+- Parameters: map[string]string{},
+- ClaimSize: "1Gi",
+- ExpectedSize: "1Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- },
+- },
+- }
+-
+- var betaTest *testsuites.StorageClassTest
+- for i, t := range tests {
+- // Beware of clojure, use local variables instead of those from
+- // outer scope
+- test := t
+-
+- if !framework.ProviderIs(test.CloudProviders...) {
+- framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
+- continue
+- }
+-
+- // Remember the last supported test for subsequent test of beta API
+- betaTest = &test
+-
+- ginkgo.By("Testing " + test.Name)
+- suffix := fmt.Sprintf("%d", i)
+- test.Client = c
+- test.Class = newStorageClass(test, ns, suffix)
+- test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+- ClaimSize: test.ClaimSize,
+- StorageClassName: &test.Class.Name,
+- VolumeMode: &test.VolumeMode,
+- }, ns)
+- test.TestDynamicProvisioning()
+- }
+-
+- // Run the last test with storage.k8s.io/v1beta1 on pvc
+- if betaTest != nil {
+- ginkgo.By("Testing " + betaTest.Name + " with beta volume provisioning")
+- class := newBetaStorageClass(*betaTest, "beta")
+- // we need to create the class manually, testDynamicProvisioning does not accept beta class
+- class, err := c.StorageV1beta1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
+- framework.ExpectNoError(err)
+- defer deleteStorageClass(c, class.Name)
+-
+- betaTest.Client = c
+- betaTest.Class = nil
+- betaTest.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+- ClaimSize: betaTest.ClaimSize,
+- StorageClassName: &class.Name,
+- VolumeMode: &betaTest.VolumeMode,
+- }, ns)
+- betaTest.Claim.Spec.StorageClassName = &(class.Name)
+- (*betaTest).TestDynamicProvisioning()
+- }
+- })
+-
+- ginkgo.It("should provision storage with non-default reclaim policy Retain", func() {
+- e2eskipper.SkipUnlessProviderIs("gce", "gke")
+-
+- test := testsuites.StorageClassTest{
+- Client: c,
+- Name: "HDD PD on GCE/GKE",
+- CloudProviders: []string{"gce", "gke"},
+- Provisioner: "kubernetes.io/gce-pd",
+- Parameters: map[string]string{
+- "type": "pd-standard",
+- },
+- ClaimSize: "1Gi",
+- ExpectedSize: "1Gi",
+- PvCheck: func(claim *v1.PersistentVolumeClaim) {
+- volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
+- gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
+-
+- err := checkGCEPD(volume, "pd-standard")
+- framework.ExpectNoError(err, "checkGCEPD")
+- },
+- }
+- test.Class = newStorageClass(test, ns, "reclaimpolicy")
+- retain := v1.PersistentVolumeReclaimRetain
+- test.Class.ReclaimPolicy = &retain
+- test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+- ClaimSize: test.ClaimSize,
+- StorageClassName: &test.Class.Name,
+- VolumeMode: &test.VolumeMode,
+- }, ns)
+- pv := test.TestDynamicProvisioning()
+-
+- ginkgo.By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
+- framework.ExpectNoError(e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
+-
+- ginkgo.By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
+- framework.ExpectNoError(e2epv.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName))
+-
+- ginkgo.By(fmt.Sprintf("deleting the PV %q", pv.Name))
+- framework.ExpectNoError(e2epv.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
+- framework.ExpectNoError(e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second))
+- })
+-
+- ginkgo.It("should not provision a volume in an unmanaged GCE zone.", func() {
+- e2eskipper.SkipUnlessProviderIs("gce", "gke")
+- var suffix string = "unmananged"
+-
+- ginkgo.By("Discovering an unmanaged zone")
+- allZones := sets.NewString() // all zones in the project
+-
+- gceCloud, err := gce.GetGCECloud()
+- framework.ExpectNoError(err)
+-
+- // Get all k8s managed zones (same as zones with nodes in them for test)
+- managedZones, err := gceCloud.GetAllZonesFromCloudProvider()
+- framework.ExpectNoError(err)
+-
+- // Get a list of all zones in the project
+- zones, err := gceCloud.ComputeServices().GA.Zones.List(framework.TestContext.CloudConfig.ProjectID).Do()
+- framework.ExpectNoError(err)
+- for _, z := range zones.Items {
+- allZones.Insert(z.Name)
+- }
+-
+- // Get the subset of zones not managed by k8s
+- var unmanagedZone string
+- var popped bool
+- unmanagedZones := allZones.Difference(managedZones)
+- // And select one of them at random.
+- if unmanagedZone, popped = unmanagedZones.PopAny(); !popped {
+- e2eskipper.Skipf("No unmanaged zones found.")
+- }
+-
+- ginkgo.By("Creating a StorageClass for the unmanaged zone")
+- test := testsuites.StorageClassTest{
+- Name: "unmanaged_zone",
+- Provisioner: "kubernetes.io/gce-pd",
+- Parameters: map[string]string{"zone": unmanagedZone},
+- ClaimSize: "1Gi",
+- }
+- sc := newStorageClass(test, ns, suffix)
+- sc, err = c.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
+- framework.ExpectNoError(err)
+- defer deleteStorageClass(c, sc.Name)
+-
+- ginkgo.By("Creating a claim and expecting it to timeout")
+- pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+- ClaimSize: test.ClaimSize,
+- StorageClassName: &sc.Name,
+- VolumeMode: &test.VolumeMode,
+- }, ns)
+- pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{})
+- framework.ExpectNoError(err)
+- defer func() {
+- framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
+- }()
+-
+- // The claim should timeout phase:Pending
+- err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
+- framework.ExpectError(err)
+- framework.Logf(err.Error())
+- })
+-
+- ginkgo.It("should test that deleting a claim before the volume is provisioned deletes the volume.", func() {
+- // This case tests for the regressions of a bug fixed by PR #21268
+- // REGRESSION: Deleting the PVC before the PV is provisioned can result in the PV
+- // not being deleted.
+- // NOTE: Polls until no PVs are detected, times out at 5 minutes.
+-
+- e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
+-
+- const raceAttempts int = 100
+- var residualPVs []*v1.PersistentVolume
+- ginkgo.By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
+- test := testsuites.StorageClassTest{
+- Name: "deletion race",
+- Provisioner: "", // Use a native one based on current cloud provider
+- ClaimSize: "1Gi",
+- }
+-
+- class := newStorageClass(test, ns, "race")
+- class, err := c.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
+- framework.ExpectNoError(err)
+- defer deleteStorageClass(c, class.Name)
+-
+- // To increase chance of detection, attempt multiple iterations
+- for i := 0; i < raceAttempts; i++ {
+- prefix := fmt.Sprintf("race-%d", i)
+- claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
+- NamePrefix: prefix,
+- ClaimSize: test.ClaimSize,
+- StorageClassName: &class.Name,
+- VolumeMode: &test.VolumeMode,
+- }, ns)
+- tmpClaim, err := e2epv.CreatePVC(c, ns, claim)
+- framework.ExpectNoError(err)
+- framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns))
+- }
+-
+- ginkgo.By(fmt.Sprintf("Checking for residual PersistentVolumes associated with StorageClass %s", class.Name))
+- residualPVs, err = waitForProvisionedVolumesDeleted(c, class.Name)
+- // Cleanup the test resources before breaking
+- defer deleteProvisionedVolumesAndDisks(c, residualPVs)
+- framework.ExpectNoError(err, "PersistentVolumes were not deleted as expected. %d remain", len(residualPVs))
+-
+- framework.Logf("0 PersistentVolumes remain.")
+- })
+-
+- ginkgo.It("deletion should be idempotent", func() {
+- // This test ensures that deletion of a volume is idempotent.
+- // It creates a PV with Retain policy, deletes underlying AWS / GCE
+- // volume and changes the reclaim policy to Delete.
+- // PV controller should delete the PV even though the underlying volume
+- // is already deleted.
+- e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
+- ginkgo.By("creating PD")
+- diskName, err := e2epv.CreatePDWithRetry()
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("creating PV")
+- pv := e2epv.MakePersistentVolume(e2epv.PersistentVolumeConfig{
+- NamePrefix: "volume-idempotent-delete-",
+- // Use Retain to keep the PV, the test will change it to Delete
+- // when the time comes.
+- ReclaimPolicy: v1.PersistentVolumeReclaimRetain,
+- AccessModes: []v1.PersistentVolumeAccessMode{
+- v1.ReadWriteOnce,
+- },
+- Capacity: "1Gi",
+- // PV is bound to non-existing PVC, so it's reclaim policy is
+- // executed immediately
+- Prebind: &v1.PersistentVolumeClaim{
+- ObjectMeta: metav1.ObjectMeta{
+- Name: "dummy-claim-name",
+- Namespace: ns,
+- UID: types.UID("01234567890"),
+- },
+- },
+- })
+- switch framework.TestContext.Provider {
+- case "aws":
+- pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
+- AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
+- VolumeID: diskName,
+- },
+- }
+- case "gce", "gke":
+- pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
+- GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
+- PDName: diskName,
+- },
+- }
+- }
+- pv, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("waiting for the PV to get Released")
+- err = e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 2*time.Second, e2epv.PVReclaimingTimeout)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("deleting the PD")
+- err = e2epv.DeletePVSource(&pv.Spec.PersistentVolumeSource)
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("changing the PV reclaim policy")
+- pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
+- framework.ExpectNoError(err)
+- pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
+- pv, err = c.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{})
+- framework.ExpectNoError(err)
+-
+- ginkgo.By("waiting for the PV to get deleted")
+- err = e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 5*time.Second, e2epv.PVDeletingTimeout)
+- framework.ExpectNoError(err)
+- })
+- })
+-
+ ginkgo.Describe("DynamicProvisioner External", func() {
+ ginkgo.It("should let an external dynamic provisioner create and delete persistent volumes [Slow]", func() {
+ // external dynamic provisioner pods need additional permissions provided by the
+diff --git a/test/e2e/storage/volume_provisioning_providers.go b/test/e2e/storage/volume_provisioning_providers.go
+new file mode 100644
+index 00000000000..932c644af7a
+--- /dev/null
++++ b/test/e2e/storage/volume_provisioning_providers.go
+@@ -0,0 +1,577 @@
++// +build !providerless
++
++/*
++Copyright 2016 The Kubernetes Authors.
++
++Licensed under the Apache License, Version 2.0 (the "License");
++you may not use this file except in compliance with the License.
++You may obtain a copy of the License at
++
++ http://www.apache.org/licenses/LICENSE-2.0
++
++Unless required by applicable law or agreed to in writing, software
++distributed under the License is distributed on an "AS IS" BASIS,
++WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++See the License for the specific language governing permissions and
++limitations under the License.
++*/
++
++package storage
++
++import (
++ "context"
++ "fmt"
++ "strings"
++ "time"
++
++ "github.com/aws/aws-sdk-go/aws"
++ "github.com/aws/aws-sdk-go/aws/session"
++ "github.com/aws/aws-sdk-go/service/ec2"
++ "github.com/onsi/ginkgo"
++ "github.com/onsi/gomega"
++
++ v1 "k8s.io/api/core/v1"
++ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
++ "k8s.io/apimachinery/pkg/types"
++ "k8s.io/apimachinery/pkg/util/sets"
++ clientset "k8s.io/client-go/kubernetes"
++ "k8s.io/kubernetes/test/e2e/framework"
++ e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
++ "k8s.io/kubernetes/test/e2e/framework/providers/gce"
++ e2epv "k8s.io/kubernetes/test/e2e/framework/pv"
++ e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
++ "k8s.io/kubernetes/test/e2e/storage/testsuites"
++ "k8s.io/kubernetes/test/e2e/storage/utils"
++)
++
++// checkAWSEBS checks properties of an AWS EBS. Test framework does not
++// instantiate full AWS provider, therefore we need use ec2 API directly.
++func checkAWSEBS(volume *v1.PersistentVolume, volumeType string, encrypted bool) error {
++ diskName := volume.Spec.AWSElasticBlockStore.VolumeID
++
++ var client *ec2.EC2
++
++ tokens := strings.Split(diskName, "/")
++ volumeID := tokens[len(tokens)-1]
++
++ zone := framework.TestContext.CloudConfig.Zone
++
++ awsSession, err := session.NewSession()
++ if err != nil {
++ return fmt.Errorf("error creating session: %v", err)
++ }
++
++ if len(zone) > 0 {
++ region := zone[:len(zone)-1]
++ cfg := aws.Config{Region: ®ion}
++ framework.Logf("using region %s", region)
++ client = ec2.New(awsSession, &cfg)
++ } else {
++ framework.Logf("no region configured")
++ client = ec2.New(awsSession)
++ }
++
++ request := &ec2.DescribeVolumesInput{
++ VolumeIds: []*string{&volumeID},
++ }
++ info, err := client.DescribeVolumes(request)
++ if err != nil {
++ return fmt.Errorf("error querying ec2 for volume %q: %v", volumeID, err)
++ }
++ if len(info.Volumes) == 0 {
++ return fmt.Errorf("no volumes found for volume %q", volumeID)
++ }
++ if len(info.Volumes) > 1 {
++ return fmt.Errorf("multiple volumes found for volume %q", volumeID)
++ }
++
++ awsVolume := info.Volumes[0]
++ if awsVolume.VolumeType == nil {
++ return fmt.Errorf("expected volume type %q, got nil", volumeType)
++ }
++ if *awsVolume.VolumeType != volumeType {
++ return fmt.Errorf("expected volume type %q, got %q", volumeType, *awsVolume.VolumeType)
++ }
++ if encrypted && awsVolume.Encrypted == nil {
++ return fmt.Errorf("expected encrypted volume, got no encryption")
++ }
++ if encrypted && !*awsVolume.Encrypted {
++ return fmt.Errorf("expected encrypted volume, got %v", *awsVolume.Encrypted)
++ }
++ return nil
++}
++
++func checkGCEPD(volume *v1.PersistentVolume, volumeType string) error {
++ cloud, err := gce.GetGCECloud()
++ if err != nil {
++ return err
++ }
++ diskName := volume.Spec.GCEPersistentDisk.PDName
++ disk, err := cloud.GetDiskByNameUnknownZone(diskName)
++ if err != nil {
++ return err
++ }
++
++ if !strings.HasSuffix(disk.Type, volumeType) {
++ return fmt.Errorf("unexpected disk type %q, expected suffix %q", disk.Type, volumeType)
++ }
++ return nil
++}
++
++var _ = utils.SIGDescribe("Dynamic Provisioning with cloud providers", func() {
++ f := framework.NewDefaultFramework("volume-provisioning")
++
++ // filled in BeforeEach
++ var c clientset.Interface
++ var ns string
++
++ ginkgo.BeforeEach(func() {
++ c = f.ClientSet
++ ns = f.Namespace.Name
++ })
++
++ ginkgo.Describe("DynamicProvisioner [Slow]", func() {
++ ginkgo.It("should provision storage with different parameters", func() {
++
++ // This test checks that dynamic provisioning can provision a volume
++ // that can be used to persist data among pods.
++ tests := []testsuites.StorageClassTest{
++ // GCE/GKE
++ {
++ Name: "SSD PD on GCE/GKE",
++ CloudProviders: []string{"gce", "gke"},
++ Provisioner: "kubernetes.io/gce-pd",
++ Parameters: map[string]string{
++ "type": "pd-ssd",
++ "zone": getRandomClusterZone(c),
++ },
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "2Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkGCEPD(volume, "pd-ssd")
++ framework.ExpectNoError(err, "checkGCEPD pd-ssd")
++ },
++ },
++ {
++ Name: "HDD PD on GCE/GKE",
++ CloudProviders: []string{"gce", "gke"},
++ Provisioner: "kubernetes.io/gce-pd",
++ Parameters: map[string]string{
++ "type": "pd-standard",
++ },
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "2Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkGCEPD(volume, "pd-standard")
++ framework.ExpectNoError(err, "checkGCEPD pd-standard")
++ },
++ },
++ // AWS
++ {
++ Name: "gp2 EBS on AWS",
++ CloudProviders: []string{"aws"},
++ Provisioner: "kubernetes.io/aws-ebs",
++ Parameters: map[string]string{
++ "type": "gp2",
++ "zone": getRandomClusterZone(c),
++ },
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "2Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkAWSEBS(volume, "gp2", false)
++ framework.ExpectNoError(err, "checkAWSEBS gp2")
++ },
++ },
++ {
++ Name: "io1 EBS on AWS",
++ CloudProviders: []string{"aws"},
++ Provisioner: "kubernetes.io/aws-ebs",
++ Parameters: map[string]string{
++ "type": "io1",
++ "iopsPerGB": "50",
++ },
++ ClaimSize: "3.5Gi",
++ ExpectedSize: "4Gi", // 4 GiB is minimum for io1
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkAWSEBS(volume, "io1", false)
++ framework.ExpectNoError(err, "checkAWSEBS io1")
++ },
++ },
++ {
++ Name: "sc1 EBS on AWS",
++ CloudProviders: []string{"aws"},
++ Provisioner: "kubernetes.io/aws-ebs",
++ Parameters: map[string]string{
++ "type": "sc1",
++ },
++ ClaimSize: "500Gi", // minimum for sc1
++ ExpectedSize: "500Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkAWSEBS(volume, "sc1", false)
++ framework.ExpectNoError(err, "checkAWSEBS sc1")
++ },
++ },
++ {
++ Name: "st1 EBS on AWS",
++ CloudProviders: []string{"aws"},
++ Provisioner: "kubernetes.io/aws-ebs",
++ Parameters: map[string]string{
++ "type": "st1",
++ },
++ ClaimSize: "500Gi", // minimum for st1
++ ExpectedSize: "500Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkAWSEBS(volume, "st1", false)
++ framework.ExpectNoError(err, "checkAWSEBS st1")
++ },
++ },
++ {
++ Name: "encrypted EBS on AWS",
++ CloudProviders: []string{"aws"},
++ Provisioner: "kubernetes.io/aws-ebs",
++ Parameters: map[string]string{
++ "encrypted": "true",
++ },
++ ClaimSize: "1Gi",
++ ExpectedSize: "1Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkAWSEBS(volume, "gp2", true)
++ framework.ExpectNoError(err, "checkAWSEBS gp2 encrypted")
++ },
++ },
++ // OpenStack generic tests (works on all OpenStack deployments)
++ {
++ Name: "generic Cinder volume on OpenStack",
++ CloudProviders: []string{"openstack"},
++ Provisioner: "kubernetes.io/cinder",
++ Parameters: map[string]string{},
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "2Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ },
++ },
++ {
++ Name: "Cinder volume with empty volume type and zone on OpenStack",
++ CloudProviders: []string{"openstack"},
++ Provisioner: "kubernetes.io/cinder",
++ Parameters: map[string]string{
++ "type": "",
++ "availability": "",
++ },
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "2Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ },
++ },
++ // vSphere generic test
++ {
++ Name: "generic vSphere volume",
++ CloudProviders: []string{"vsphere"},
++ Provisioner: "kubernetes.io/vsphere-volume",
++ Parameters: map[string]string{},
++ ClaimSize: "1.5Gi",
++ ExpectedSize: "1.5Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ },
++ },
++ // Azure
++ {
++ Name: "Azure disk volume with empty sku and location",
++ CloudProviders: []string{"azure"},
++ Provisioner: "kubernetes.io/azure-disk",
++ Parameters: map[string]string{},
++ ClaimSize: "1Gi",
++ ExpectedSize: "1Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ },
++ },
++ }
++
++ var betaTest *testsuites.StorageClassTest
++ for i, t := range tests {
++ // Beware of clojure, use local variables instead of those from
++ // outer scope
++ test := t
++
++ if !framework.ProviderIs(test.CloudProviders...) {
++ framework.Logf("Skipping %q: cloud providers is not %v", test.Name, test.CloudProviders)
++ continue
++ }
++
++ // Remember the last supported test for subsequent test of beta API
++ betaTest = &test
++
++ ginkgo.By("Testing " + test.Name)
++ suffix := fmt.Sprintf("%d", i)
++ test.Client = c
++ test.Class = newStorageClass(test, ns, suffix)
++ test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
++ ClaimSize: test.ClaimSize,
++ StorageClassName: &test.Class.Name,
++ VolumeMode: &test.VolumeMode,
++ }, ns)
++ test.TestDynamicProvisioning()
++ }
++
++ // Run the last test with storage.k8s.io/v1beta1 on pvc
++ if betaTest != nil {
++ ginkgo.By("Testing " + betaTest.Name + " with beta volume provisioning")
++ class := newBetaStorageClass(*betaTest, "beta")
++ // we need to create the class manually, testDynamicProvisioning does not accept beta class
++ class, err := c.StorageV1beta1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
++ framework.ExpectNoError(err)
++ defer deleteStorageClass(c, class.Name)
++
++ betaTest.Client = c
++ betaTest.Class = nil
++ betaTest.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
++ ClaimSize: betaTest.ClaimSize,
++ StorageClassName: &class.Name,
++ VolumeMode: &betaTest.VolumeMode,
++ }, ns)
++ betaTest.Claim.Spec.StorageClassName = &(class.Name)
++ (*betaTest).TestDynamicProvisioning()
++ }
++ })
++
++ ginkgo.It("should provision storage with non-default reclaim policy Retain", func() {
++ e2eskipper.SkipUnlessProviderIs("gce", "gke")
++
++ test := testsuites.StorageClassTest{
++ Client: c,
++ Name: "HDD PD on GCE/GKE",
++ CloudProviders: []string{"gce", "gke"},
++ Provisioner: "kubernetes.io/gce-pd",
++ Parameters: map[string]string{
++ "type": "pd-standard",
++ },
++ ClaimSize: "1Gi",
++ ExpectedSize: "1Gi",
++ PvCheck: func(claim *v1.PersistentVolumeClaim) {
++ volume := testsuites.PVWriteReadSingleNodeCheck(c, claim, e2epod.NodeSelection{})
++ gomega.Expect(volume).NotTo(gomega.BeNil(), "get bound PV")
++
++ err := checkGCEPD(volume, "pd-standard")
++ framework.ExpectNoError(err, "checkGCEPD")
++ },
++ }
++ test.Class = newStorageClass(test, ns, "reclaimpolicy")
++ retain := v1.PersistentVolumeReclaimRetain
++ test.Class.ReclaimPolicy = &retain
++ test.Claim = e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
++ ClaimSize: test.ClaimSize,
++ StorageClassName: &test.Class.Name,
++ VolumeMode: &test.VolumeMode,
++ }, ns)
++ pv := test.TestDynamicProvisioning()
++
++ ginkgo.By(fmt.Sprintf("waiting for the provisioned PV %q to enter phase %s", pv.Name, v1.VolumeReleased))
++ framework.ExpectNoError(e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 1*time.Second, 30*time.Second))
++
++ ginkgo.By(fmt.Sprintf("deleting the storage asset backing the PV %q", pv.Name))
++ framework.ExpectNoError(e2epv.DeletePDWithRetry(pv.Spec.GCEPersistentDisk.PDName))
++
++ ginkgo.By(fmt.Sprintf("deleting the PV %q", pv.Name))
++ framework.ExpectNoError(e2epv.DeletePersistentVolume(c, pv.Name), "Failed to delete PV ", pv.Name)
++ framework.ExpectNoError(e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 1*time.Second, 30*time.Second))
++ })
++
++ ginkgo.It("should not provision a volume in an unmanaged GCE zone.", func() {
++ e2eskipper.SkipUnlessProviderIs("gce", "gke")
++ var suffix string = "unmananged"
++
++ ginkgo.By("Discovering an unmanaged zone")
++ allZones := sets.NewString() // all zones in the project
++
++ gceCloud, err := gce.GetGCECloud()
++ framework.ExpectNoError(err)
++
++ // Get all k8s managed zones (same as zones with nodes in them for test)
++ managedZones, err := gceCloud.GetAllZonesFromCloudProvider()
++ framework.ExpectNoError(err)
++
++ // Get a list of all zones in the project
++ zones, err := gceCloud.ComputeServices().GA.Zones.List(framework.TestContext.CloudConfig.ProjectID).Do()
++ framework.ExpectNoError(err)
++ for _, z := range zones.Items {
++ allZones.Insert(z.Name)
++ }
++
++ // Get the subset of zones not managed by k8s
++ var unmanagedZone string
++ var popped bool
++ unmanagedZones := allZones.Difference(managedZones)
++ // And select one of them at random.
++ if unmanagedZone, popped = unmanagedZones.PopAny(); !popped {
++ e2eskipper.Skipf("No unmanaged zones found.")
++ }
++
++ ginkgo.By("Creating a StorageClass for the unmanaged zone")
++ test := testsuites.StorageClassTest{
++ Name: "unmanaged_zone",
++ Provisioner: "kubernetes.io/gce-pd",
++ Parameters: map[string]string{"zone": unmanagedZone},
++ ClaimSize: "1Gi",
++ }
++ sc := newStorageClass(test, ns, suffix)
++ sc, err = c.StorageV1().StorageClasses().Create(context.TODO(), sc, metav1.CreateOptions{})
++ framework.ExpectNoError(err)
++ defer deleteStorageClass(c, sc.Name)
++
++ ginkgo.By("Creating a claim and expecting it to timeout")
++ pvc := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
++ ClaimSize: test.ClaimSize,
++ StorageClassName: &sc.Name,
++ VolumeMode: &test.VolumeMode,
++ }, ns)
++ pvc, err = c.CoreV1().PersistentVolumeClaims(ns).Create(context.TODO(), pvc, metav1.CreateOptions{})
++ framework.ExpectNoError(err)
++ defer func() {
++ framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, pvc.Name, ns), "Failed to delete PVC ", pvc.Name)
++ }()
++
++ // The claim should timeout phase:Pending
++ err = e2epv.WaitForPersistentVolumeClaimPhase(v1.ClaimBound, c, ns, pvc.Name, 2*time.Second, framework.ClaimProvisionShortTimeout)
++ framework.ExpectError(err)
++ framework.Logf(err.Error())
++ })
++
++ ginkgo.It("should test that deleting a claim before the volume is provisioned deletes the volume.", func() {
++ // This case tests for the regressions of a bug fixed by PR #21268
++ // REGRESSION: Deleting the PVC before the PV is provisioned can result in the PV
++ // not being deleted.
++ // NOTE: Polls until no PVs are detected, times out at 5 minutes.
++
++ e2eskipper.SkipUnlessProviderIs("openstack", "gce", "aws", "gke", "vsphere", "azure")
++
++ const raceAttempts int = 100
++ var residualPVs []*v1.PersistentVolume
++ ginkgo.By(fmt.Sprintf("Creating and deleting PersistentVolumeClaims %d times", raceAttempts))
++ test := testsuites.StorageClassTest{
++ Name: "deletion race",
++ Provisioner: "", // Use a native one based on current cloud provider
++ ClaimSize: "1Gi",
++ }
++
++ class := newStorageClass(test, ns, "race")
++ class, err := c.StorageV1().StorageClasses().Create(context.TODO(), class, metav1.CreateOptions{})
++ framework.ExpectNoError(err)
++ defer deleteStorageClass(c, class.Name)
++
++ // To increase chance of detection, attempt multiple iterations
++ for i := 0; i < raceAttempts; i++ {
++ prefix := fmt.Sprintf("race-%d", i)
++ claim := e2epv.MakePersistentVolumeClaim(e2epv.PersistentVolumeClaimConfig{
++ NamePrefix: prefix,
++ ClaimSize: test.ClaimSize,
++ StorageClassName: &class.Name,
++ VolumeMode: &test.VolumeMode,
++ }, ns)
++ tmpClaim, err := e2epv.CreatePVC(c, ns, claim)
++ framework.ExpectNoError(err)
++ framework.ExpectNoError(e2epv.DeletePersistentVolumeClaim(c, tmpClaim.Name, ns))
++ }
++
++ ginkgo.By(fmt.Sprintf("Checking for residual PersistentVolumes associated with StorageClass %s", class.Name))
++ residualPVs, err = waitForProvisionedVolumesDeleted(c, class.Name)
++ // Cleanup the test resources before breaking
++ defer deleteProvisionedVolumesAndDisks(c, residualPVs)
++ framework.ExpectNoError(err, "PersistentVolumes were not deleted as expected. %d remain", len(residualPVs))
++
++ framework.Logf("0 PersistentVolumes remain.")
++ })
++
++ ginkgo.It("deletion should be idempotent", func() {
++ // This test ensures that deletion of a volume is idempotent.
++ // It creates a PV with Retain policy, deletes underlying AWS / GCE
++ // volume and changes the reclaim policy to Delete.
++ // PV controller should delete the PV even though the underlying volume
++ // is already deleted.
++ e2eskipper.SkipUnlessProviderIs("gce", "gke", "aws")
++ ginkgo.By("creating PD")
++ diskName, err := e2epv.CreatePDWithRetry()
++ framework.ExpectNoError(err)
++
++ ginkgo.By("creating PV")
++ pv := e2epv.MakePersistentVolume(e2epv.PersistentVolumeConfig{
++ NamePrefix: "volume-idempotent-delete-",
++ // Use Retain to keep the PV, the test will change it to Delete
++ // when the time comes.
++ ReclaimPolicy: v1.PersistentVolumeReclaimRetain,
++ AccessModes: []v1.PersistentVolumeAccessMode{
++ v1.ReadWriteOnce,
++ },
++ Capacity: "1Gi",
++ // PV is bound to non-existing PVC, so it's reclaim policy is
++ // executed immediately
++ Prebind: &v1.PersistentVolumeClaim{
++ ObjectMeta: metav1.ObjectMeta{
++ Name: "dummy-claim-name",
++ Namespace: ns,
++ UID: types.UID("01234567890"),
++ },
++ },
++ })
++ switch framework.TestContext.Provider {
++ case "aws":
++ pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
++ AWSElasticBlockStore: &v1.AWSElasticBlockStoreVolumeSource{
++ VolumeID: diskName,
++ },
++ }
++ case "gce", "gke":
++ pv.Spec.PersistentVolumeSource = v1.PersistentVolumeSource{
++ GCEPersistentDisk: &v1.GCEPersistentDiskVolumeSource{
++ PDName: diskName,
++ },
++ }
++ }
++ pv, err = c.CoreV1().PersistentVolumes().Create(context.TODO(), pv, metav1.CreateOptions{})
++ framework.ExpectNoError(err)
++
++ ginkgo.By("waiting for the PV to get Released")
++ err = e2epv.WaitForPersistentVolumePhase(v1.VolumeReleased, c, pv.Name, 2*time.Second, e2epv.PVReclaimingTimeout)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("deleting the PD")
++ err = e2epv.DeletePVSource(&pv.Spec.PersistentVolumeSource)
++ framework.ExpectNoError(err)
++
++ ginkgo.By("changing the PV reclaim policy")
++ pv, err = c.CoreV1().PersistentVolumes().Get(context.TODO(), pv.Name, metav1.GetOptions{})
++ framework.ExpectNoError(err)
++ pv.Spec.PersistentVolumeReclaimPolicy = v1.PersistentVolumeReclaimDelete
++ pv, err = c.CoreV1().PersistentVolumes().Update(context.TODO(), pv, metav1.UpdateOptions{})
++ framework.ExpectNoError(err)
++
++ ginkgo.By("waiting for the PV to get deleted")
++ err = e2epv.WaitForPersistentVolumeDeleted(c, pv.Name, 5*time.Second, e2epv.PVDeletingTimeout)
++ framework.ExpectNoError(err)
++ })
++ })
++})
+diff --git a/test/e2e/upgrades/nvidia-gpu.go b/test/e2e/upgrades/nvidia-gpu.go
+index cf3b8c0cda3..30515197ef7 100644
+--- a/test/e2e/upgrades/nvidia-gpu.go
++++ b/test/e2e/upgrades/nvidia-gpu.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2018 The Kubernetes Authors.
+
+diff --git a/test/e2e/instrumentation/logging/imports.go b/test/e2e/instrumentation/logging/imports.go
+index 5dd66717db1..fc15c04bfef 100644
+--- a/test/e2e/instrumentation/logging/imports.go
++++ b/test/e2e/instrumentation/logging/imports.go
+@@ -1,3 +1,5 @@
++// +build !providerless
++
+ /*
+ Copyright 2017 The Kubernetes Authors.
+
+--
+2.25.1
+
diff --git a/third_party/go/patches/k8s-kubernetes.patch b/third_party/go/patches/k8s-kubernetes.patch
index 49250e5..8b766a6 100644
--- a/third_party/go/patches/k8s-kubernetes.patch
+++ b/third_party/go/patches/k8s-kubernetes.patch
@@ -348,3 +348,58 @@
_ "github.com/google/cadvisor/utils/cloudinfo/gce"
"github.com/google/cadvisor/cache/memory"
+
+--- io_k8s_kubernetes.orig/test/e2e/BUILD 2020-07-22 10:51:20.277358305 +0200
++++ io_k8s_kubernetes/test/e2e/BUILD 2020-07-22 10:56:43.451577495 +0200
+@@ -5,7 +5,7 @@
+ go_test = "go_test_conditional_pure",
+ )
+ load("@io_bazel_rules_go//go:def.bzl", "go_library")
+-load("//staging/src/k8s.io/component-base/version:def.bzl", "version_x_defs")
++load("@//third_party/go:kubernetes_version_def.bzl", "version_x_defs")
+
+ go_test(
+ name = "go_default_test",
+
+--- io_k8s_kubernetes.orig/test/e2e/generated/BUILD
++++ io_k8s_kubernetes/test/e2e/generated/BUILD
+@@ -4,23 +4,24 @@ load(
+ "@io_bazel_rules_go//go:def.bzl",
+ "go_library",
+ )
+-load("//build:bindata.bzl", "go_bindata")
++load("@nexantic//build/bindata:bindata.bzl", "bindata")
+
+ go_library(
+ name = "go_default_library",
+ srcs = [
+- "bindata.go",
+ "gobindata_util.go",
+ "main.go",
+ ],
++ embed = [
++ ":bindata",
++ ],
+ importpath = "k8s.io/kubernetes/test/e2e/generated",
+ deps = [
+ "@io_k8s_klog_v2//:go_default_library",
+ ],
+ )
+
+-# IMPORTANT: if you make any changes here, you must also update hack/generate-bindata.sh.
+-go_bindata(
++bindata(
+ name = "bindata",
+ srcs = [
+ "//test/conformance/testdata:all-srcs",
+@@ -29,9 +30,7 @@ go_bindata(
+ "//test/fixtures:all-srcs",
+ "//test/images:all-srcs",
+ ],
+- outs = ["bindata.go"],
+- compress = True,
+- include_metadata = False,
++ package = "generated",
+ )
+
+ filegroup(
\ No newline at end of file
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index a65eb2c..39fbda0 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -465,6 +465,12 @@
sum = "h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo=",
)
go_repository(
+ name = "com_github_elazarl_goproxy",
+ importpath = "github.com/elazarl/goproxy",
+ version = "v0.0.0-20180725130230-947c36da3153",
+ sum = "h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc=",
+ )
+ go_repository(
name = "com_github_emicklei_go_restful",
importpath = "github.com/emicklei/go-restful",
version = "v2.9.5+incompatible",
@@ -879,6 +885,12 @@
sum = "h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=",
)
go_repository(
+ name = "com_github_hpcloud_tail",
+ importpath = "github.com/hpcloud/tail",
+ version = "v1.0.0",
+ sum = "h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=",
+ )
+ go_repository(
name = "com_github_imdario_mergo",
importpath = "github.com/imdario/mergo",
version = "v0.3.7",
@@ -1156,6 +1168,18 @@
sum = "h1:58+kh9C6jJVXYjt8IE48G2eWl6BjwU5Gj0gqY84fy78=",
)
go_repository(
+ name = "com_github_onsi_ginkgo",
+ importpath = "github.com/onsi/ginkgo",
+ version = "v1.11.0",
+ sum = "h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw=",
+ )
+ go_repository(
+ name = "com_github_onsi_gomega",
+ importpath = "github.com/onsi/gomega",
+ version = "v1.7.0",
+ sum = "h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME=",
+ )
+ go_repository(
name = "com_github_opencontainers_go_digest",
importpath = "github.com/opencontainers/go-digest",
version = "v1.0.0",
@@ -1633,6 +1657,12 @@
sum = "h1:UCvDKl1L/fmBygl2Y7hubXCnY7t4Yj46ZrBFNUipFbM=",
)
go_repository(
+ name = "in_gopkg_fsnotify_v1",
+ importpath = "gopkg.in/fsnotify.v1",
+ version = "v1.4.7",
+ sum = "h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4=",
+ )
+ go_repository(
name = "in_gopkg_gorp_v1",
importpath = "gopkg.in/gorp.v1",
version = "v1.7.2",
@@ -1663,6 +1693,12 @@
sum = "h1:orlkJ3myw8CN1nVQHBFfloD+L3egixIa4FvUP6RosSA=",
)
go_repository(
+ name = "in_gopkg_tomb_v1",
+ importpath = "gopkg.in/tomb.v1",
+ version = "v1.0.0-20141024135613-dd632973f1e7",
+ sum = "h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=",
+ )
+ go_repository(
name = "in_gopkg_yaml_v2",
importpath = "gopkg.in/yaml.v2",
version = "v2.2.8",
@@ -1707,9 +1743,8 @@
importpath = "k8s.io/client-go",
version = "v0.19.0-rc.0",
sum = "h1:6WW8MElhoLeYcLiN4ky1159XG5E39KYdmLCrV/6lNiE=",
- patches = [
+ pre_patches = [
"//third_party/go/patches:k8s-client-go.patch",
- "//third_party/go/patches:k8s-client-go-build.patch",
],
patch_args = ["-p1"],
)
@@ -1821,6 +1856,9 @@
"//third_party/go/patches:k8s-native-metrics.patch",
"//third_party/go/patches:k8s-use-native.patch",
],
+ pre_patches = [
+ "//third_party/go/patches:k8s-e2e-tests-providerless.patch",
+ ],
patch_args = ["-p1"],
)
go_repository(
diff --git a/third_party/go/shelf.pb.text b/third_party/go/shelf.pb.text
index 3e0eded..ead136d 100644
--- a/third_party/go/shelf.pb.text
+++ b/third_party/go/shelf.pb.text
@@ -741,6 +741,13 @@
semver: "v1.0.0"
>
entry: <
+ import_path: "github.com/elazarl/goproxy"
+ version: "v0.0.0-20180725130230-947c36da3153"
+ bazel_name: "com_github_elazarl_goproxy"
+ sum: "h1:yUdfgN0XgIJw7foRItutHYUIhlcKzcSf5vDpdhQAKTc="
+ semver: "v0.0.0-20180725130230-947c36da3153"
+>
+entry: <
import_path: "github.com/emicklei/go-restful"
version: "b993709ae1a4f6dd19cfa475232614441b11c9d5"
bazel_name: "com_github_emicklei_go_restful"
@@ -1329,6 +1336,13 @@
semver: "v0.8.5"
>
entry: <
+ import_path: "github.com/hpcloud/tail"
+ version: "v1.0.0"
+ bazel_name: "com_github_hpcloud_tail"
+ sum: "h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI="
+ semver: "v1.0.0"
+>
+entry: <
import_path: "github.com/imdario/mergo"
version: "7c29201646fa3de8506f701213473dd407f19646"
bazel_name: "com_github_imdario_mergo"
@@ -1735,6 +1749,20 @@
semver: "v0.0.0-20170122224234-a0225b3f23b5"
>
entry: <
+ import_path: "github.com/onsi/ginkgo"
+ version: "v1.11.0"
+ bazel_name: "com_github_onsi_ginkgo"
+ sum: "h1:JAKSXpt1YjtLA7YpPiqO9ss6sNXEsPfSGdwN0UHqzrw="
+ semver: "v1.11.0"
+>
+entry: <
+ import_path: "github.com/onsi/gomega"
+ version: "v1.7.0"
+ bazel_name: "com_github_onsi_gomega"
+ sum: "h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME="
+ semver: "v1.7.0"
+>
+entry: <
import_path: "github.com/op/go-logging"
version: "v0.0.0-20160315200505-970db520ece7"
bazel_name: "com_github_op_go_logging"
@@ -2519,6 +2547,13 @@
semver: "v1.2.0"
>
entry: <
+ import_path: "gopkg.in/fsnotify.v1"
+ version: "v1.4.7"
+ bazel_name: "in_gopkg_fsnotify_v1"
+ sum: "h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4="
+ semver: "v1.4.7"
+>
+entry: <
import_path: "gopkg.in/gorp.v1"
version: "v1.7.2"
bazel_name: "in_gopkg_gorp_v1"
@@ -2561,6 +2596,13 @@
semver: "v2.2.2"
>
entry: <
+ import_path: "gopkg.in/tomb.v1"
+ version: "v1.0.0-20141024135613-dd632973f1e7"
+ bazel_name: "in_gopkg_tomb_v1"
+ sum: "h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ="
+ semver: "v1.0.0-20141024135613-dd632973f1e7"
+>
+entry: <
import_path: "gopkg.in/yaml.v2"
version: "53403b58ad1b561927d19068c655246f2db79d48"
bazel_name: "in_gopkg_yaml_v2"