Increase E2E gRPC connection reliability
The E2E test suite sometimes fails because of weird interaction between gRPC and SLIRP
where gRPC has an open connection with SLIRP, but SLIRP hanged while connecting to the actual
service. In that case gRPC hangs waiting for the server handshake which will never come as the
upstream connection is never properly established. This forcibly terminates each gRPC call
after 5s to make sure that multiple calls are attempted and it doesn't hang.
Test Plan: Hard to test, I observe less of these errors
X-Origin-Diff: phab/D583
GitOrigin-RevId: cc1336aab7f4d13412bdb2e93be027d7e3eb57fc
diff --git a/core/tests/e2e/condition_helpers.go b/core/tests/e2e/condition_helpers.go
index f7d5c8e..df5d38c 100644
--- a/core/tests/e2e/condition_helpers.go
+++ b/core/tests/e2e/condition_helpers.go
@@ -27,7 +27,9 @@
func waitForCondition(ctx context.Context, client apipb.NodeDebugServiceClient, condition string) error {
var lastErr = errors.New("No RPC for checking condition completed")
for {
- res, err := client.GetCondition(ctx, &apipb.GetConditionRequest{Name: condition})
+ reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ res, err := client.GetCondition(reqCtx, &apipb.GetConditionRequest{Name: condition})
if err != nil {
if err == ctx.Err() {
return err
diff --git a/core/tests/e2e/kubernetes_helpers.go b/core/tests/e2e/kubernetes_helpers.go
index 264793a..e633538 100644
--- a/core/tests/e2e/kubernetes_helpers.go
+++ b/core/tests/e2e/kubernetes_helpers.go
@@ -38,7 +38,9 @@
func getKubeClientSet(ctx context.Context, client apipb.NodeDebugServiceClient, port uint16) (kubernetes.Interface, error) {
var lastErr = errors.New("context canceled before any operation completed")
for {
- res, err := client.GetDebugKubeconfig(context.Background(), &apipb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
+ reqCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ res, err := client.GetDebugKubeconfig(reqCtx, &apipb.GetDebugKubeconfigRequest{Id: "debug-user", Groups: []string{"system:masters"}})
if err == nil {
rawClientConfig, err := clientcmd.NewClientConfigFromBytes([]byte(res.DebugKubeconfig))
if err != nil {