go/net/ssh: improve throughput by more than 100x
Use concurrent writes (this is safe for our usage) and use
ReadFromWithConcurrency to extract maximum concurrency from the process.
Increase the window to 1024 pending reqeuests for a bandwidth-delay
product of 32MiB, sufficient for ~1Gbps at 300ms RTT.
In practice this improves upload performance from ~13Mbit/s on 15ms RTT
up to over 1.5Gbit/s, now being limited by the weak network
implementation on the small GCP instance I tested this on. Maximum
performance with a bigger instance and <1ms RTT was over 5Gbps, becoming
disk-limited.
Change-Id: I2f08d2c815fd15794f497ed9dc7a7f8a2e351992
Reviewed-on: https://review.monogon.dev/c/monogon/+/3435
Tested-by: Jenkins CI
Reviewed-by: Tim Windelschmidt <tim@monogon.tech>
diff --git a/go/net/ssh/ssh_client.go b/go/net/ssh/ssh_client.go
index 4e3cab6..1c52484 100644
--- a/go/net/ssh/ssh_client.go
+++ b/go/net/ssh/ssh_client.go
@@ -102,7 +102,7 @@
}
func (p *directConn) Upload(ctx context.Context, targetPath string, src io.Reader) error {
- sc, err := sftp.NewClient(p.cl)
+ sc, err := sftp.NewClient(p.cl, sftp.UseConcurrentWrites(true), sftp.MaxConcurrentRequestsPerFile(1024))
if err != nil {
return fmt.Errorf("while building sftp client: %w", err)
}
@@ -116,7 +116,7 @@
doneC := make(chan error, 1)
go func() {
- _, err := io.Copy(df, src)
+ _, err := df.ReadFromWithConcurrency(src, 0)
df.Close()
doneC <- err
}()