cloud/shepherd/equinix/wrapngo: init

This adds a wrapper extending packngo for use with the upcoming
Shepherd implementation.

Supersedes: https://review.monogon.dev/c/monogon/+/989
Change-Id: I55d1a609a8b5241704c5fe4ce8c2294122cfa0c8
Reviewed-on: https://review.monogon.dev/c/monogon/+/1128
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
Reviewed-by: Mateusz Zalega <mateusz@monogon.tech>
diff --git a/cloud/shepherd/equinix/wrapngo/BUILD.bazel b/cloud/shepherd/equinix/wrapngo/BUILD.bazel
new file mode 100644
index 0000000..3694fe1
--- /dev/null
+++ b/cloud/shepherd/equinix/wrapngo/BUILD.bazel
@@ -0,0 +1,29 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
+
+go_library(
+    name = "wrapngo",
+    srcs = [
+        "duct_tape.go",
+        "wrapn.go",
+    ],
+    importpath = "source.monogon.dev/cloud/shepherd/equinix/wrapngo",
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_cenkalti_backoff_v4//:backoff",
+        "@com_github_google_uuid//:uuid",
+        "@com_github_packethost_packngo//:packngo",
+        "@io_k8s_klog_v2//:klog",
+    ],
+)
+
+go_test(
+    name = "wrapngo_test",
+    timeout = "eternal",
+    srcs = ["wrapngo_test.go"],
+    args = ["-test.v"],
+    embed = [":wrapngo"],
+    deps = [
+        "@com_github_packethost_packngo//:packngo",
+        "@org_golang_x_crypto//ssh",
+    ],
+)
diff --git a/cloud/shepherd/equinix/wrapngo/duct_tape.go b/cloud/shepherd/equinix/wrapngo/duct_tape.go
new file mode 100644
index 0000000..c9e156b
--- /dev/null
+++ b/cloud/shepherd/equinix/wrapngo/duct_tape.go
@@ -0,0 +1,119 @@
+package wrapngo
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/packethost/packngo"
+	"k8s.io/klog/v2"
+)
+
+// wrap a given fn in some reliability-increasing duct tape: context support and
+// exponential backoff retries for intermittent connectivity issues. This allows
+// us to use packngo code instead of writing our own API stub for Equinix Metal.
+//
+// The given fn will be retried until it returns a 'permanent' Equinix error (see
+// isPermanentEquinixError) or the given context expires. Additionally, fn will
+// be called with a brand new packngo client tied to the context of the wrap
+// call. Finally, the given client will also have some logging middleware
+// attached to it which can be activated by setting verbosity 5 (or greater) on
+// this file.
+//
+// The wrapped fn can be either just a plain packngo method or some complicated
+// idempotent logic, as long as it cooperates with the above contract.
+func wrap[U any](ctx context.Context, cl *client, fn func(*packngo.Client) (U, error)) (U, error) {
+	var zero U
+	select {
+	case cl.serializer <- struct{}{}:
+	case <-ctx.Done():
+		return zero, ctx.Err()
+	}
+	defer func() {
+		<-cl.serializer
+	}()
+
+	bc := backoff.WithContext(cl.o.BackOff(), ctx)
+	pngo, err := cl.clientForContext(ctx)
+	if err != nil {
+		// Generally this shouldn't happen other than with programming errors, so we
+		// don't back this off.
+		return zero, fmt.Errorf("could not crate equinix client: %w", err)
+	}
+
+	var res U
+	err = backoff.Retry(func() error {
+		res, err = fn(pngo)
+		if isPermanentEquinixError(err) {
+			return backoff.Permanent(err)
+		}
+		return err
+	}, bc)
+	if err != nil {
+		return zero, err
+	}
+	return res, nil
+}
+
+type injectContextRoundTripper struct {
+	ctx      context.Context
+	original http.RoundTripper
+}
+
+func (r *injectContextRoundTripper) RoundTrip(req *http.Request) (*http.Response, error) {
+	klog.V(5).Infof("Request -> %v", req.URL.String())
+	res, err := r.original.RoundTrip(req.WithContext(r.ctx))
+	klog.V(5).Infof("Response <- %v", res.Status)
+	return res, err
+}
+
+func (c *client) clientForContext(ctx context.Context) (*packngo.Client, error) {
+	httpcl := &http.Client{
+		Transport: &injectContextRoundTripper{
+			ctx:      ctx,
+			original: http.DefaultTransport,
+		},
+	}
+	return packngo.NewClient(packngo.WithAuth(c.username, c.token), packngo.WithHTTPClient(httpcl))
+}
+
+// httpStatusCode extracts the status code from error values returned by
+// packngo methods.
+func httpStatusCode(err error) int {
+	var er *packngo.ErrorResponse
+	if err != nil && errors.As(err, &er) {
+		return er.Response.StatusCode
+	}
+	return -1
+}
+
+// IsNotFound returns true if the given error is an Equinix packngo/wrapngo 'not
+// found' error.
+func IsNotFound(err error) bool {
+	return httpStatusCode(err) == http.StatusNotFound
+}
+
+func isPermanentEquinixError(err error) bool {
+	// Invalid argument/state errors from wrapping.
+	if errors.Is(err, ErrRaceLost) {
+		return true
+	}
+	if errors.Is(err, ErrNoReservationProvided) {
+		return true
+	}
+	// Real errors returned from equinix.
+	st := httpStatusCode(err)
+	switch st {
+	case http.StatusUnauthorized:
+		return true
+	case http.StatusForbidden:
+		return true
+	case http.StatusNotFound:
+		return true
+	case http.StatusUnprocessableEntity:
+		return true
+	}
+	return false
+}
diff --git a/cloud/shepherd/equinix/wrapngo/wrapn.go b/cloud/shepherd/equinix/wrapngo/wrapn.go
new file mode 100644
index 0000000..16478d5
--- /dev/null
+++ b/cloud/shepherd/equinix/wrapngo/wrapn.go
@@ -0,0 +1,304 @@
+// Package wrapngo wraps packngo methods providing the following usability
+// enhancements:
+// - API call rate limiting
+// - resource-aware call retries
+// - use of a configurable back-off algorithm implementation
+// - context awareness
+//
+// The implementation is provided with the following caveats:
+//
+// There can be only one call in flight. Concurrent calls to API-related
+// methods of the same client will block. Calls returning packngo structs will
+// return nil data when a non-nil error value is returned. An
+// os.ErrDeadlineExceeded will be returned after the underlying API calls time
+// out beyond the chosen back-off algorithm implementation's maximum allowed
+// retry interval. Other errors, excluding context.Canceled and
+// context.DeadlineExceeded, indicate either an error originating at Equinix'
+// API endpoint (which may still stem from invalid call inputs), or a network
+// error.
+//
+// Packngo wrappers included below may return timeout errors even after the
+// wrapped calls succeed in the event server reply could not have been
+// received.
+//
+// This implies that effects of mutating calls can't always be verified
+// atomically, requiring explicit synchronization between API users, regardless
+// of the retry/recovery logic used.
+//
+// Having that in mind, some call wrappers exposed by this package will attempt
+// to recover from this kind of situations by requesting information on any
+// resources created, and retrying the call if needed. This approach assumes
+// any concurrent mutating API users will be synchronized, as it should be in
+// any case.
+//
+// Another way of handling this problem would be to leave it up to the user to
+// retry calls if needed, though this would leak Equinix Metal API, and
+// complicate implementations depending on this package. Due to that, the prior
+// approach was chosen.
+package wrapngo
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/cenkalti/backoff/v4"
+	"github.com/google/uuid"
+	"github.com/packethost/packngo"
+)
+
+// Opts conveys configurable Client parameters.
+type Opts struct {
+	// User and APIKey are the credentials used to authenticate with
+	// Metal API.
+
+	User   string
+	APIKey string
+
+	// Optional parameters:
+
+	// BackOff controls the client's behavior in the event of API calls failing
+	// due to IO timeouts by adjusting the lower bound on time taken between
+	// subsequent calls.
+	BackOff func() backoff.BackOff
+
+	// APIRate is the minimum time taken between subsequent API calls.
+	APIRate time.Duration
+}
+
+func (o *Opts) RegisterFlags() {
+	flag.StringVar(&o.User, "equinix_api_username", "", "Username for Equinix API")
+	flag.StringVar(&o.APIKey, "equinix_api_key", "", "Key/token/password for Equinix API")
+}
+
+// Client is a limited interface of methods that the Shepherd uses on Equinix. It
+// is provided to allow for dependency injection of a fake equinix API for tests.
+type Client interface {
+	// GetDevice wraps packngo's cl.Devices.Get.
+	GetDevice(ctx context.Context, pid, did string) (*packngo.Device, error)
+	// ListDevices wraps packngo's cl.Device.List.
+	ListDevices(ctx context.Context, pid string) ([]packngo.Device, error)
+	// CreateDevice attempts to create a new device according to the provided
+	// request. The request _must_ configure a HardwareReservationID. This call
+	// attempts to be as idempotent as possible, and will return ErrRaceLost if a
+	// retry was needed but in the meantime the requested hardware reservation from
+	// which this machine was requested got lost.
+	CreateDevice(ctx context.Context, request *packngo.DeviceCreateRequest) (*packngo.Device, error)
+
+	// ListReservations returns a complete list of hardware reservations associated
+	// with project pid. This is an expensive method that takes a while to execute,
+	// handle with care.
+	ListReservations(ctx context.Context, pid string) ([]packngo.HardwareReservation, error)
+
+	// ListSSHKeys wraps packngo's cl.Keys.List.
+	ListSSHKeys(ctx context.Context) ([]packngo.SSHKey, error)
+	// CreateSSHKey is idempotent - the key label can be used only once. Further
+	// calls referring to the same label and key will not yield errors. See the
+	// package comment for more info on this method's behavior and returned error
+	// values.
+	CreateSSHKey(ctx context.Context, req *packngo.SSHKeyCreateRequest) (*packngo.SSHKey, error)
+	// UpdateSSHKey is idempotent - values included in r can be applied only once,
+	// while subsequent updates using the same data don't produce errors. See the
+	// package comment for information on this method's behavior and returned error
+	// values.
+	UpdateSSHKey(ctx context.Context, kid string, req *packngo.SSHKeyUpdateRequest) (*packngo.SSHKey, error)
+
+	Close()
+}
+
+// client implements the Client interface.
+type client struct {
+	username string
+	token    string
+	o        *Opts
+	rlt      *time.Ticker
+
+	// serializer is a 1-semaphore channel (effectively a mutex) which is used to
+	// limit the number of concurrent calls to the Equinix API.
+	serializer chan (struct{})
+}
+
+// New creates a Client instance based on Opts. PACKNGO_DEBUG environment
+// variable can be set prior to the below call to enable verbose packngo
+// debug logs.
+func New(opts *Opts) Client {
+	return new(opts)
+}
+
+func new(opts *Opts) *client {
+	// Apply the defaults.
+	if opts.APIRate == 0 {
+		opts.APIRate = 2 * time.Second
+	}
+	if opts.BackOff == nil {
+		opts.BackOff = func() backoff.BackOff {
+			return backoff.NewExponentialBackOff()
+		}
+	}
+
+	return &client{
+		username: opts.User,
+		token:    opts.APIKey,
+		o:        opts,
+		rlt:      time.NewTicker(opts.APIRate),
+
+		serializer: make(chan struct{}, 1),
+	}
+}
+
+func (c *client) Close() {
+	c.rlt.Stop()
+}
+
+var (
+	ErrRaceLost              = errors.New("race lost with another API user")
+	ErrNoReservationProvided = errors.New("hardware reservation must be set")
+)
+
+func (e *client) CreateDevice(ctx context.Context, r *packngo.DeviceCreateRequest) (*packngo.Device, error) {
+	if r.HardwareReservationID == "" {
+		return nil, ErrNoReservationProvided
+	}
+	// Add a tag to the request to detect if someone snatches a hardware reservation
+	// from under us.
+	witnessTag := fmt.Sprintf("wrapngo-idempotency-%s", uuid.New().String())
+	r.Tags = append(r.Tags, witnessTag)
+
+	return wrap(ctx, e, func(cl *packngo.Client) (*packngo.Device, error) {
+		//Does the device already exist?
+		res, _, err := cl.HardwareReservations.Get(r.HardwareReservationID, nil)
+		if err != nil {
+			return nil, fmt.Errorf("couldn't check if device already exists: %w", err)
+		}
+		if res == nil {
+			return nil, fmt.Errorf("unexpected nil response")
+		}
+		if res.Device != nil {
+			// Check if we lost the race for this hardware reservation.
+			tags := make(map[string]bool)
+			for _, tag := range res.Device.Tags {
+				tags[tag] = true
+			}
+			if !tags[witnessTag] {
+				return nil, ErrRaceLost
+			}
+			return res.Device, nil
+		}
+
+		// No device yet. Try to create it.
+		dev, _, err := cl.Devices.Create(r)
+		if err == nil {
+			return dev, nil
+		}
+		// In case of a transient failure (eg. network issue), we retry the whole
+		// operation, which means we first check again if the device already exists. If
+		// it's a permanent error from the API, the backoff logic will fail immediately.
+		return nil, fmt.Errorf("couldn't create device: %w", err)
+	})
+}
+
+func (e *client) ListDevices(ctx context.Context, pid string) ([]packngo.Device, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) ([]packngo.Device, error) {
+		res, _, err := cl.Devices.List(pid, nil)
+		return res, err
+	})
+}
+
+func (e *client) GetDevice(ctx context.Context, pid, did string) (*packngo.Device, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) (*packngo.Device, error) {
+		d, _, err := cl.Devices.Get(did, nil)
+		return d, err
+	})
+}
+
+// Currently unexported, only used in tests.
+func (e *client) deleteDevice(ctx context.Context, did string) error {
+	_, err := wrap(ctx, e, func(cl *packngo.Client) (*struct{}, error) {
+		_, err := cl.Devices.Delete(did, false)
+		if httpStatusCode(err) == http.StatusNotFound {
+			// 404s may pop up as an after effect of running the back-off
+			// algorithm, and as such should not be propagated.
+			return nil, nil
+		}
+		return nil, err
+	})
+	return err
+}
+
+func (e *client) ListReservations(ctx context.Context, pid string) ([]packngo.HardwareReservation, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) ([]packngo.HardwareReservation, error) {
+		res, _, err := cl.HardwareReservations.List(pid, nil)
+		return res, err
+	})
+}
+
+func (e *client) CreateSSHKey(ctx context.Context, r *packngo.SSHKeyCreateRequest) (*packngo.SSHKey, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) (*packngo.SSHKey, error) {
+		// Does the key already exist?
+		ks, _, err := cl.SSHKeys.List()
+		if err != nil {
+			return nil, fmt.Errorf("SSHKeys.List: %w", err)
+		}
+		for _, k := range ks {
+			if k.Label == r.Label {
+				if k.Key != r.Key {
+					return nil, fmt.Errorf("key label already in use for a different key")
+				}
+				return &k, nil
+			}
+		}
+
+		// No key yet. Try to create it.
+		k, _, err := cl.SSHKeys.Create(r)
+		if err != nil {
+			return nil, fmt.Errorf("SSHKeys.Create: %w", err)
+		}
+		return k, nil
+	})
+}
+
+func (e *client) UpdateSSHKey(ctx context.Context, id string, r *packngo.SSHKeyUpdateRequest) (*packngo.SSHKey, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) (*packngo.SSHKey, error) {
+		k, _, err := cl.SSHKeys.Update(id, r)
+		if err != nil {
+			return nil, fmt.Errorf("SSHKeys.Update: %w", err)
+		}
+		return k, err
+	})
+}
+
+// Currently unexported, only used in tests.
+func (e *client) deleteSSHKey(ctx context.Context, id string) error {
+	_, err := wrap(ctx, e, func(cl *packngo.Client) (struct{}, error) {
+		_, err := cl.SSHKeys.Delete(id)
+		if err != nil {
+			return struct{}{}, fmt.Errorf("SSHKeys.Delete: %w", err)
+		}
+		return struct{}{}, err
+	})
+	return err
+}
+
+func (e *client) ListSSHKeys(ctx context.Context) ([]packngo.SSHKey, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) ([]packngo.SSHKey, error) {
+		ks, _, err := cl.SSHKeys.List()
+		if err != nil {
+			return nil, fmt.Errorf("SSHKeys.List: %w", err)
+		}
+		return ks, nil
+	})
+}
+
+// Currently unexported, only used in tests.
+func (e *client) getSSHKey(ctx context.Context, id string) (*packngo.SSHKey, error) {
+	return wrap(ctx, e, func(cl *packngo.Client) (*packngo.SSHKey, error) {
+		k, _, err := cl.SSHKeys.Get(id, nil)
+		if err != nil {
+			return nil, fmt.Errorf("SSHKeys.Get: %w", err)
+		}
+		return k, nil
+	})
+}
diff --git a/cloud/shepherd/equinix/wrapngo/wrapngo_test.go b/cloud/shepherd/equinix/wrapngo/wrapngo_test.go
new file mode 100644
index 0000000..d69ff4e
--- /dev/null
+++ b/cloud/shepherd/equinix/wrapngo/wrapngo_test.go
@@ -0,0 +1,412 @@
+package wrapngo
+
+import (
+	"context"
+	"crypto/ed25519"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/packethost/packngo"
+	"golang.org/x/crypto/ssh"
+)
+
+var (
+	ctx context.Context
+
+	// apiuser and apikey are the Equinix credentials necessary to test the
+	// client package.
+
+	apiuser string = os.Getenv("EQUINIX_USER")
+	apikey  string = os.Getenv("EQUINIX_APIKEY")
+
+	// apipid references the Equinix Metal project used. It's recommended to use
+	// non-production projects in the context of testing.
+	apipid string = os.Getenv("EQUINIX_PROJECT_ID")
+	// apios specifies the operating system installed on newly provisioned
+	// devices. See Equinix Metal API documentation for details.
+	apios string = os.Getenv("EQUINIX_DEVICE_OS")
+
+	// sshKeyID identifies the SSH public key registered with Equinix.
+	sshKeyID string
+	// sshKeyLabel is the label used to register the SSH key with Equinix.
+	sshKeyLabel string = "shepherd-client-testkey"
+
+	// testDevice is the device created in TestCreateDevice, and later
+	// referenced to exercise implementation operating on Equinix Metal device
+	// objects.
+	testDevice *packngo.Device
+	// testDeviceHostname is the hostname used to register and reference the
+	// test device.
+	testDeviceHostname string = "shepherd-client-testdev"
+)
+
+// ensureParams returns false if any of the required environment variable
+// parameters are missing.
+func ensureParams() bool {
+	if apiuser == "" {
+		log.Print("EQUINIX_USER must be set.")
+		return false
+	}
+	if apikey == "" {
+		log.Print("EQUINIX_APIKEY must be set.")
+		return false
+	}
+	if apipid == "" {
+		log.Print("EQUINIX_PROJECT_ID must be set.")
+		return false
+	}
+	if apios == "" {
+		log.Print("EQUINIX_DEVICE_OS must be set.")
+		return false
+	}
+	return true
+}
+
+// awaitDeviceState returns nil after device matching the id reaches one of the
+// provided states. It will return a non-nil value in case of an API error, and
+// particularly if there exists no device matching id.
+func awaitDeviceState(ctx context.Context, t *testing.T, cl *client, id string, states ...string) error {
+	if t != nil {
+		t.Helper()
+	}
+
+	for {
+		d, err := cl.GetDevice(ctx, apipid, id)
+		if err != nil {
+			if errors.Is(err, os.ErrDeadlineExceeded) {
+				continue
+			}
+			return fmt.Errorf("while fetching device info: %w", err)
+		}
+		if d == nil {
+			return fmt.Errorf("expected the test device (ID: %s) to exist.", id)
+		}
+		for _, s := range states {
+			if d.State == s {
+				return nil
+			}
+		}
+		log.Printf("Waiting for device to be provisioned (ID: %s, current state: %q)", id, d.State)
+		time.Sleep(time.Second)
+	}
+}
+
+// cleanup ensures both the test device and the test key are deleted at
+// Equinix.
+func cleanup(ctx context.Context, cl *client) {
+	log.Print("Cleaning up.")
+
+	// Ensure the device matching testDeviceHostname is deleted.
+	ds, err := cl.ListDevices(ctx, apipid)
+	if err != nil {
+		log.Fatalf("while listing devices: %v", err)
+	}
+	var td *packngo.Device
+	for _, d := range ds {
+		if d.Hostname == testDeviceHostname {
+			td = &d
+			break
+		}
+	}
+	if td != nil {
+		log.Printf("Found a test device (ID: %s) that needs to be deleted before progressing further.", td.ID)
+
+		// Devices currently being provisioned can't be deleted. After it's
+		// provisioned, device's state will match either "active", or "failed".
+		if err := awaitDeviceState(ctx, nil, cl, td.ID, "active", "failed"); err != nil {
+			log.Fatalf("while waiting for device to be provisioned: %v", err)
+		}
+		if err := cl.deleteDevice(ctx, td.ID); err != nil {
+			log.Fatalf("while deleting test device: %v", err)
+		}
+	}
+
+	// Ensure the key matching sshKeyLabel is deleted.
+	ks, err := cl.ListSSHKeys(ctx)
+	if err != nil {
+		log.Fatalf("while listing SSH keys: %v", err)
+	}
+	for _, k := range ks {
+		if k.Label == sshKeyLabel {
+			log.Printf("Found a SSH test key (ID: %s) - deleting...", k.ID)
+			if err := cl.deleteSSHKey(ctx, k.ID); err != nil {
+				log.Fatalf("while deleting an SSH key: %v", err)
+			}
+			log.Printf("Deleted a SSH test key (ID: %s).", k.ID)
+		}
+	}
+}
+
+func TestMain(m *testing.M) {
+	if !ensureParams() {
+		log.Print("Skipping due to missing parameters.")
+		return
+	}
+	ctx = context.Background()
+
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+	defer cl.Close()
+
+	cleanup(ctx, cl)
+	code := m.Run()
+	cleanup(ctx, cl)
+	os.Exit(code)
+}
+
+// Most test cases depend on the preceding cases having been executed. The
+// test cases can't be run in parallel.
+
+func TestListReservations(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	_, err := cl.ListReservations(ctx, apipid)
+	if err != nil {
+		t.Errorf("while listing hardware reservations: %v", err)
+	}
+}
+
+// createSSHAuthKey returns an SSH public key in OpenSSH authorized_keys
+// format.
+func createSSHAuthKey(t *testing.T) string {
+	t.Helper()
+	pub, _, err := ed25519.GenerateKey(rand.Reader)
+	if err != nil {
+		t.Errorf("while generating SSH key: %v", err)
+	}
+
+	sshpub, err := ssh.NewPublicKey(pub)
+	if err != nil {
+		t.Errorf("while generating SSH public key: %v", err)
+	}
+	return string(ssh.MarshalAuthorizedKey(sshpub))
+}
+
+func TestCreateSSHKey(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	nk, err := cl.CreateSSHKey(ctx, &packngo.SSHKeyCreateRequest{
+		Label:     sshKeyLabel,
+		Key:       createSSHAuthKey(t),
+		ProjectID: apipid,
+	})
+	if err != nil {
+		t.Errorf("while creating an SSH key: %v", err)
+	}
+	if nk.Label != sshKeyLabel {
+		t.Errorf("key labels don't match.")
+	}
+	t.Logf("Created an SSH key (ID: %s)", nk.ID)
+	sshKeyID = nk.ID
+}
+
+var (
+	// dummySSHPK2 is the alternate key used to exercise TestUpdateSSHKey and
+	// TestGetSSHKey.
+	dummySSHPK2 string
+)
+
+func TestUpdateSSHKey(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	if sshKeyID == "" {
+		t.Skip("SSH key couldn't have been created - skipping...")
+	}
+
+	dummySSHPK2 = createSSHAuthKey(t)
+	k, err := cl.UpdateSSHKey(ctx, sshKeyID, &packngo.SSHKeyUpdateRequest{
+		Key: &dummySSHPK2,
+	})
+	if err != nil {
+		t.Errorf("while updating an SSH key: %v", err)
+	}
+	if k.Key != dummySSHPK2 {
+		t.Errorf("updated SSH key doesn't match the original.")
+	}
+}
+
+func TestGetSSHKey(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	if sshKeyID == "" {
+		t.Skip("SSH key couldn't have been created - skipping...")
+	}
+
+	k, err := cl.getSSHKey(ctx, sshKeyID)
+	if err != nil {
+		t.Errorf("while getting an SSH key: %v", err)
+	}
+	if k.Key != dummySSHPK2 {
+		t.Errorf("got key contents that don't match the original.")
+	}
+}
+
+func TestListSSHKeys(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	if sshKeyID == "" {
+		t.Skip("SSH key couldn't have been created - skipping...")
+	}
+
+	ks, err := cl.ListSSHKeys(ctx)
+	if err != nil {
+		t.Errorf("while listing SSH keys: %v", err)
+	}
+
+	// Check that our key is part of the list.
+	found := false
+	for _, k := range ks {
+		if k.ID == sshKeyID {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Errorf("SSH key not listed.")
+	}
+}
+
+func TestCreateDevice(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	// Find a provisionable hardware reservation the device will be created with.
+	rvs, err := cl.ListReservations(ctx, apipid)
+	if err != nil {
+		t.Errorf("while listing hardware reservations: %v", err)
+	}
+	var rv *packngo.HardwareReservation
+	for _, r := range rvs {
+		if r.Provisionable {
+			rv = &r
+			break
+		}
+	}
+	if rv == nil {
+		t.Skip("could not find a provisionable hardware reservation - skipping...")
+	}
+
+	d, err := cl.CreateDevice(ctx, &packngo.DeviceCreateRequest{
+		Hostname:              testDeviceHostname,
+		OS:                    apios,
+		Plan:                  rv.Plan.Slug,
+		HardwareReservationID: rv.ID,
+		ProjectID:             apipid,
+	})
+	if err != nil {
+		t.Errorf("while creating a device: %v", err)
+	}
+	t.Logf("Created a new test device (ID: %s)", d.ID)
+	testDevice = d
+}
+
+func TestGetDevice(t *testing.T) {
+	if testDevice == nil {
+		t.Skip("the test device couldn't have been created - skipping...")
+	}
+
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	d, err := cl.GetDevice(ctx, apipid, testDevice.ID)
+	if err != nil {
+		t.Errorf("while fetching device info: %v", err)
+	}
+	if d == nil {
+		t.Errorf("expected the test device (ID: %s) to exist.", testDevice.ID)
+	}
+	if d.ID != testDevice.ID {
+		t.Errorf("got device ID that doesn't match the original.")
+	}
+}
+
+func TestListDevices(t *testing.T) {
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	ds, err := cl.ListDevices(ctx, apipid)
+	if err != nil {
+		t.Errorf("while listing devices: %v", err)
+	}
+	if len(ds) == 0 {
+		t.Errorf("expected at least one device.")
+	}
+}
+
+func TestDeleteDevice(t *testing.T) {
+	if testDevice == nil {
+		t.Skip("the test device couldn't have been created - skipping...")
+	}
+
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	// Devices currently being provisioned can't be deleted. After it's
+	// provisioned, device's state will match either "active", or "failed".
+	if err := awaitDeviceState(ctx, t, cl, testDevice.ID, "active", "failed"); err != nil {
+		t.Errorf("while waiting for device to be provisioned: %v", err)
+	}
+	t.Logf("Deleting the test device (ID: %s)", testDevice.ID)
+	if err := cl.deleteDevice(ctx, testDevice.ID); err != nil {
+		t.Errorf("while deleting a device: %v", err)
+	}
+	d, err := cl.GetDevice(ctx, apipid, testDevice.ID)
+	if err != nil && !IsNotFound(err) {
+		t.Errorf("while fetching device info: %v", err)
+	}
+	if d != nil {
+		t.Errorf("device should not exist.")
+	}
+	t.Logf("Deleted the test device (ID: %s)", testDevice.ID)
+}
+
+func TestDeleteSSHKey(t *testing.T) {
+	if sshKeyID == "" {
+		t.Skip("SSH key couldn't have been created - skipping...")
+	}
+
+	cl := new(&Opts{
+		User:   apiuser,
+		APIKey: apikey,
+	})
+
+	t.Logf("Deleting the test SSH key (ID: %s)", sshKeyID)
+	if err := cl.deleteSSHKey(ctx, sshKeyID); err != nil {
+		t.Errorf("couldn't delete an SSH key: %v", err)
+	}
+	_, err := cl.getSSHKey(ctx, sshKeyID)
+	if err == nil {
+		t.Errorf("SSH key should not exist")
+	}
+	t.Logf("Deleted the test SSH key (ID: %s)", sshKeyID)
+}