m/n/c/network: implement ARP announcements
This implements ARP announcements in the Metropolis network stack.
Its intent is to help IP stacks on the same broadcast domain to update
their ARP entries once a Metropolis network stack comes up.
The format of the ARP packets is chosen to bypass most EVPN ARP
suppression mechanisms to ensure this also works with these systems.
Change-Id: I2db1248f7034ea56930cf6f4a93de598b0f8c7de
Reviewed-on: https://review.monogon.dev/c/monogon/+/3074
Reviewed-by: Leopold Schabel <leo@monogon.tech>
Tested-by: Jenkins CI
diff --git a/go.mod b/go.mod
index fc42bf3..3ff67fb 100644
--- a/go.mod
+++ b/go.mod
@@ -103,6 +103,8 @@
github.com/kevinburke/go-bindata v3.23.0+incompatible
github.com/lib/pq v1.10.9
github.com/mattn/go-shellwords v1.0.12
+ github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
+ github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118
github.com/mdlayher/ethtool v0.1.0
github.com/mdlayher/genetlink v1.3.2
github.com/mdlayher/kobject v0.0.0-20200520190114-19ca17470d7d
diff --git a/go.sum b/go.sum
index 11423fe..47153dd 100644
--- a/go.sum
+++ b/go.sum
@@ -2203,6 +2203,10 @@
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg=
github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k=
github.com/maxbrunsfeld/counterfeiter/v6 v6.2.2/go.mod h1:eD9eIE7cdwcMi9rYluz88Jz2VyhSmden33/aXg4oVIY=
+github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875 h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY=
+github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875/go.mod h1:kfOoFJuHWp76v1RgZCb9/gVUc7XdY877S2uVYbNliGc=
+github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE=
+github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118/go.mod h1:ZFUnHIVchZ9lJoWoEGUg8Q3M4U8aNNWA3CVSUTkW4og=
github.com/mdlayher/genetlink v1.2.0/go.mod h1:ra5LDov2KrUCZJiAtEvXXZBxGMInICMXIwshlJ+qRxQ=
github.com/mdlayher/genetlink v1.3.2 h1:KdrNKe+CTu+IbZnm/GVUMXSqBBLqcGpRDa0xkQy56gw=
github.com/mdlayher/genetlink v1.3.2/go.mod h1:tcC3pkCrPUGIKKsCsp0B3AdaaKuHtaxoJRz3cc+528o=
@@ -2216,10 +2220,12 @@
github.com/mdlayher/netlink v1.6.0/go.mod h1:0o3PlBmGst1xve7wQ7j/hwpNaFaH4qCRyWCdcZk8/vA=
github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g=
github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw=
+github.com/mdlayher/packet v1.0.0/go.mod h1:eE7/ctqDhoiRhQ44ko5JZU2zxB88g+JH/6jmnjzPjOU=
github.com/mdlayher/packet v1.1.2 h1:3Up1NG6LZrsgDVn6X4L9Ge/iyRyxFEFD9o6Pr3Q1nQY=
github.com/mdlayher/packet v1.1.2/go.mod h1:GEu1+n9sG5VtiRE4SydOmX5GTwyyYlteZiFU+x0kew4=
github.com/mdlayher/socket v0.0.0-20211102153432-57e3fa563ecb/go.mod h1:nFZ1EtZYK8Gi/k6QNu7z7CgO20i/4ExeQswwWuPmG/g=
github.com/mdlayher/socket v0.1.1/go.mod h1:mYV5YIZAfHh4dzDVzI8x8tWLWCliuX8Mon5Awbj+qDs=
+github.com/mdlayher/socket v0.2.1/go.mod h1:QLlNPkFR88mRUNQIzRBMfXxwKal8H7u1h3bL1CV+f0E=
github.com/mdlayher/socket v0.4.1/go.mod h1:cAqeGjoufqdxWkD7DkpyS+wcefOtmu5OQ8KuoJGIReA=
github.com/mdlayher/socket v0.5.0 h1:ilICZmJcQz70vrWVes1MFera4jGiWNocSkykwwoy3XI=
github.com/mdlayher/socket v0.5.0/go.mod h1:WkcBFfvyG8QENs5+hfQPl1X6Jpd2yeLIYgrGFmJiJxI=
diff --git a/metropolis/node/core/network/BUILD.bazel b/metropolis/node/core/network/BUILD.bazel
index 07120e4..8584a3b 100644
--- a/metropolis/node/core/network/BUILD.bazel
+++ b/metropolis/node/core/network/BUILD.bazel
@@ -4,6 +4,7 @@
name = "network",
srcs = [
"main.go",
+ "neigh.go",
"quirks.go",
"static.go",
],
@@ -22,6 +23,8 @@
"@com_github_google_nftables//:nftables",
"@com_github_google_nftables//expr",
"@com_github_insomniacslk_dhcp//dhcpv4",
+ "@com_github_mdlayher_arp//:arp",
+ "@com_github_mdlayher_ethernet//:ethernet",
"@com_github_mdlayher_ethtool//:ethtool",
"@com_github_vishvananda_netlink//:netlink",
"@org_golang_x_sys//unix",
diff --git a/metropolis/node/core/network/main.go b/metropolis/node/core/network/main.go
index 5404660..078eab8 100644
--- a/metropolis/node/core/network/main.go
+++ b/metropolis/node/core/network/main.go
@@ -178,7 +178,7 @@
}
s.dhcp.VendorClassIdentifier = s.DHCPVendorClassID
s.dhcp.RequestedOptions = []dhcpv4.OptionCode{dhcpv4.OptionRouter, dhcpv4.OptionDomainNameServer, dhcpv4.OptionClasslessStaticRoute}
- s.dhcp.LeaseCallback = dhcpcb.Compose(dhcpcb.ManageIP(iface), dhcpcb.ManageRoutes(iface), s.statusCallback(ctx))
+ s.dhcp.LeaseCallback = dhcpcb.Compose(dhcpcb.ManageIP(iface), arpAnnounceCB(iface), dhcpcb.ManageRoutes(iface), s.statusCallback(ctx))
err = supervisor.Run(ctx, "dhcp", s.dhcp.Run)
if err != nil {
return err
@@ -219,6 +219,8 @@
logger.Errorf("Applying quirks failed, continuing without: %v", err)
}
+ supervisor.Run(ctx, "announce", s.runNeighborAnnounce)
+
// Choose between autoconfig and static config runnables
if s.StaticConfig == nil {
supervisor.Run(ctx, "dynamic", s.runDynamicConfig)
diff --git a/metropolis/node/core/network/neigh.go b/metropolis/node/core/network/neigh.go
new file mode 100644
index 0000000..0d434ab
--- /dev/null
+++ b/metropolis/node/core/network/neigh.go
@@ -0,0 +1,140 @@
+package network
+
+import (
+ "context"
+ "fmt"
+ "net"
+ "net/netip"
+ "strings"
+ "time"
+
+ "github.com/mdlayher/arp"
+ "github.com/mdlayher/ethernet"
+ "github.com/vishvananda/netlink"
+ "golang.org/x/sys/unix"
+
+ "source.monogon.dev/metropolis/node/core/network/dhcp4c"
+ "source.monogon.dev/osbase/supervisor"
+)
+
+var ethernetNull = net.HardwareAddr{0, 0, 0, 0, 0, 0}
+
+// runNeighborAnnounce announces all configured IPs marked as permanent via ARP
+// every time an interface comes up. Non-permanent IPs are handled via
+// arpAnnounceCB. This is done to update ARP tables on all attached hosts,
+// which commonly has very large (hours) timeouts otherwise. The packets are
+// crafted to bypass EVPN ARP suppression to ensure every attached host gets
+// the update.
+func (s *Service) runNeighborAnnounce(ctx context.Context) error {
+ l := supervisor.Logger(ctx)
+ linkUpdates := make(chan netlink.LinkUpdate, 10)
+ if err := netlink.LinkSubscribe(linkUpdates, ctx.Done()); err != nil {
+ return fmt.Errorf("while subscribing to netlink link updates: %w", err)
+ }
+ lastIfState := make(map[string]bool)
+ for {
+ select {
+ case u := <-linkUpdates:
+ attrs := u.Link.Attrs()
+ before := lastIfState[attrs.Name]
+ now := attrs.RawFlags&unix.IFF_RUNNING != 0
+ lastIfState[attrs.Name] = now
+
+ if !before && now {
+ if err := sendARPAnnouncements(u.Link); err != nil {
+ l.Warningf("Failed sending ARP announcements for interface %q: %v", attrs.Name, err)
+ }
+ // Send a second one after 10s if the network infrastructure is
+ // slow to configure itself after link up or the first one got
+ // lost.
+ time.AfterFunc(10*time.Second, func() {
+ if err := sendARPAnnouncements(u.Link); err != nil {
+ l.Warningf("Failed sending repeated ARP announcements for interface %q: %v", attrs.Name, err)
+ }
+ })
+ l.Infof("Interface %q is up", attrs.Name)
+ } else if before && !now {
+ l.Infof("Interface %q is down", attrs.Name)
+ }
+ case <-ctx.Done():
+ return ctx.Err()
+ }
+ }
+}
+
+// sendARPAnnouncements sends an ARP announcement (a form of gratuitous ARP)
+// for every permanent IPv4 address configured on the interface.
+func sendARPAnnouncements(l netlink.Link) error {
+ ac, err := arp.Dial(netlinkLinkToNetInterface(l))
+ if err != nil {
+ // If no IPv4 address is found this is not an error, just return as
+ // there is nothing to do. Sadly errNoIPv4Addr is not exported, so a
+ // string match has to be used.
+ if strings.Contains(err.Error(), "no IPv4 address available for interface") {
+ return nil
+ }
+ return fmt.Errorf("while creating ARP socket: %w", err)
+ }
+ ac.SetWriteDeadline(time.Now().Add(5 * time.Second))
+ defer ac.Close()
+ addrs, err := netlink.AddrList(l, netlink.FAMILY_V4)
+ if err != nil {
+ return fmt.Errorf("while listing configured IPs: %w", err)
+ }
+ for _, addr := range addrs {
+ if addr.Flags&unix.IFA_F_PERMANENT != 0 && (addr.IP.IsGlobalUnicast() || addr.IP.IsLinkLocalUnicast()) {
+ addrIP, ok := netip.AddrFromSlice(addr.IP)
+ if !ok {
+ continue
+ }
+ garpPkt, err := arp.NewPacket(arp.OperationRequest, l.Attrs().HardwareAddr, addrIP, ethernetNull, addrIP)
+ if err != nil {
+ continue
+ }
+ if err := ac.WriteTo(garpPkt, ethernet.Broadcast); err != nil {
+ continue
+ }
+ }
+ }
+ return nil
+}
+
+// A DHCPv4 callback function which announces acquired IPv4 addresses via
+// ARP announcement.
+func arpAnnounceCB(l netlink.Link) dhcp4c.LeaseCallback {
+ var lastIP net.IP
+ return func(lease *dhcp4c.Lease) error {
+ var currentIP net.IP
+ if lease != nil {
+ currentIP = lease.AssignedIP
+ }
+ needsAnnounce := !lastIP.Equal(currentIP) && (currentIP.IsGlobalUnicast() || currentIP.IsLinkLocalUnicast())
+ lastIP = currentIP
+ if needsAnnounce {
+ // This function is best-effort, do not return an error as that
+ // can impair DHCP function.
+ ac, err := arp.Dial(netlinkLinkToNetInterface(l))
+ if err != nil {
+ //nolint:returnerrcheck
+ return nil
+ }
+ ac.SetWriteDeadline(time.Now().Add(5 * time.Second))
+ defer ac.Close()
+ addrIP, ok := netip.AddrFromSlice(currentIP)
+ if !ok {
+ //nolint:returnerrcheck
+ return nil
+ }
+ garpPkt, err := arp.NewPacket(arp.OperationRequest, l.Attrs().HardwareAddr, addrIP, ethernetNull, addrIP)
+ if err != nil {
+ //nolint:returnerrcheck
+ return nil
+ }
+ if err := ac.WriteTo(garpPkt, ethernet.Broadcast); err != nil {
+ //nolint:returnerrcheck
+ return nil
+ }
+ }
+ return nil
+ }
+}
diff --git a/metropolis/node/core/network/static.go b/metropolis/node/core/network/static.go
index f0cc4e2..93b9306 100644
--- a/metropolis/node/core/network/static.go
+++ b/metropolis/node/core/network/static.go
@@ -200,7 +200,7 @@
return fmt.Errorf("failed creating DHCPv4 client: %w", err)
}
c.RequestedOptions = []dhcpv4.OptionCode{dhcpv4.OptionRouter, dhcpv4.OptionDomainNameServer, dhcpv4.OptionClasslessStaticRoute}
- c.LeaseCallback = dhcpcb.Compose(dhcpcb.ManageIP(lnk), dhcpcb.ManageRoutes(lnk), s.statusCallback(ctx))
+ c.LeaseCallback = dhcpcb.Compose(dhcpcb.ManageIP(lnk), arpAnnounceCB(lnk), dhcpcb.ManageRoutes(lnk), s.statusCallback(ctx))
return supervisor.Run(ctx, "dhcp-"+lnk.Attrs().Name, c.Run)
}
diff --git a/third_party/go/repositories.bzl b/third_party/go/repositories.bzl
index dcffde4..ca0dd2e 100644
--- a/third_party/go/repositories.bzl
+++ b/third_party/go/repositories.bzl
@@ -3562,6 +3562,18 @@
version = "v6.2.2",
)
go_repository(
+ name = "com_github_mdlayher_arp",
+ importpath = "github.com/mdlayher/arp",
+ sum = "h1:ql8x//rJsHMjS+qqEag8n3i4azw1QneKh5PieH9UEbY=",
+ version = "v0.0.0-20220512170110-6706a2966875",
+ )
+ go_repository(
+ name = "com_github_mdlayher_ethernet",
+ importpath = "github.com/mdlayher/ethernet",
+ sum = "h1:2oDp6OOhLxQ9JBoUuysVz9UZ9uI6oLUbvAZu0x8o+vE=",
+ version = "v0.0.0-20220221185849-529eae5b6118",
+ )
+ go_repository(
name = "com_github_mdlayher_ethtool",
importpath = "github.com/mdlayher/ethtool",
replace = "github.com/monogon-dev/ethtool",