fix(rt_tables): add path fallback logic

Ever since version v6.5.0 of iproute2, iproute2 no longer automatically
creates the /etc/iproute2 files, instead preferring to add files to
/usr/lib/iproute2 and then later on /usr/share/iproute2.

This adds fallback path matching to kube-router so that it can find
/etc/iproute2/rt_tables wherever it is defined instead of just failing.

This also means people running kube-router in containers will need to
change their mounts depending on where this file is located on their
host OS. However, ensuring that this file is copied to `/etc/iproute2`
is a legitimate way to ensure that this is consistent across a fleet of
multiple OS versions.

Author: aauren

docs/dsr.md

@@ -27,7 +27,11 @@ Requirements:
   * `hostIPC: true` must be set for the pod
   * `hostPID: true` must be set for the pod
   * The container runtime socket must be mounted into the kube-router pod via a `hostPath` volume mount.
-  * `/etc/iproute2/rt_tables` must be read/write mounted into the kube-router pod via a `hostPath` volume mount.
+  * `/etc/iproute2/rt_tables` (or similar) must be read/write mounted into the kube-router pod via a `hostPath` volume
+    mount. NOTE: since v6.5.0 of iproute2 this file has been moved underneath `/usr` in either
+    `/usr/lib/iproute2/rt_tables` or `/usr/share/iproute2/rt_tables` instead of in `/etc` so this mount may need to be
+    updated depending on which version of Linux you're deploying against. kube-router will check all 3 locations and
+    use them in order of the above.
 * A pod network that allows for IPIP encapsulated traffic. The most notable exception to this is that Azure does not
   transit IPIP encapsulated packets on their network. In this scenario, the end-user may be able to get around this
   issue by enabling FoU (`--overlay-encap=fou`) and full overlay networking (`--overlay-type=full`) options in

pkg/controllers/proxy/linux_networking.go

@@ -426,22 +426,11 @@ func (ln *linuxNetworking) ipvsAddServer(service *ipvs.Service, dest *ipvs.Desti
 // http://www.austintek.com/LVS/LVS-HOWTO/HOWTO/LVS-HOWTO.routing_to_VIP-less_director.html
 // setupPolicyRoutingForDSR: setups policy routing so that FWMARKed packets are delivered locally
 func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) error {
-	b, err := os.ReadFile("/etc/iproute2/rt_tables")
+	err := utils.RouteTableAdd(customDSRRouteTableID, customDSRRouteTableName)
 	if err != nil {
 		return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err)
 	}
 
-	if !strings.Contains(string(b), customDSRRouteTableName) {
-		f, err := os.OpenFile("/etc/iproute2/rt_tables", os.O_APPEND|os.O_WRONLY, 0600)
-		if err != nil {
-			return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err)
-		}
-		defer utils.CloseCloserDisregardError(f)
-		if _, err = f.WriteString(customDSRRouteTableID + " " + customDSRRouteTableName + "\n"); err != nil {
-			return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err)
-		}
-	}
-
 	if setupIPv4 {
 		out, err := exec.Command("ip", "route", "list", "table", customDSRRouteTableID).Output()
 		if err != nil || !strings.Contains(string(out), " lo ") {
@@ -470,20 +459,9 @@ func (ln *linuxNetworking) setupPolicyRoutingForDSR(setupIPv4, setupIPv6 bool) e
 
 func (ln *linuxNetworking) setupRoutesForExternalIPForDSR(serviceInfoMap serviceInfoMap,
 	setupIPv4, setupIPv6 bool) error {
-	b, err := os.ReadFile("/etc/iproute2/rt_tables")
+	err := utils.RouteTableAdd(externalIPRouteTableID, externalIPRouteTableName)
 	if err != nil {
-		return fmt.Errorf("failed to setup external ip routing table required for DSR due to %v", err)
-	}
-
-	if !strings.Contains(string(b), externalIPRouteTableName) {
-		f, err := os.OpenFile("/etc/iproute2/rt_tables", os.O_APPEND|os.O_WRONLY, 0600)
-		if err != nil {
-			return fmt.Errorf("failed setup external ip routing table required for DSR due to %v", err)
-		}
-		defer utils.CloseCloserDisregardError(f)
-		if _, err = f.WriteString(externalIPRouteTableID + " " + externalIPRouteTableName + "\n"); err != nil {
-			return fmt.Errorf("failed setup external ip routing table required for DSR due to %v", err)
-		}
+		return fmt.Errorf("failed to setup policy routing required for DSR due to %v", err)
 	}
 
 	setupIPRulesAndRoutes := func(ipArgs []string) error {

pkg/controllers/routing/pbr.go

@@ -2,7 +2,6 @@ package routing
 
 import (
 	"fmt"
-	"os"
 	"os/exec"
 	"strings"
 
@@ -37,7 +36,7 @@ func ipRuleAbstraction(ipProtocol, ipOp, cidr string) error {
 // setup a custom routing table that will be used for policy based routing to ensure traffic originating
 // on tunnel interface only leaves through tunnel interface irrespective rp_filter enabled/disabled
 func (nrc *NetworkRoutingController) enablePolicyBasedRouting() error {
-	err := rtTablesAdd(customRouteTableID, customRouteTableName)
+	err := utils.RouteTableAdd(customRouteTableID, customRouteTableName)
 	if err != nil {
 		return fmt.Errorf("failed to update rt_tables file: %s", err)
 	}
@@ -61,7 +60,7 @@ func (nrc *NetworkRoutingController) enablePolicyBasedRouting() error {
 }
 
 func (nrc *NetworkRoutingController) disablePolicyBasedRouting() error {
-	err := rtTablesAdd(customRouteTableID, customRouteTableName)
+	err := utils.RouteTableAdd(customRouteTableID, customRouteTableName)
 	if err != nil {
 		return fmt.Errorf("failed to update rt_tables file: %s", err)
 	}
@@ -83,23 +82,3 @@ func (nrc *NetworkRoutingController) disablePolicyBasedRouting() error {
 
 	return nil
 }
-
-func rtTablesAdd(tableNumber, tableName string) error {
-	b, err := os.ReadFile("/etc/iproute2/rt_tables")
-	if err != nil {
-		return fmt.Errorf("failed to read: %s", err.Error())
-	}
-
-	if !strings.Contains(string(b), tableName) {
-		f, err := os.OpenFile("/etc/iproute2/rt_tables", os.O_APPEND|os.O_WRONLY, 0600)
-		if err != nil {
-			return fmt.Errorf("failed to open: %s", err.Error())
-		}
-		defer utils.CloseCloserDisregardError(f)
-		if _, err = f.WriteString(tableNumber + " " + tableName + "\n"); err != nil {
-			return fmt.Errorf("failed to write: %s", err.Error())
-		}
-	}
-
-	return nil
-}

pkg/utils/linux_routing.go

@@ -0,0 +1,56 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+const (
+	rtTablesFileName = "rt_tables"
+	iproutePkg       = "iproute2"
+)
+
+var (
+	rtTablesPosLoc = []string{
+		fmt.Sprintf("/etc/%s/%s", iproutePkg, rtTablesFileName),
+		fmt.Sprintf("/usr/lib/%s/%s", iproutePkg, rtTablesFileName),
+		fmt.Sprintf("/usr/share/%s/%s", iproutePkg, rtTablesFileName),
+	}
+)
+
+// RouteTableAdd adds a new named table to iproute's rt_tables configuration file
+func RouteTableAdd(tableNumber, tableName string) error {
+	var rtTablesLoc string
+	for _, possibleLoc := range rtTablesPosLoc {
+		_, err := os.Stat(possibleLoc)
+		if err != nil {
+			klog.V(2).Infof("Did not find iproute2's rt_tables in location %s", possibleLoc)
+			continue
+		}
+		rtTablesLoc = possibleLoc
+	}
+	if rtTablesLoc == "" {
+		return fmt.Errorf("did not find rt_tables in any of the expected locations: %s", rtTablesFileName)
+	}
+
+	b, err := os.ReadFile(rtTablesLoc)
+	if err != nil {
+		return fmt.Errorf("failed to read: %s", err.Error())
+	}
+
+	if !strings.Contains(string(b), tableName) {
+		f, err := os.OpenFile(rtTablesLoc, os.O_APPEND|os.O_WRONLY, 0600)
+		if err != nil {
+			return fmt.Errorf("failed to open: %s", err.Error())
+		}
+		defer CloseCloserDisregardError(f)
+		if _, err = f.WriteString(tableNumber + " " + tableName + "\n"); err != nil {
+			return fmt.Errorf("failed to write: %s", err.Error())
+		}
+	}
+
+	return nil
+}