[PATCH net-next 14/17] net: dsa: mv88e6xxx: Time RMU operations and disable if slow

[Luke Howard]

From: Andrew Lunn <andrew@xxxxxxx>

MII access to the switch generally has consistent latency. Sending and
receiving frames on the other hand has a big variance in
latency. Receiver interrupt coalescence can result in delays of
100us. As a result, the RMU can be slower than MDIO.

Measure the latency of the first 16 read requests performed using the
RMU, and compare it to the latency on an MDIO read. If the RMU is
slower than MDIO, do not use if for single register reads, writes and
waiting for bits to be set. Do however use it for retrieving MIB
values since a single RMU requests replaces a large number of MIDO
transactions.

Testing on ZII Devel C, which uses a FEC, and Marvel 370RD using
mvneta has show RMU is slower than MDIO. ZII RAP however, makes use of
bit banging MDIO and an IGB. IGB by default has a short interrupt
coalesce period, making RMU much faster than bit banging, by around 20
times.

Signed-off-by: Andrew Lunn <andrew@xxxxxxx>
Signed-off-by: Luke Howard <lukeh@xxxxxxxx>
---
 drivers/net/dsa/mv88e6xxx/chip.h |  6 +++++
 drivers/net/dsa/mv88e6xxx/rmu.c  | 58 ++++++++++++++++++++++++++++++++++++----
 2 files changed, 59 insertions(+), 5 deletions(-)

diff --git a/drivers/net/dsa/mv88e6xxx/chip.h b/drivers/net/dsa/mv88e6xxx/chip.h
index 7e483ba8c58d9..fdc87d02469b5 100644
--- a/drivers/net/dsa/mv88e6xxx/chip.h
+++ b/drivers/net/dsa/mv88e6xxx/chip.h
@@ -355,6 +355,8 @@ enum mv88e6xxx_rmu_state {
 	 */
 };
 
+#define MV88E6XXX_RMU_IS_SLOW	BIT(0)
+
 struct mv88e6xxx_chip {
 	const struct mv88e6xxx_info *info;
 
@@ -471,6 +473,10 @@ struct mv88e6xxx_chip {
 	struct net_device *rmu_conduit;
 	struct dsa_inband rmu_inband;
 	enum mv88e6xxx_rmu_state rmu_state;
+	u8 rmu_flags;
+	ktime_t rmu_read_latencies[16];
+	u32 rmu_samples;
+	ktime_t smi_read_latency;
 };
 
 #define TCAM_MATCH_SIZE 96
diff --git a/drivers/net/dsa/mv88e6xxx/rmu.c b/drivers/net/dsa/mv88e6xxx/rmu.c
index 88b02030a774f..5660372f41ad7 100644
--- a/drivers/net/dsa/mv88e6xxx/rmu.c
+++ b/drivers/net/dsa/mv88e6xxx/rmu.c
@@ -221,7 +221,8 @@ int mv88e6xxx_rmu_write(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val)
 	int resp_len;
 	int ret = -1;
 
-	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED)
+	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED ||
+	    (chip->rmu_flags & MV88E6XXX_RMU_IS_SLOW))
 		return -EOPNOTSUPP;
 
 	resp_len = sizeof(resp);
@@ -249,6 +250,33 @@ int mv88e6xxx_rmu_write(struct mv88e6xxx_chip *chip, int addr, int reg, u16 val)
 	return 0;
 }
 
+static void mv88e6xxx_rmu_read_latency(struct mv88e6xxx_chip *chip,
+				       ktime_t latency)
+{
+	ktime_t average = 0;
+	int i;
+
+	if (chip->rmu_samples >= ARRAY_SIZE(chip->rmu_read_latencies))
+		return;
+
+	chip->rmu_read_latencies[chip->rmu_samples++] = latency;
+
+	if (chip->rmu_samples == ARRAY_SIZE(chip->rmu_read_latencies)) {
+		for (i = 0; i < ARRAY_SIZE(chip->rmu_read_latencies); i++)
+			average += chip->rmu_read_latencies[i];
+		average = average / ARRAY_SIZE(chip->rmu_read_latencies);
+
+		dev_dbg(chip->dev, "RMU %lldus, smi %lldus\n",
+			div_u64(average, 1000),
+			div_u64(chip->smi_read_latency, 1000));
+
+		if (chip->smi_read_latency < average)
+			chip->rmu_flags |= MV88E6XXX_RMU_IS_SLOW;
+
+		chip->rmu_samples = U32_MAX;
+	}
+}
+
 int mv88e6xxx_rmu_read(struct mv88e6xxx_chip *chip, int addr, int reg,
 		       u16 *val)
 {
@@ -263,11 +291,15 @@ int mv88e6xxx_rmu_read(struct mv88e6xxx_chip *chip, int addr, int reg,
 	};
 	struct mv88e6xxx_rmu_rw_resp resp;
 	int resp_len;
-	int ret = -1;
+	ktime_t start;
+	int ret;
 
-	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED)
+	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED ||
+	    (chip->rmu_flags & MV88E6XXX_RMU_IS_SLOW))
 		return -EOPNOTSUPP;
 
+	start = ktime_get();
+
 	resp_len = sizeof(resp);
 	ret = mv88e6xxx_rmu_request(chip, req, sizeof(req),
 				    &resp, resp_len,
@@ -290,6 +322,8 @@ int mv88e6xxx_rmu_read(struct mv88e6xxx_chip *chip, int addr, int reg,
 		return -EPROTO;
 	}
 
+	mv88e6xxx_rmu_read_latency(chip, ktime_get() - start);
+
 	*val = ntohs(resp.value);
 
 	return 0;
@@ -312,7 +346,8 @@ int mv88e6xxx_rmu_wait_bit(struct mv88e6xxx_chip *chip, int addr, int reg,
 	int resp_len;
 	int ret = -1;
 
-	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED)
+	if (chip->rmu_state == MV88E6XXX_RMU_DISABLED ||
+	    (chip->rmu_flags & MV88E6XXX_RMU_IS_SLOW))
 		return -EOPNOTSUPP;
 
 	resp_len = sizeof(resp);
@@ -389,8 +424,10 @@ void mv88e6xxx_rmu_conduit_state_change(struct dsa_switch *ds,
 {
 	struct dsa_port *cpu_dp = conduit->dsa_ptr;
 	struct mv88e6xxx_chip *chip = ds->priv;
+	ktime_t start;
 	int port;
 	int ret;
+	u16 id;
 
 	port = dsa_towards_port(ds, cpu_dp->ds->index, cpu_dp->index);
 
@@ -414,13 +451,24 @@ void mv88e6xxx_rmu_conduit_state_change(struct dsa_switch *ds,
 		if (ret < 0) {
 			dev_err(chip->dev, "RMU: initialization check failed %pe",
 				ERR_PTR(ret));
-			goto out;
+			goto out_disable;
 		}
+
+		start = ktime_get();
+		ret = mv88e6xxx_port_read(chip, 0, MV88E6XXX_PORT_SWITCH_ID,
+					  &id);
+		if (ret < 0) {
+			dev_err(chip->dev, "RMU: SMI latency read failed %pe",
+				ERR_PTR(ret));
+			goto out_disable;
+		}
+		chip->smi_read_latency = ktime_get() - start;
 		chip->rmu_state = MV88E6XXX_RMU_ENABLED;
 
 		dev_info(chip->dev, "RMU: enabled on port %d via conduit device %s",
 			 port, chip->rmu_conduit->name);
 	} else {
+out_disable:
 		if (chip->info->ops->rmu_disable)
 			chip->info->ops->rmu_disable(chip);
 

-- 
2.43.0