InfUdDriver: obtain network bandwidth properly by querying the port

Author: yilongli

src/InfUdDriver.cc

@@ -98,22 +98,7 @@ InfUdDriver::InfUdDriver(Context* context, const ServiceLocator *sl,
  , qpn(0)
  , localMac()
  , locatorString("infud:")
- // As of 11/2018, the network bandwidth of our RC machines at Stanford is
- // actually limited by the effective bandwidth of PCIe 2.0x4, which should
- // be ~29Gbps when taking into account the overhead of PCIe headers.
- // For example, suppose the HCA's MTU is 256B and the PCIe headers are 24B
- // in total, the effective bandwidth of PCIe 2.0x4 is
- // 32Gbps * 256 / (256 + 24) = 29.25Gbps
- // Unfortunately, it appears that our ConnextX-2 HCA somehow cannot fully
- // utilize the 29Gbps PCIe bandwidth when sending UD packets. This can be
- // verified by running one or more ib_send_bw programs on two machines.
- // The maximum outgoing bandwidth we can achieve in practice is ~3020MB/s,
- // or 23.6Gbps. Note that we need to set the outgoing bandwidth slightly
- // higher than 24Gbps in order to saturate the 23.6Gbps outgoing bandwidth.
- // This is because the throughput of the HCA has non-negligible variation:
- // when it's running faster than 24Gbps, we don't want the transport to
- // throttle the throughput and leave the HCA idle.
- , bandwidthGbps(26) // Default outgoing bandwidth in gbs
+ , bandwidthGbps(~0u)
  , zeroCopyStart(NULL)
  , zeroCopyEnd(NULL)
  , zeroCopyRegion(NULL)
@@ -146,6 +131,27 @@ InfUdDriver::InfUdDriver(Context* context, const ServiceLocator *sl,
  } catch (ServiceLocator::NoSuchKeyException& e) {}
  }
  infiniband = realInfiniband.construct(ibDeviceName);
+#if 1
+ bandwidthGbps = std::min(bandwidthGbps,
+ static_cast<int>(infiniband->getBandwidthGbps(ibPhysicalPort)));
+#else
+ // As of 11/2018, the network bandwidth of our RC machines at Stanford is
+ // actually limited by the effective bandwidth of PCIe 2.0x4, which should
+ // be ~29Gbps when taking into account the overhead of PCIe headers.
+ // For example, suppose the HCA's MTU is 256B and the PCIe headers are 24B
+ // in total, the effective bandwidth of PCIe 2.0x4 is
+ // 32Gbps * 256 / (256 + 24) = 29.25Gbps
+ // Unfortunately, it appears that our ConnextX-2 HCA somehow cannot fully
+ // utilize the 29Gbps PCIe bandwidth when sending UD packets. This can be
+ // verified by running one or more ib_send_bw programs on two machines.
+ // The maximum outgoing bandwidth we can achieve in practice is ~3020MB/s,
+ // or 23.6Gbps. Note that we need to set the outgoing bandwidth slightly
+ // higher than 24Gbps in order to saturate the 23.6Gbps outgoing bandwidth.
+ // This is because the throughput of the HCA has non-negligible variation:
+ // when it's running faster than 24Gbps, we don't want the transport to
+ // throttle the throughput and leave the HCA idle.
+ bandwidthGbps = std::min(bandwidthGbps, 26);
+#endif
  mtu = infiniband->getMtu(ibPhysicalPort);
  queueEstimator.setBandwidth(1000*bandwidthGbps);
  maxTransmitQueueSize = (uint32_t) (static_cast<double>(bandwidthGbps)

src/Infiniband.cc

@@ -140,6 +140,52 @@ Infiniband::getLid(int port)
  return ipa.lid;
 }
 
+/**
+ * Obtain the link speed on this port to transmit and receive.
+ *
+ * \param[in] port
+ * Port on the device we're looking up. This value is typically 1, except
+ * on adapters with multiple physical ports.
+ * \return
+ * The bandwidth, in Gbps.
+ * \throw
+ * TransportException if the port cannot be queried.
+ */
+uint32_t
+Infiniband::getBandwidthGbps(int port)
+{
+ ibv_port_attr ipa;
+ int ret = ibv_query_port(device.ctxt, downCast<uint8_t>(port), &ipa);
+ if (ret) {
+ RAMCLOUD_LOG(ERROR, "ibv_query_port failed on port %d", port);
+ throw TransportException(HERE, ret);
+ }
+ // The meaning of fields active_width and active_speed can be found at:
+ // https://www.rdmamojo.com/2012/07/21/ibv_query_port/
+ uint32_t gbps = 1;
+ switch (ipa.active_width) {
+ case 1: gbps = 1; break;
+ case 2: gbps = 4; break;
+ case 4: gbps = 8; break;
+ case 8: gbps = 12; break;
+ default:
+ LOG(ERROR, "unexpected active width %u on port %d",
+ ipa.active_width, port);
+ }
+ switch (ipa.active_speed) {
+ case 1: gbps = gbps*5/2; break;
+ case 2: gbps *= 5; break;
+ case 4:
+ case 8: gbps *= 10; break;
+ case 16: gbps *= 14; break;
+ case 32: gbps *= 25; break;
+ default:
+ LOG(ERROR, "unexpected active speed %u on port %d",
+ ipa.active_speed, port);
+ }
+ return gbps;
+}
+
 /**
  * Obtain the MTU enabled on this port to transmit and receive.
  *

src/Infiniband.h

@@ -456,8 +456,8 @@ class Infiniband {
 
  int
  getLid(int port);
- uint32_t
- getMtu(int port);
+ uint32_t getBandwidthGbps(int port);
+ uint32_t getMtu(int port);
 
  BufferDescriptor*
  tryReceive(QueuePair* qp, Tub<Address>* sourceAddress = NULL);