[Lguest] [PATCH 2/2] lguest: Implement the merged receive buffers allocation scheme

[Mark McLoughlin]

Implement support for using receive buffers less than the maximum
packet size by merging the smaller buffers together as needed.

Signed-off-by: Mark McLoughlin <markmc at redhat.com>
Cc: Herbert Xu <herbert.xu at redhat.com>

---
 Documentation/lguest/lguest.c |   98 +++++++++++++++++++++++++++++++++-------
 1 files changed, 81 insertions(+), 17 deletions(-)

diff --git a/Documentation/lguest/lguest.c b/Documentation/lguest/lguest.c
index 2ddf0c6..da934c2 100644
--- a/Documentation/lguest/lguest.c
+++ b/Documentation/lguest/lguest.c
@@ -68,6 +68,8 @@ typedef uint8_t u8;
 #define DEVICE_PAGES 256
 /* This will occupy 3 pages: it must be a power of 2. */
 #define VIRTQUEUE_NUM 256
+/* The largest MAX_SKB_FRAGS pages supported. */
+#define NET_MAX_RECV_PAGES 18
 
 /*L:120 verbose is both a global flag and a macro.  The C preprocessor allows
  * this, and although I wouldn't recommend it, it works quite nicely here. */
@@ -970,16 +972,15 @@ static void handle_net_output(int fd, struct virtqueue *vq, bool timeout)
 	}
 }
 
-/* This is where we handle a packet coming in from the tun device to our
- * Guest. */
-static bool handle_tun_input(int fd, struct device *dev)
+/* Network receive buffers are odd creatures; the Guest usually gives us single
+ * page buffers and - if we're handling big packets - we use multiple buffers per
+ * packet. */
+static unsigned int get_net_recv_head(struct device *dev, struct iovec *iov,
+				      unsigned int *in_num)
 {
-	unsigned int head, in_num, out_num;
-	int len;
-	struct iovec iov[dev->vq->vring.num];
+	unsigned int head, out_num;
 
-	/* First we need a network buffer from the Guests's recv virtqueue. */
-	head = get_vq_desc(dev->vq, iov, &out_num, &in_num);
+	head = get_vq_desc(dev->vq, iov, &out_num, in_num);
 	if (head == dev->vq->vring.num) {
 		/* Now, it's expected that if we try to send a packet too
 		 * early, the Guest won't be ready yet.  Wait until the device
@@ -989,23 +990,85 @@ static bool handle_tun_input(int fd, struct device *dev)
 		/* Now tell it we want to know if new things appear. */
 		dev->vq->vring.used->flags &= ~VRING_USED_F_NO_NOTIFY;
 		wmb();
-
-		/* We'll turn this back on if input buffers are registered. */
-		return false;
 	} else if (out_num)
 		errx(1, "Output buffers in network recv queue?");
 
+	return head;
+}
+
+/* Here we add used recv buffers to the used queue but, also, return unused
+ * buffers to the avail queue. */
+static void add_net_recv_used(struct device *dev, unsigned int *heads,
+			      int *bufsizes, int nheads, int used_len)
+{
+	int len, idx;
+
+	/* Add the buffers we've actually used to the used queue */
+	len = idx = 0;
+	while (len < used_len) {
+		add_used(dev->vq, heads[idx], used_len, idx);
+		len += bufsizes[idx++];
+	}
+
+	/* Return the rest of them back to the avail queue */
+	lg_last_avail(dev->vq) -= nheads - idx;
+	dev->vq->inflight      -= nheads - idx;
+
+	/* Finally, tell the other side */
+	flush_used(dev->vq, idx);
+}
+
+/* This is where we handle a packet coming in from the tun device to our
+ * Guest. */
+static bool handle_tun_input(int fd, struct device *dev)
+{
+	struct iovec iov[dev->vq->vring.num];
+	unsigned int heads[NET_MAX_RECV_PAGES];
+	int bufsizes[NET_MAX_RECV_PAGES];
+	int nheads, len, iovcnt;
+
+	nheads = len = iovcnt = 0;
+
+	/* First we need enough network buffers from the Guests's recv
+	 * virtqueue for the largest possible packet. */
+	while (len < (getpagesize() * NET_MAX_RECV_PAGES)) {
+		unsigned int in_num;
+
+		heads[nheads] = get_net_recv_head(dev, &iov[iovcnt], &in_num);
+		if (heads[nheads] == dev->vq->vring.num) {
+			/* It's all our nothing - give these back again */
+			lg_last_avail(dev->vq) -= nheads;
+			dev->vq->inflight      -= nheads;
+
+			/* We'll turn this back on if input buffers are
+			 * registered. */
+			return false;
+		}
+
+		bufsizes[nheads] = 0;
+		while (in_num--)
+			bufsizes[nheads] += iov[iovcnt++].iov_len;
+
+		/* Enforce a minimum of a page per buffer */
+		if (bufsizes[nheads] < getpagesize())
+			errx(1, "Network receive buffer too small");
+
+		len += bufsizes[nheads++];
+	}
+
 	/* Read the packet from the device directly into the Guest's buffer. */
-	len = readv(dev->fd, iov, in_num);
+	len = readv(dev->fd, iov, iovcnt);
 	if (len <= 0)
 		err(1, "reading network");
 
-	/* Tell the Guest about the new packet. */
-	add_used_and_trigger(fd, dev->vq, head, len);
+	/* Return unused buffers to the recv queue */
+	add_net_recv_used(dev, heads, bufsizes, nheads, len);
+
+	/* Fire in the hole ! */
+	trigger_irq(fd, dev->vq);
 
-	verbose("tun input packet len %i [%02x %02x] (%s)\n", len,
-		((u8 *)iov[1].iov_base)[0], ((u8 *)iov[1].iov_base)[1],
-		head != dev->vq->vring.num ? "sent" : "discarded");
+	verbose("tun input packet len %i [%02x %02x] sent\n", len,
+		((u8 *)iov[0].iov_base)[10], ((u8 *)iov[0].iov_base)[11]);
 
 	/* All good. */
 	return true;
@@ -1561,6 +1624,7 @@ static void setup_tun_net(char *arg)
 	add_feature(dev, VIRTIO_NET_F_HOST_TSO4);
 	add_feature(dev, VIRTIO_NET_F_HOST_TSO6);
 	add_feature(dev, VIRTIO_NET_F_HOST_ECN);
+	add_feature(dev, VIRTIO_NET_F_MRG_RXBUF);
 	set_config(dev, sizeof(conf), &conf);
 
 	/* We don't need the socket any more; setup is done. */
-- 
1.5.4.3