completely revamp examples

Author: drunkirishcoder

Cargo.toml

@@ -6,7 +6,6 @@ members = [
     "examples/nat64",
     "examples/ping4d",
     "examples/pktdump",
-    "examples/signals",
     "examples/skeleton",
     "examples/syn-flood",
     "ffi",

core/src/net/mac.rs

@@ -32,7 +32,7 @@ impl MacAddr {
 
     /// Creates a MAC address from 6 octets.
     #[allow(clippy::many_single_char_names)]
-    pub fn new(a: u8, b: u8, c: u8, d: u8, e: u8, f: u8) -> Self {
+    pub const fn new(a: u8, b: u8, c: u8, d: u8, e: u8, f: u8) -> Self {
         MacAddr([a, b, c, d, e, f])
     }
 

core/src/packets/mod.rs

@@ -30,8 +30,8 @@ mod udp;
 pub use self::ethernet::*;
 pub use self::tcp::*;
 pub use self::udp::*;
+pub use crate::dpdk::Mbuf;
 
-use crate::Mbuf;
 use anyhow::{Context, Result};
 use std::fmt;
 use std::marker::PhantomData;
@@ -336,6 +336,20 @@ impl<T> Deref for Immutable<'_, T> {
     }
 }
 
+/// Mark of the packet as either `Emit` or `Drop`.
+///
+/// Together, a `Result<Postmark>` represents all three possible outcome
+/// of packet processing. A packet can either be emitted through port TX,
+/// intentionally dropped, or aborted due to an error.
+#[derive(Debug)]
+pub enum Postmark {
+    /// Packet emitted through a port TX.
+    Emit,
+
+    /// Packet intentionally dropped.
+    Drop(Mbuf),
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;

core/src/packets/udp.rs

@@ -146,6 +146,19 @@ impl<E: IpPacket> Udp<E> {
         self.header_mut().checksum = u16be::default();
     }
 
+    /// Returns the data as a `u8` slice.
+    #[inline]
+    pub fn data(&self) -> &[u8] {
+        if let Ok(data) = self
+            .mbuf()
+            .read_data_slice(self.payload_offset(), self.payload_len())
+        {
+            unsafe { &*data.as_ptr() }
+        } else {
+            unreachable!()
+        }
+    }
+
     /// Returns the 5-tuple that uniquely identifies a UDP connection.
     #[inline]
     pub fn flow(&self) -> Flow {
@@ -394,6 +407,7 @@ mod tests {
         assert_eq!(1087, udp.dst_port());
         assert_eq!(18, udp.length());
         assert_eq!(0x7228, udp.checksum());
+        assert_eq!(10, udp.data().len());
     }
 
     #[capsule::test]

core/src/rt2/lcore.rs

@@ -23,12 +23,13 @@ use anyhow::Result;
 use async_executor::Executor;
 use futures_lite::future;
 use std::collections::HashMap;
+use std::fmt;
 use std::future::Future;
 use std::sync::Arc;
 use thiserror::Error;
 
 /// An async executor abstraction on top of a DPDK logical core.
-pub(crate) struct Lcore {
+pub struct Lcore {
     id: LcoreId,
     executor: Arc<Executor<'static>>,
     shutdown: Option<ShutdownTrigger>,
@@ -75,11 +76,17 @@ impl Lcore {
     }
 
     /// Spawns a background async task.
-    pub(crate) fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
+    pub fn spawn(&self, future: impl Future<Output = ()> + Send + 'static) {
         self.executor.spawn(future).detach();
     }
 }
 
+impl fmt::Debug for Lcore {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        f.debug_struct("Lcore").field("id", &self.id()).finish()
+    }
+}
+
 impl Drop for Lcore {
     fn drop(&mut self) {
         if let Some(trigger) = self.shutdown.take() {
@@ -92,19 +99,20 @@ impl Drop for Lcore {
 /// Lcore not found error.
 #[derive(Debug, Error)]
 #[error("lcore not found.")]
-pub(crate) struct LcoreNotFound;
+pub struct LcoreNotFound;
 
 /// Map to lookup the lcore by the assigned id.
-pub(crate) struct LcoreMap(HashMap<usize, Lcore>);
+#[derive(Debug)]
+pub struct LcoreMap(HashMap<usize, Lcore>);
 
 impl LcoreMap {
     /// Returns the lcore with the assigned id.
-    pub(crate) fn get(&self, id: usize) -> Result<&Lcore> {
+    pub fn get(&self, id: usize) -> Result<&Lcore> {
         self.0.get(&id).ok_or_else(|| LcoreNotFound.into())
     }
 
     /// Returns a lcore iterator.
-    pub(crate) fn iter(&self) -> impl Iterator<Item = &Lcore> {
+    pub fn iter(&self) -> impl Iterator<Item = &Lcore> {
         self.0.values()
     }
 }

core/src/rt2/mempool.rs

@@ -26,10 +26,8 @@ use std::ptr::{self, NonNull};
 use thiserror::Error;
 
 /// A memory pool is an allocator of message buffers, or `Mbuf`. For best
-/// performance, each socket should have a dedicated `Mempool`. However,
-/// for simplicity, we currently only support one global Mempool. Multi-
-/// socket support may be added in the future.
-pub(crate) struct Mempool {
+/// performance, each socket should have a dedicated `Mempool`.
+pub struct Mempool {
     ptr: MempoolPtr,
 }
 
@@ -75,13 +73,13 @@ impl Mempool {
 
     /// Returns the maximum number of Mbufs in the pool.
     #[inline]
-    pub(crate) fn capacity(&self) -> usize {
+    pub fn capacity(&self) -> usize {
         self.ptr.size as usize
     }
 
     /// Returns the per core cache size.
     #[inline]
-    pub(crate) fn cache_size(&self) -> usize {
+    pub fn cache_size(&self) -> usize {
         self.ptr.cache_size as usize
     }
 

core/src/rt2/mod.rs

@@ -16,18 +16,22 @@
 * SPDX-License-Identifier: Apache-2.0
 */
 
+//! Capsule runtime.
+
 mod config;
 mod lcore;
 mod mempool;
 mod port;
 
 pub use self::config::*;
 pub(crate) use self::lcore::*;
+pub use self::lcore::{Lcore, LcoreMap, LcoreNotFound};
+pub use self::mempool::Mempool;
 pub(crate) use self::mempool::*;
 pub use self::port::{Outbox, Port, PortError, PortMap};
-pub use crate::dpdk::Mbuf;
 
 use crate::ffi::dpdk::{self, LcoreId};
+use crate::packets::{Mbuf, Postmark};
 use crate::{debug, info};
 use anyhow::Result;
 use async_channel::{self, Receiver, Sender};
@@ -88,6 +92,19 @@ pub struct Runtime {
 }
 
 impl Runtime {
+    /// Returns the mempool.
+    ///
+    /// For simplicity, we currently only support one global Mempool. Multi-
+    /// socket support may be added in the future.
+    pub fn mempool(&self) -> &Mempool {
+        &self.mempool
+    }
+
+    /// Returns the lcores.
+    pub fn lcores(&self) -> &LcoreMap {
+        &self.lcores
+    }
+
     /// Returns the configured ports.
     pub fn ports(&self) -> &PortMap {
         &self.ports
@@ -151,7 +168,7 @@ impl Runtime {
     /// Sets the packet processing pipeline for port.
     pub fn set_port_pipeline<F>(&self, port: &str, f: F) -> Result<()>
     where
-        F: Fn(Mbuf) -> Result<()> + Clone + Send + Sync + 'static,
+        F: Fn(Mbuf) -> Result<Postmark> + Clone + Send + Sync + 'static,
     {
         let port = self.ports.get(port)?;
         port.spawn_rx_loops(f, &self.lcores)?;

core/src/rt2/port.rs

@@ -16,10 +16,10 @@
 * SPDX-License-Identifier: Apache-2.0
 */
 
-use super::{LcoreMap, Mbuf, Mempool, ShutdownTrigger};
+use super::{LcoreMap, Mempool, ShutdownTrigger};
 use crate::ffi::dpdk::{self, LcoreId, MbufPtr, PortId, PortRxQueueId, PortTxQueueId};
 use crate::net::MacAddr;
-use crate::packets::Packet;
+use crate::packets::{Mbuf, Packet, Postmark};
 use crate::{debug, ensure, info, warn};
 use anyhow::Result;
 use async_channel::{self, Receiver, Sender};
@@ -96,7 +96,7 @@ impl Port {
     /// Spawns the port receiving loop.
     pub(crate) fn spawn_rx_loops<F>(&self, f: F, lcores: &LcoreMap) -> Result<()>
     where
-        F: Fn(Mbuf) -> Result<()> + Clone + Send + Sync + 'static,
+        F: Fn(Mbuf) -> Result<Postmark> + Clone + Send + Sync + 'static,
     {
         // port is built with the builder, this would not panic.
         let shutdown = self.shutdown.as_ref().unwrap();
@@ -231,7 +231,7 @@ async fn rx_loop<F>(
     batch_size: usize,
     f: F,
 ) where
-    F: Fn(Mbuf) -> Result<()> + Send + Sync + 'static,
+    F: Fn(Mbuf) -> Result<Postmark> + Send + Sync + 'static,
 {
     debug!(port = ?port_name, lcore = ?LcoreId::current(), "executing rx loop.");
 
@@ -240,8 +240,18 @@ async fn rx_loop<F>(
 
     loop {
         rxq.receive(&mut ptrs);
+        let mut drops = vec![];
+
         for ptr in ptrs.drain(..) {
-            let _ = f(Mbuf::from_easyptr(ptr));
+            match f(Mbuf::from_easyptr(ptr)) {
+                Ok(Postmark::Emit) => (),
+                Ok(Postmark::Drop(ptr)) => drops.push(ptr),
+                Err(_) => (),
+            }
+        }
+
+        if !drops.is_empty() {
+            Mbuf::free_bulk(drops);
         }
 
         // cooperatively moves to the back of the execution queue,
@@ -325,7 +335,7 @@ impl PortMap {
     }
 
     /// Returns a port iterator.
-    pub(crate) fn iter(&self) -> impl Iterator<Item = &Port> {
+    pub fn iter(&self) -> impl Iterator<Item = &Port> {
         self.0.values()
     }
 

examples/kni/Cargo.toml

@@ -18,7 +18,7 @@ doctest = false
 [dependencies]
 anyhow = "1.0"
 capsule = { version = "0.1", path = "../../core" }
-metrics-core = "0.5"
-metrics-observer-yaml = "0.1"
+colored = "2.0"
+signal-hook = "0.3"
 tracing = "0.1"
 tracing-subscriber = "0.2"

examples/kni/README.md

@@ -1,19 +1,21 @@
 # Kernel NIC interface example
 
-The Kernel NIC Interface (KNI) is a DPDK control plane solution that allows userspace applications to exchange packets with the Linux kernel networking stack. See DPDK's [KNI documentation](https://doc.dpdk.org/guides/prog_guide/kernel_nic_interface.html) for more information. This example is a minimum program that can forward packets to and from the Linux kernel.
+The Kernel NIC Interface (KNI) is a DPDK control plane solution that allows userspace applications to exchange packets with the Linux kernel networking stack. See DPDK's [KNI documentation](https://doc.dpdk.org/guides/prog_guide/kernel_nic_interface.html) for more information.
 
 ## Overview
 
-KNI is useful for applications that want to conceptually share the port with the Linux kernel. For example, the application may want to leverage the kernel's built-in ability to handle [ARP](https://tools.ietf.org/html/rfc826) traffic instead of implementing the protocol natively. By enabling KNI for a port, a virtual device with the same name and MAC address as the port is exposed to the Linux kernel. The kernel will be able to receive all packets that are forwarded to this virtual device and the application will receive all packets the kernel sends to it.
+KNI is useful for Capsule applications that want to delegate the processing of various network control plane protocols to either the Linux kernel or other implementations that run on Linux. For example, [Address Resolution Protocol](https://tools.ietf.org/html/rfc826) is the mechanism for hosts to discover each other's link layer address on an IPv4 network. Typically, the Linux kernel handles the ARP discovery for all the network interfaces on the host. However, because Capsule-bound network devices are not visible to the kernel, each Capsule application needs its own ARP implementation, otherwise the network won't be able to route packets to it. Or alternatively, an easier approach is for the application to simply leverage the kernel stack implementation by delegating and forwarding ARP packets via KNI.
+
+This example demonstrates said approach by delegating the processing of [Neighbor Discovery Procotol](https://tools.ietf.org/html/rfc4861), the IPv6 equivalent of ARP, to the Linux kernel.
 
 ## Prerequisite
 
-This application requires the kernel module `rte_kni`. Kernel modules are version specific. If you are using our `Vagrant` with `Docker` setup, the module is already preloaded. Otherwise, you will have to compile it by installing the kernel headers or sources required to build kernel modules on your system, then [build `DPDK` from source](https://doc.dpdk.org/guides/linux_gsg/build_dpdk.html).
+This application requires the kernel module `rte_kni`. Kernel modules are version specific. If you are using our Vagrant with Docker setup, the module is already preloaded. Otherwise, you will have to compile it by installing the kernel headers or sources required to build kernel modules on your system, then [build `DPDK` from source](https://doc.dpdk.org/guides/linux_gsg/build_dpdk.html).
 
 Once the build is complete, load the module with command:
 
-```
-$ sudo insmod /lib/modules/`uname -r`/extra/dpdk/rte_kni.ko
+```bash
+$ sudo insmod /lib/modules/`uname -r`/extra/dpdk/rte_kni.ko carrier=on
 ```
 
 We may provide precompiled modules for different kernel versions and Linux distributions in the future. 
@@ -22,26 +24,65 @@ We may provide precompiled modules for different kernel versions and Linux distr
 
 The example is located in the `examples/kni` sub-directory. To run the application,
 
-```
+```bash
 /examples/kni$ cargo run -- -f kni.toml
 ```
 
-While the application is running, the new virtual device is exposed to the kernel,
+While the application is running, in a seperate Vagrant VM terminal, check that a new virtual device `kni0` is exposed to the kernel,
 
+```bash
+vagrant$ ip link show dev kni0
+
+14: kni0: <BROADCAST,MULTICAST> mtu 2034 qdisc noop state DOWN mode DEFAULT group default qlen 1000
+    link/ether 6a:80:63:c3:01:42 brd ff:ff:ff:ff:ff:ff
 ```
-$ ip link
 
-254: kni0: <BROADCAST,MULTICAST> mtu 1500 qdisc noop state DOWN mode DEFAULT group default qlen 1000
-    link/ether ba:dc:af:eb:ee:f1 brd ff:ff:ff:ff:ff:ff
+Change the MAC address of `kni0` to match the MAC address of the physical interface first; then bring up the link,
+
+```bash
+vagrant$ sudo ip link set dev kni0 address 02:00:00:ff:ff:00
+vagrant$ sudo ip link set dev kni0 up
 ```
 
-The kernel can assign an IP address to the device and bring the link up, at which point the kernel and the application can forward each other packets.
+Once `kni0` is up, it should be automatically assigned an IPv6 address, we will need this address for the next step,
 
+```bash
+vagrant$ ip addr show dev kni0
+
+14: kni0: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 2034 qdisc pfifo_fast state UP group default qlen 1000
+    link/ether 02:00:00:ff:ff:00 brd ff:ff:ff:ff:ff:ff
+    inet6 fe80::ff:feff:ff00/64 scope link
+       valid_lft forever preferred_lft forever
 ```
-$ sudo ip addr add dev kni0 10.0.2.16/24
-$ sudo ip link set up dev kni0
+
+Finally use `socat` to read from `stdin`, and send the input messages as UDP packets to this IPv6 address. These packets will be routed to the running Capsule application,
+
+```bash
+vagrant$ socat -d -d -u - udp6:[fe80::ff:feff:ff00%eth3]:6667
+
+Hello?
+Is there anybody in there?
+```
+
+The running application should print out,
+
+```
+Mar 28 19:59:34.805  INFO kni: to kni0: Neighbor Solicitation
+Mar 28 19:59:34.810  INFO kni: from kni0: Neighbor Advertisement
+Mar 28 19:59:34.811  INFO kni: you said: Hello?
+
+Mar 28 19:59:39.475  INFO kni: you said: Is there anybody in there?
+
 ```
 
 # Explanation
 
-The assigned port `0000:00:08.0` has KNI support turned on by setting the `kni` flag to `true`. To forward packets received on the port to the kernel, the application adds a simple forwarding pipeline by calling `add_pipeline_to_port`. To forward packets received from the kernel through the port, the application adds another forwarding pipeline by calling `add_kni_rx_pipeline_to_port`.
+Capsule leverages the KNI poll mode driver instead of the `librte_kni` API directly. This lets the application to interact with KNI the same way as any other physical or virtual port device.
+
+The example is configured with one PCI port, `cap0`, and one KNI port, `kni0`. As new packets arrive through `cap0`'s rx, the application will forward all ICMPv6 packets to the `kni0`'s tx. For the sake of simplicity, it is assumed that all ICMPv6 packets received in this example will be NDP messages, and the application is delegating this link layer address discovery process to the kernel stack. In the reverse direction, kernel stack's NDP responses will come in through `kni0`'s rx, and immediately forwarded out through `cap0`'s tx without modifications. Because we assigned `cap0`'s link layer MAC address, `02:00:00:ff:ff:00`, to `kni0` with the `ip link set` command, the NDP responses already contain the correct link layer information.
+
+When `socat` sends out an UDP packet via `eth3` to `kni0`'s IPv6 address `fe80::ff:feff:ff00`, a lookup is performed trying to find the link layer address of the destination. On the very first attempt, that link layer address is not found through the lookup. A neighbor solicitation message is broadcasted instead to initiate the discovery process.
+
+`cap0` receives the broadcasted neighbor solicitation message and forwards it to the kernel stack via `kni0`. Kernel responds with a neighbor advertisement message because the IPv6 address matches the address of the `kni0` interface. This response is sent back to `eth3` through `kni0`'s rx then `cap0`'s tx, completing the discovery.
+
+The link layer address from the response is cached, all UDP packets are routed to `cap0` with this lookup until the cached entry expires. The Capsule application will receive the UDP packets from `socat`. It parses and prints out the data payload.