From be98de1777c0a40ee998e912efb0d5d104e43b57 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Mon, 24 May 2021 10:53:14 +0800 Subject: [PATCH 1/4] Add plugin mechanism for QUIC agent. --- .../quic/plugins/QuicAgentPluginInterface.h | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 source/agent/addons/quic/plugins/QuicAgentPluginInterface.h diff --git a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h new file mode 100644 index 000000000..d8e2a8516 --- /dev/null +++ b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h @@ -0,0 +1,66 @@ +/* + * Copyright (C) 2021 Intel Corporation + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#ifndef QUIC_QUICAGENTPLUGININTERFACE_H_ +#define QUIC_QUICAGENTPLUGININTERFACE_H_ + +#include + +// Most classes of addons don't have a namespace. Because developers may need to include this file when develop plugins, we add a namespace for it to avoid potential conflicts. +namespace owt { +namespace quic_agent_plugin { + // Connection types for source and sink. + enum class ConnectionType { + kUnknown = 0, + // A WebTransport connection usually means a connection between client and server. + kWebTransport, + // An internal connection is a connection between different agents. + kInternalConnection, + }; + + // This struct represents a data flow's information. + // See https://github.com/open-webrtc-toolkit/owt-server/blob/master/doc/Client-Portal%20Protocol.md for definitions of publication and subscription. + struct PipelineInfo { + const ConnectionType publicationType; + const char* publicationId; + const ConnectionType subscriptionType; + const char* subscriptionId; + }; + + // A frame received of sent by QUIC agent. It's not a QUIC stream frame defined in QUIC spec. + struct Frame { + // Payload of the frame. + uint8_t* payload; + // Payload's length. + uint32_t length; + }; + + // Interface for data processors. + class ProcessorInterface { + public: + virtual ~ProcessorInterface() = default; + // Invoked when a new frame is available. `frame` will not be sent back to the pipeline. If you need the frame to be delivered to the next node, please call `FrameSink`'s `deliverFrame` method explicitly. + virtual void OnFrame(const Frame& frame); + }; + + class FrameSink { + public: + virtual ~FrameSink() = default; + // Call this function when a new frame is ready to be delivered to the next node. + virtual void DeliverFrame(const Frame& frame); + }; + + // The interface for QUIC agent plugins. It allows developers to process data received or sent by QUIC agent. + class PluginInterface { + public: + virtual ~PluginInterface() = default; + // Create a data processor for a newly created pipeline. This method will be invoked when a new pipeline is created in QUIC agent. Returns nullptr if you don't need a processor for this pipeline. Since you have a reference to `frameSink`, you can push data to the next node whenever you want. + virtual ProcessorInterface* CreateDataProcessor(const PipelineInfo& info, const FrameSink& frameSink); + }; +} +} + +#endif \ No newline at end of file From fb977cab89c70c277371b607ace51c4c87aeb6a9 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Tue, 25 May 2021 14:50:35 +0800 Subject: [PATCH 2/4] Add doc for QUIC agent plugin. --- doc/servermd/QuicAgentPluginDataflow.svg | 185 ++++++++++++++++++ doc/servermd/QuicAgentPluginGuide.md | 13 ++ .../quic/plugins/QuicAgentPluginInterface.h | 8 + source/agent/quic/agent.toml | 5 +- 4 files changed, 210 insertions(+), 1 deletion(-) create mode 100644 doc/servermd/QuicAgentPluginDataflow.svg create mode 100644 doc/servermd/QuicAgentPluginGuide.md diff --git a/doc/servermd/QuicAgentPluginDataflow.svg b/doc/servermd/QuicAgentPluginDataflow.svg new file mode 100644 index 000000000..eb963a4eb --- /dev/null +++ b/doc/servermd/QuicAgentPluginDataflow.svg @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + Page-1 + + + + Rectangle + QUIC agent plugin + + + + + + + QUIC agent plugin + + Simple Arrow + Publication or internal IO input + + + + + + + + Publication or internal IO input + + Rectangle.5 + Data processor + + + + + + + Data processor + + Major PipelineR + + + + + + + + + + + + + + + + + + + + + + + + + + Rectangle.15 + Another agent + + + + + + + Another agent + + Simple Arrow.18 + Internal IO + + + + + + + + Internal IO + + Rectangle.19 + Data processor + + + + + + + Data processor + + Rectangle.20 + WebTransport client + + + + + + + WebTransport client + + Simple Arrow.21 + WebTransport + + + + + + + + WebTransport + + Major PipelineR.22 + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/servermd/QuicAgentPluginGuide.md b/doc/servermd/QuicAgentPluginGuide.md new file mode 100644 index 000000000..ce8732a12 --- /dev/null +++ b/doc/servermd/QuicAgentPluginGuide.md @@ -0,0 +1,13 @@ +# Introduction +QUIC agent plays the role as a WebTransport server. It's intended to establish WebTransport connections with clients, send and receive arbitrary data to and from clients. The data received from client can be piped to another agent for audio or video processing, or simply forward to another clients. QUIC agent plugin mechanism allows developers to process data before it's delivered to the next node. + +# Dataflow +The picture below shows how data flows for a single publication when QUIC agent plugin is enabled. + +![QUIC agent plugin dataflow](QuicAgentPluginDataflow.svg) + +# Development +To develop a QUIC agent plugin, please implement the interface defined in `source/agent/addons/quic/plugins/QuicAgentPluginInterface.h` and compile your code as a shared library. It's highly recommend to compile your plugin with the same toolchain as OWT conference server, although sometimes it works even they are compiled with different toolchains. + +# Deployment +To deploy your plugin to a QUIC agent, please update the value of `pluginPath` in QUIC agent's agent.toml. It will take effect next time QUIC agent boots. \ No newline at end of file diff --git a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h index d8e2a8516..3ca839100 100644 --- a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h +++ b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h @@ -4,6 +4,14 @@ * SPDX-License-Identifier: Apache-2.0 */ +/* + * A plugin instance (PluginInterface) is created when the QUIC agent is + * started. A data processor (ProcessorInterface) is created when a new pipeline + * (publication - subscription pair) is created. + * + * See doc/servermd/QuicAgentPluginGuide.md for detailed information. + */ + #ifndef QUIC_QUICAGENTPLUGININTERFACE_H_ #define QUIC_QUICAGENTPLUGININTERFACE_H_ diff --git a/source/agent/quic/agent.toml b/source/agent/quic/agent.toml index 58e82df2a..e76dd771d 100644 --- a/source/agent/quic/agent.toml +++ b/source/agent/quic/agent.toml @@ -55,4 +55,7 @@ keystorePath = "./cert/certificate.pfx" port = 7700 #default: 7700 # FQDN of QUIC agent. It's included in WebTransport tokens as a part of the WebTransport URL client connects to. IP address will be included in WebTransport tokens if hostname is empty. -hostname = "" \ No newline at end of file +hostname = "" + +# Path to QUIC agent plugin. Plugin is enabled only when the value is not an empty string. Default: "". +pluginPath = "" \ No newline at end of file From 708bbd0c56e064e0aa2fccdaa8c6f4f12b1596a6 Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Wed, 26 May 2021 15:35:29 +0800 Subject: [PATCH 3/4] Add creation and deletion functions for Frame and clarify its ownership. --- .../quic/plugins/QuicAgentPluginInterface.h | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h index 3ca839100..4060709ff 100644 --- a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h +++ b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h @@ -40,25 +40,33 @@ namespace quic_agent_plugin { // A frame received of sent by QUIC agent. It's not a QUIC stream frame defined in QUIC spec. struct Frame { + // Create a frame. + static Frame* Create(); + // Delete the frame. + void Dispose(); // Payload of the frame. uint8_t* payload; // Payload's length. uint32_t length; + + private: + Frame(); + virtual ~Frame(); }; // Interface for data processors. class ProcessorInterface { public: virtual ~ProcessorInterface() = default; - // Invoked when a new frame is available. `frame` will not be sent back to the pipeline. If you need the frame to be delivered to the next node, please call `FrameSink`'s `deliverFrame` method explicitly. - virtual void OnFrame(const Frame& frame); + // Invoked when a new frame is available. Ownership of `frame` is moved to processor. `frame` will not be sent back to the pipeline. If you need the frame to be delivered to the next node, please call `FrameSink`'s `deliverFrame` method explicitly. + virtual void OnFrame(Frame* frame); }; class FrameSink { public: virtual ~FrameSink() = default; - // Call this function when a new frame is ready to be delivered to the next node. - virtual void DeliverFrame(const Frame& frame); + // Call this function when a new frame is ready to be delivered to the next node. Ownership of `frame` is moved to the next node. + virtual void DeliverFrame(Frame* frame); }; // The interface for QUIC agent plugins. It allows developers to process data received or sent by QUIC agent. From ca97b45b0b7bd161abd81c0eb2f1cb63b4d84d8d Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Thu, 27 May 2021 16:41:36 +0800 Subject: [PATCH 4/4] Processor is created per publication instead of per pipeline. --- doc/servermd/QuicAgentPluginDataflow.svg | 217 +++++++++--------- .../quic/plugins/QuicAgentPluginInterface.h | 29 ++- 2 files changed, 127 insertions(+), 119 deletions(-) diff --git a/doc/servermd/QuicAgentPluginDataflow.svg b/doc/servermd/QuicAgentPluginDataflow.svg index eb963a4eb..32d7f4fad 100644 --- a/doc/servermd/QuicAgentPluginDataflow.svg +++ b/doc/servermd/QuicAgentPluginDataflow.svg @@ -2,20 +2,24 @@ - + xmlns:v="http://schemas.microsoft.com/visio/2003/SVGExtensions/" width="11in" height="8.5in" viewBox="0 0 792 612" + xml:space="preserve" color-interpolation-filters="sRGB" class="st7"> + + + + + + @@ -27,159 +31,148 @@ - - + + - - + + + Page-1 - - Rectangle - QUIC agent plugin + + + Rectangle.1001 + QUIC agent - - - - QUIC agent plugin - - Simple Arrow - Publication or internal IO input + + + + QUIC agent + + Rectangle + QUIC agent plugin - - - - - Publication or internal IO input - + + + + QUIC agent plugin + Rectangle.5 Data processor - - - Data + + Data processor - - Major PipelineR - - - - - - - - + + Rectangle.15 + Another agent - - - - - - - - - - - - - - - Rectangle.15 - Another agent + + + + Another agent + + Rectangle.20 + WebTransport client - - - Another agent - - Simple Arrow.18 - Internal IO + + + WebTransport client + + Sheet.1002 + + + + Rectangle.1003 + Client - - - - Internal IO - - Rectangle.19 - Data processor + + + Client + + Rectangle.1004 + Other agent - - - Data processor - - Rectangle.20 - WebTransport client + + + Other agent + + Rectangle.1005 + Data processor - - - WebTransport client - - Simple Arrow.21 - WebTransport + + + Data processor + + Sheet.1006 + + + + Sheet.1007 + + + + Sheet.1008 + + + + Rectangle.1009 + Another agent - - - - WebTransport - - Major PipelineR.22 - - - - - - - - + + + Another agent + + Rectangle.1010 + WebTransport client - - - - - - - - - - - - + + + + WebTransport client + + Sheet.1011 + + + + Sheet.1012 + diff --git a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h index 4060709ff..3c446516e 100644 --- a/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h +++ b/source/agent/addons/quic/plugins/QuicAgentPluginInterface.h @@ -31,11 +31,10 @@ namespace quic_agent_plugin { // This struct represents a data flow's information. // See https://github.com/open-webrtc-toolkit/owt-server/blob/master/doc/Client-Portal%20Protocol.md for definitions of publication and subscription. - struct PipelineInfo { - const ConnectionType publicationType; - const char* publicationId; - const ConnectionType subscriptionType; - const char* subscriptionId; + struct ConnectionInfo { + const ConnectionType type; + // Publication ID or subscription ID. + const char* id; }; // A frame received of sent by QUIC agent. It's not a QUIC stream frame defined in QUIC spec. @@ -58,8 +57,21 @@ namespace quic_agent_plugin { class ProcessorInterface { public: virtual ~ProcessorInterface() = default; + // Invoked when a new subscription is requested for the publication associated with this processor. + virtual void AddSink(const FrameSink& sink); // Invoked when a new frame is available. Ownership of `frame` is moved to processor. `frame` will not be sent back to the pipeline. If you need the frame to be delivered to the next node, please call `FrameSink`'s `deliverFrame` method explicitly. virtual void OnFrame(Frame* frame); + // Invoked when a new feedback is available. Ownership of `feedback` is moved to processor. `feedback` will not be sent back to the pipeline. If you need the frame to be delivered to the next node, please call `FrameSource`'s `deliverFeedback` method explicitly. + virtual void OnFeedback(Frame* feedback); + }; + + class FrameSource { + public: + virtual ~FrameSource() = default; + // Call this function when a new feedback is ready to be delivered to the next node. Ownership of `feedback` is moved to the next node. + virtual void DeliverFeedback(Frame* feedback); + // Get publication infomation. + virtual const ConnectionInfo& ConnectionInfo() const; }; class FrameSink { @@ -67,14 +79,17 @@ namespace quic_agent_plugin { virtual ~FrameSink() = default; // Call this function when a new frame is ready to be delivered to the next node. Ownership of `frame` is moved to the next node. virtual void DeliverFrame(Frame* frame); + // Get subscription infomation. + virtual const ConnectionInfo& ConnectionInfo() const; }; // The interface for QUIC agent plugins. It allows developers to process data received or sent by QUIC agent. class PluginInterface { public: virtual ~PluginInterface() = default; - // Create a data processor for a newly created pipeline. This method will be invoked when a new pipeline is created in QUIC agent. Returns nullptr if you don't need a processor for this pipeline. Since you have a reference to `frameSink`, you can push data to the next node whenever you want. - virtual ProcessorInterface* CreateDataProcessor(const PipelineInfo& info, const FrameSink& frameSink); + // Create a data processor for a newly created data source. This method will be invoked when a new data source is created in QUIC agent. Returns nullptr if you don't need a processor for this source. + virtual ProcessorInterface* CreateDataProcessor(const FrameSource& source); + // Other methods for loading a shared library. }; } }