@@ -9,6 +9,7 @@ An SDK written in Rust for the [Inference Gateway](https://github.com/inference-
99 - [ Listing Models] ( #listing-models )
1010 - [ Listing Models from a specific provider] ( #listing-models-from-a-specific-provider )
1111 - [ Generating Content] ( #generating-content )
12+ - [ Streaming Content] ( #streaming-content )
1213 - [ Health Check] ( #health-check )
1314 - [ Contributing] ( #contributing )
1415 - [ License] ( #license )
@@ -169,6 +170,58 @@ log::info!("Generated response: {:?}", resp.response.role);
169170log :: info! (" Generated content: {:?}" , resp . response. content);
170171```
171172
173+ ### Streaming Content
174+
175+ You need to add the following tiny dependencies:
176+
177+ - ` futures-util ` for the ` StreamExt ` trait
178+
179+ ``` rust
180+ use inference_gateway_sdk :: {
181+ InferenceGatewayAPI ,
182+ InferenceGatewayClient , Message , MessageRole , Provider , ResponseContent
183+ };
184+ use futures_util :: {StreamExt , pin_mut};
185+ // ...rest of the imports
186+
187+ // ...main function
188+ let system_message = " You are an helpful assistent." . to_string ();
189+ let model = " deepseek-r1-distill-llama-70b" ;
190+ let messages = vec! [
191+ Message {
192+ role : MessageRole :: System ,
193+ content : system_message ,
194+ },
195+ Message {
196+ role : MessageRole :: User ,
197+ content : " Write a poem" . to_string (),
198+ },
199+ ];
200+ let client = InferenceGatewayClient :: new (" http://localhost:8080" );
201+ let stream = client . generate_content_stream (Provider :: Groq , model , messages );
202+ pin_mut! (stream );
203+ let content_delta = Some (" content-delta" . to_string ());
204+ // Iterate over the stream of Server Sent Events
205+ while let Some (ssevent ) = stream . next (). await {
206+ let resp = ssevent ? ;
207+
208+ // Only content-delta events contains the actual tokens
209+ // There are also events like:
210+ // - content-start
211+ // - content-end
212+ // - etc..
213+ if resp . event != content_delta {
214+ continue ;
215+ }
216+
217+ // Deserialize the event response
218+ let generate_response : ResponseContent = serde_json :: from_str (& resp . data)? ;
219+ // Print the token out as it's being sent from the server
220+ print! (" {}" , generate_response . content);
221+ }
222+ // ...rest of the main function
223+ ```
224+
172225### Health Check
173226
174227To check if the Inference Gateway is running, use the ` health_check ` method:
0 commit comments