From ed8a66f432dfbdfc1e31d6904b7323eab0e5c98f Mon Sep 17 00:00:00 2001 From: Hadi Khazaee Asl Date: Sat, 26 Sep 2026 10:46:14 +0330 Subject: [PATCH] refactor for better support of streaming and etc ... --- Controllers/XAiServiceControllerBase.cs | 90 +++++++++++++++++++++- Interfaces/IXAiEmbeddingServiceBase.cs | 4 +- Interfaces/IXAiServiceBase.cs | 4 +- Interfaces/IXAiServiceControllerBase.cs | 16 +++- Providers/XAIServiceBase.cs | 55 +++----------- Providers/XAiEmbeddingServiceBase.cs | 99 +++++-------------------- 6 files changed, 132 insertions(+), 136 deletions(-) diff --git a/Controllers/XAiServiceControllerBase.cs b/Controllers/XAiServiceControllerBase.cs index ce7d8a3..445ee06 100644 --- a/Controllers/XAiServiceControllerBase.cs +++ b/Controllers/XAiServiceControllerBase.cs @@ -1,7 +1,9 @@ using System; +using System.Text.Json; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.Extensions.Logging; using xAiApi.Interfaces; @@ -12,7 +14,7 @@ using xCommons.Providers; namespace xAiApi.Controllers { [Authorize] - [Route ("[controller]")] + [Route("[controller]")] public abstract class XAiServiceControllerBase : XBaseController, IXAiServiceControllerBase where TAiService : IXAiServiceBase { @@ -39,8 +41,8 @@ namespace xAiApi.Controllers /// /// [AllowAnonymous] - [HttpGet("Ask")] - public virtual async Task> Ask( + [HttpGet("AskText")] + public virtual async Task> AskText( [FromQuery] string question, CancellationToken cancellationToken = default ) @@ -53,7 +55,7 @@ namespace xAiApi.Controllers ValidationProvider.NotEmpty(question); // - var result = await aiService.AskAsync( + var result = await aiService.AskTextAsync( prompt: question, cancellationToken: cancellationToken ); @@ -68,5 +70,85 @@ namespace xAiApi.Controllers return result; } } + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [AllowAnonymous] + [HttpGet("AskTextStream")] + public virtual async Task AskTextStream( + [FromQuery] string question, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + ValidationProvider.NotEmpty(question); + + // + // Configure SSE Response Headers ... + Response.Headers["X-Accel-Buffering"] = "no"; + Response.Headers["Connection"] = "keep-alive"; + Response.Headers["Content-Type"] = "text/event-stream"; + Response.Headers["Cache-Control"] = "no-cache, no-store"; + + // + // Disable response buffering ... + var responseBodyFeature = HttpContext.Features + .Get(); + responseBodyFeature?.DisableBuffering(); + + // + // Stream Response ... + var jsonOptions = new JsonSerializerOptions + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + // + var enumerable = aiService.AskTextAsEnumerable( + prompt: question, + cancellationToken: cancellationToken + ); + + // + await foreach (var chunk in enumerable) + { + // + // Skip empty chunks ... + if (string.IsNullOrEmpty(chunk)) + { + continue; + } + + // + // Build SSE message ... + // Format: data: \n\n + // var payload = JsonSerializer.Serialize( + // new { text = chunk }, + // jsonOptions + // ); + var payload = chunk; + + // + // Write SSE Event ... + await Response.WriteAsync($"{payload}\n\n", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + + // + await Response.Body.FlushAsync(cancellationToken); + } + catch (Exception) + { + await Response.Body.FlushAsync(); + } + } } } \ No newline at end of file diff --git a/Interfaces/IXAiEmbeddingServiceBase.cs b/Interfaces/IXAiEmbeddingServiceBase.cs index 1c2037f..14886a5 100644 --- a/Interfaces/IXAiEmbeddingServiceBase.cs +++ b/Interfaces/IXAiEmbeddingServiceBase.cs @@ -34,11 +34,9 @@ namespace xAiApi.Interfaces ); /// - /// Access Embedding Client ... + /// Embedding Generator ... /// /// - OllamaApiClient GetEmbeddingClient(); - IEmbeddingGenerator> GetEmbeddingGenerator(); } } \ No newline at end of file diff --git a/Interfaces/IXAiServiceBase.cs b/Interfaces/IXAiServiceBase.cs index 01b9b4d..82f0545 100644 --- a/Interfaces/IXAiServiceBase.cs +++ b/Interfaces/IXAiServiceBase.cs @@ -18,7 +18,7 @@ namespace xAiApi.Interfaces /// /// /// - Task AskAsync( + Task AskTextAsync( string prompt, CancellationToken cancellationToken = default ); @@ -29,7 +29,7 @@ namespace xAiApi.Interfaces /// /// /// - IAsyncEnumerable AskAsEnumerable( + IAsyncEnumerable AskTextAsEnumerable( string prompt, CancellationToken cancellationToken = default ); diff --git a/Interfaces/IXAiServiceControllerBase.cs b/Interfaces/IXAiServiceControllerBase.cs index fe26abf..8e4aa52 100644 --- a/Interfaces/IXAiServiceControllerBase.cs +++ b/Interfaces/IXAiServiceControllerBase.cs @@ -12,10 +12,22 @@ namespace xAiApi.Interfaces /// /// /// - [HttpGet("Ask")] - Task> Ask( + [HttpGet("AskText")] + Task> AskText( [FromQuery] string question, CancellationToken cancellationToken = default ); + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [HttpGet("AskTextStream")] + Task AskTextStream( + [FromQuery] string question, + CancellationToken cancellationToken = default + ); } } \ No newline at end of file diff --git a/Providers/XAIServiceBase.cs b/Providers/XAIServiceBase.cs index ce3739c..afb5302 100644 --- a/Providers/XAIServiceBase.cs +++ b/Providers/XAIServiceBase.cs @@ -47,7 +47,7 @@ namespace xAiApi.Providers /// /// /// - public virtual async Task AskAsync( + public virtual async Task AskTextAsync( string prompt, CancellationToken cancellationToken = default ) @@ -81,10 +81,6 @@ namespace xAiApi.Providers // Retrieve Response Text ... var result = response.Text; - // - // Dispose Client ... - client.Dispose(); - // return result; } @@ -95,7 +91,7 @@ namespace xAiApi.Providers /// /// /// - public virtual async IAsyncEnumerable AskAsEnumerable( + public virtual async IAsyncEnumerable AskTextAsEnumerable( string prompt, [EnumeratorCancellation] CancellationToken cancellationToken = default @@ -117,7 +113,6 @@ namespace xAiApi.Providers ); // - var result = ""; await foreach (var res in enumerable) { // @@ -128,15 +123,8 @@ namespace xAiApi.Providers } // - result += res.Text; + yield return res.Text; } - - // - // Dispose Client ... - client.Dispose(); - - // - yield return result; } /// @@ -172,6 +160,7 @@ namespace xAiApi.Providers var ollamaClient = new OllamaApiClient(httpClient, model); result = new ChatClientBuilder(ollamaClient) + .UseFunctionInvocation() .Build(); break; @@ -188,9 +177,12 @@ namespace xAiApi.Providers } ); result = - openAiClient - .GetChatClient(model) - .AsIChatClient(); + new ChatClientBuilder( + openAiClient + .GetChatClient(model) + .AsIChatClient()) + .UseFunctionInvocation() + .Build(); break; // @@ -206,29 +198,6 @@ namespace xAiApi.Providers break; } - // // - // if (isOllamaBased) - // { - // } - // else if (isDeepSeekBased) - // { - // // - // if (apiKey is null || - // string.IsNullOrWhiteSpace(apiKey)) - // { - // throw new Exception("Inavlid Args ..."); - // } - - // // - // var deepseekClient = new DeepSeekChatClient(apiKey: apiKey); - // result = deepseekClient; - // } - // else - // { - // // - // // TODO: Implement this ... - // } - // if (result.IsNullOrDefault()) { @@ -330,7 +299,7 @@ namespace xAiApi.Providers // Preparing Introduction Prompt ... var introductionPrompt = new ChatMessage( ChatRole.System, - configuration.GetPrompt() + configuration.GetPrompt(XAiApiConstants.XAiApiIntroductionPromptName) ); // @@ -339,7 +308,7 @@ namespace xAiApi.Providers introductionPrompt, ]; if (forceMessage && - message is not null) + !message.IsNullOrDefault()) { result.Add(message); } diff --git a/Providers/XAiEmbeddingServiceBase.cs b/Providers/XAiEmbeddingServiceBase.cs index 8925cf7..62a63d9 100644 --- a/Providers/XAiEmbeddingServiceBase.cs +++ b/Providers/XAiEmbeddingServiceBase.cs @@ -37,11 +37,7 @@ namespace xAiApi.Providers XAiApiConfiguration configuration, ILogger logger, string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName - ) : base( - dataProvider, - logger, - configuration - ) + ) : base(dataProvider, logger, configuration) { // // Prepare Embedding Model Descriptor ... @@ -75,15 +71,13 @@ namespace xAiApi.Providers // using var generator = GetEmbeddingGenerator(); + + // GeneratedEmbeddings> response = null; - try - { - response = await generator.GenerateAsync([content]); - } - catch (Exception ex) - { - logger.LogError(ex.Message); - } + response = await generator.GenerateAsync( + values: [content], + cancellationToken: cancellationToken + ); // if (response.IsNullOrDefault()) @@ -103,40 +97,6 @@ namespace xAiApi.Providers XException.ActionFailed.Throw(); } - // - // Retrieve Ollama Client ... - // using var client = GetEmbeddingClient(); - - // // - // // Create Embedding Request .... - // var request = new EmbedRequest - // { - // Input = [content], - // Model = descriptor.LLM, - // }; - - // // - // // Make Request and Recieve Response ... - // var response = await client.EmbedAsync( - // request: request, - // cancellationToken: cancellationToken - // ); - // if (response.IsNullOrDefault()) - // { - // XException.ActionFailed.Throw(); - // } - - // // - // // Generae Result ... - // var result = response? - // .Embeddings? - // .FirstOrDefault() - // ?? []; - // if (result.Length == 0) - // { - // XException.ActionFailed.Throw(); - // } - // // Normalize ... VectorHelper.Normalize(result); @@ -166,41 +126,25 @@ namespace xAiApi.Providers } // - // Retrieve Ollama Client ... - using var client = GetEmbeddingClient(); + using var generator = GetEmbeddingGenerator(); // - // Create Embedding Request .... - var request = new EmbedRequest - { - Input = [.. batch], - Model = descriptor.LLM, - }; - - // - // Make Request and Recieve Response ... - var response = await client.EmbedAsync( - request: request, + GeneratedEmbeddings> response = null; + response = await generator.GenerateAsync( + values: [.. batch], cancellationToken: cancellationToken ); + + // if (response.IsNullOrDefault()) { XException.ActionFailed.Throw(); } // - // Generae Result ... - var result = response? - .Embeddings - .ToArray() - ?? []; - - // - // Validate Result ... - if (result.Length == 0) - { - XException.ActionFailed.Throw(); - } + var result = response + .Select(e => e.Vector.ToArray()) + .ToList(); // // Normalize ... @@ -214,18 +158,9 @@ namespace xAiApi.Providers } /// - /// Access Embedding Client ... + /// Embedding Generator ... /// /// - public OllamaApiClient GetEmbeddingClient() - { - // - var result = new OllamaApiClient(new Uri(descriptor.Url)); - - // - return result; - } - public IEmbeddingGenerator> GetEmbeddingGenerator() { //