From 92988ea155bb9d7536d5a77f1a55ce8f49655421 Mon Sep 17 00:00:00 2001 From: Hadi Khazaee Asl Date: Fri, 2 Oct 2026 22:13:01 +0330 Subject: [PATCH] last ... --- Documents/Docs/Developer/5.html | 466 ++ Documents/Docs/Developer/6.html | 1346 +++++ .../xDashboard-xAiApi-20261002_203559.txt | 5010 +++++++++++++++++ Modules/xAiModels | 2 +- xAiApi | 2 +- 5 files changed, 6824 insertions(+), 2 deletions(-) create mode 100644 Documents/Docs/Developer/5.html create mode 100644 Documents/Docs/Developer/6.html create mode 100644 Documents/GeneratedContext/xDashboard-xAiApi-20261002_203559.txt diff --git a/Documents/Docs/Developer/5.html b/Documents/Docs/Developer/5.html new file mode 100644 index 0000000..c923655 --- /dev/null +++ b/Documents/Docs/Developer/5.html @@ -0,0 +1,466 @@ + + + + + +افزودن پشتیبانی DOCX و Excel به FileContentExtractor - فن آوران ساحر علم + + + +
+ +
+

📄 افزودن پشتیبانی DOCX و Excel به FileContentExtractor

+
تکمیل زنجیره استخراج محتوا برای فرمت‌های آفیس در معماری xAiApi
+
+ +
+
👨‍💻 توسعه‌دهنده: هادی خزاعی اصل
+
🏢 شرکت: فن آوران ساحر علم
+
📅 تاریخ تهیه مستند: چهارشنبه ۸ مهر ۱۴۰۵
+
📦 پروژه: xAiApi / xAiService
+
+ +
+ +
+

📑 فهرست مطالب

+
    +
  1. گام ۱: نصب پکیج‌های NuGet مورد نیاز
    2. +
  2. گام ۲: پیاده‌سازی DocxContentExtractor
    3. +
  3. گام ۳: پیاده‌سازی ExcelContentExtractor
    4. +
  4. گام ۴: ثبت سرویس‌ها در Dependency Injection
    5. +
  5. گام ۵: نکات کلیدی و ملاحظات عملکردی
    6. +
+
+ + +
+

📦 گام ۱: نصب پکیج‌های NuGet مورد نیاز

+

برای استخراج متن از فایل‌های Office بدون نیاز به نصب نرم‌افزار Microsoft Office روی سرور، از کتابخانه‌های سبک و قدرتمند زیر استفاده می‌کنیم:

+ +
+ ۱ + دستورات نصب در Package Manager Console (پروژه xAiService یا xAiApi): +
# برای پردازش فایل‌های Word (DOCX)
+Install-Package DocumentFormat.OpenXml
+
+# برای پردازش فایل‌های Excel (XLSX و XLS)
+Install-Package ExcelDataReader
+Install-Package ExcelDataReader.DataSet
+
+
+ 💡 چرا این پکیج‌ها؟
+ DocumentFormat.OpenXml استاندارد رسمی مایکروسافت است و بسیار پایدار می‌باشد. ExcelDataReader فوق‌العاده سریع است و نیاز به COM Interop ندارد، که آن را برای محیط‌های سروری (Server-side) ایده‌آل می‌کند. +
+
+ + +
+

📝 گام ۲: پیاده‌سازی DocxContentExtractor

+

این کلاس مسئول استخراج متن از فایل‌های .docx است. نکته مهم در اینجا این است که استریم ورودی را به MemoryStream تبدیل می‌کنیم، زیرا OpenXml به یک استریم قابل جستجو (Seekable) نیاز دارد.

+ +
+ NEW + xAiService/Providers/DocxContentExtractor.cs +
+ +
using System;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using xAiService.Interfaces;
+using DocumentFormat.OpenXml.Packaging;
+using DocumentFormat.OpenXml.Wordprocessing;
+
+namespace xAiService.Providers
+{
+    /// <summary>
+    /// Extracts text content from DOCX files using DocumentFormat.OpenXml ...
+    /// </summary>
+    public class DocxContentExtractor : IFileContentExtractor
+    {
+        private const string DocxMimeType = "application/vnd.openxmlformats-officedocument.wordprocessingml.document";
+
+        public bool CanExtract(string mimeType)
+        {
+            return string.Equals(mimeType, DocxMimeType, StringComparison.OrdinalIgnoreCase);
+        }
+
+        public async Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            // نکته حیاتی: OpenXml به استریم قابل جستجو (Seekable) نیاز دارد.
+            // استریم IFormFile ممکن است Seekable نباشد، بنابراین آن را در حافظه کپی می‌کنیم.
+            using var memoryStream = new MemoryStream();
+            await fileStream.CopyToAsync(memoryStream, cancellationToken);
+            memoryStream.Position = 0;
+
+            var stringBuilder = new StringBuilder();
+
+            // باز کردن سند Word به حالت فقط خواندنی
+            using (var wordDocument = WordprocessingDocument.Open(memoryStream, false))
+            {
+                var body = wordDocument.MainDocumentPart?.Document?.Body;
+                if (body != null)
+                {
+                    // استخراج پاراگراف‌ها و joining آنها با خط جدید برای حفظ ساختار برای LLM
+                    var paragraphs = body.Elements<Paragraph>();
+                    foreach (var paragraph in paragraphs)
+                    {
+                        var text = paragraph.InnerText?.Trim();
+                        if (!string.IsNullOrEmpty(text))
+                        {
+                            stringBuilder.AppendLine(text);
+                        }
+                    }
+                }
+            }
+
+            return stringBuilder.ToString();
+        }
+    }
+}
+
+ + +
+

📊 گام ۳: پیاده‌سازی ExcelContentExtractor

+

این کلاس فایل‌های .xlsx و .xls را خوانده و محتوای آن را به فرمت متنی ساختاریافته (شبیه CSV) تبدیل می‌کند تا مدل زبانی (LLM) بتواند به راحتی روابط سطرها و ستون‌ها را درک کند.

+ +
+ NEW + xAiService/Providers/ExcelContentExtractor.cs +
+ +
using System;
+using System.Data;
+using System.IO;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using xAiService.Interfaces;
+using ExcelDataReader;
+
+namespace xAiService.Providers
+{
+    /// <summary>
+    /// Extracts tabular content from Excel files (XLSX, XLS) using ExcelDataReader ...
+    /// </summary>
+    public class ExcelContentExtractor : IFileContentExtractor
+    {
+        private static readonly string[] SupportedMimeTypes = new[]
+        {
+            "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", // .xlsx
+            "application/vnd.ms-excel" // .xls
+        };
+
+        public bool CanExtract(string mimeType)
+        {
+            return Array.Exists(SupportedMimeTypes, type => 
+                string.Equals(type, mimeType, StringComparison.OrdinalIgnoreCase)
+            );
+        }
+
+        public async Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            // ثبت Encoding Provider برای پشتیبانی از فرمت‌های قدیمی‌تر Excel
+            System.Text.Encoding.RegisterProvider(System.Text.CodePagesEncodingProvider.Instance);
+
+            // کپی در MemoryStream برای اطمینان از Seekable بودن
+            using var memoryStream = new MemoryStream();
+            await fileStream.CopyToAsync(memoryStream, cancellationToken);
+            memoryStream.Position = 0;
+
+            var stringBuilder = new StringBuilder();
+
+            // ایجاد Reader به صورت خودکار بر اساس فرمت فایل
+            using (var reader = ExcelReaderFactory.CreateReader(memoryStream))
+            {
+                var result = reader.AsDataSet();
+
+                foreach (DataTable table in result.Tables)
+                {
+                    stringBuilder.AppendLine($"--- Sheet: {table.TableName} ---");
+                    
+                    // افزودن هدر ستون‌ها
+                    var headers = new string[table.Columns.Count];
+                    for (int i = 0; i < table.Columns.Count; i++)
+                    {
+                        headers[i] = table.Columns[i].ColumnName;
+                    }
+                    stringBuilder.AppendLine(string.Join(" | ", headers));
+                    stringBuilder.AppendLine(new string('-', 50));
+
+                    // افزودن داده‌های سطرها
+                    foreach (DataRow row in table.Rows)
+                    {
+                        var rowValues = new string[row.ItemArray.Length];
+                        for (int i = 0; i < row.ItemArray.Length; i++)
+                        {
+                            rowValues[i] = row.ItemArray[i]?.ToString()?.Trim() ?? string.Empty;
+                        }
+                        stringBuilder.AppendLine(string.Join(" | ", rowValues));
+                    }
+                    stringBuilder.AppendLine();
+                }
+            }
+
+            return stringBuilder.ToString();
+        }
+    }
+}
+
+ + +
+

🔗 گام ۴: ثبت سرویس‌ها در Dependency Injection

+

اکنون باید Extractor های جدید را به کانتینر DI اضافه کنیم. CompositeFileContentExtractor که قبلاً طراحی شده بود، به صورت خودکار تمام پیاده‌سازی‌های IFileContentExtractor را از طریق تزریق IEnumerable<IFileContentExtractor> دریافت کرده و مدیریت می‌کند.

+ +
+ MODIFY + xAiApi/Startup.cs (متد ConfigureServices) +
+ +
public void ConfigureServices(IServiceCollection services)
+{
+    // ... (سایر ثبت‌های سرویس)
+
+    // ✅ ثبت Extractor های فایل (الگوی Composite)
+    services.AddSingleton<IFileContentExtractor, PlainTextContentExtractor>();
+    services.AddSingleton<IFileContentExtractor, PdfContentExtractor>();
+    
+    // ✅ افزودن Extractor های جدید آفیس
+    services.AddSingleton<IFileContentExtractor, DocxContentExtractor>();
+    services.AddSingleton<IFileContentExtractor, ExcelContentExtractor>();
+    
+    // ✅ ثبت Composite Extractor (این کلاس لیست بالا را در Constructor دریافت می‌کند)
+    services.AddSingleton<IFileContentExtractor, CompositeFileContentExtractor>();
+
+    // ... (سایر ثبت‌های سرویس)
+}
+
+ + +
+

⚠️ گام ۵: نکات کلیدی و ملاحظات عملکردی

+ +
+
+

🧠 محدودیت Context Window مدل

+

فایل‌های Excel بزرگ می‌توانند هزاران خط متن تولید کنند. قبل از ارسال به LLM، حتماً طول رشته استخراج شده را بررسی کنید و در صورت نیاز، آن را خلاصه (Chunk) کنید یا فقط چند سطر اول را ارسال نمایید.

+
+
+

💾 مدیریت حافظه (Memory)

+

استفاده از MemoryStream برای فایل‌های بسیار بزرگ (مثلاً بالای 50 مگابایت) ممکن است باعث OutOfMemoryException شود. در appsettings.json حداکثر حجم فایل آپلودی را محدود کنید.

+
+
+

🔒 امنیت و اعتبارسنجی

+

کتابخانه ExcelDataReader در برابر فایل‌های مخرب مقاوم است، اما همیشه قبل از پردازش، نوع فایل (MIME Type) و پسوند آن را در لایه Controller اعتبارسنجی کنید.

+
+
+

🌐 پشتیبانی از Encoding

+

خط Encoding.RegisterProvider(CodePagesEncodingProvider.Instance) در Extractor اکسل حیاتی است. بدون آن، خواندن فایل‌های .xls قدیمی با کاراکترهای فارسی با خطا مواجه می‌شود.

+
+
+ +
+ ✅ نتیجه‌گیری:
+ با افزودن این دو کلاس، معماری FileContentExtractor شما اکنون به طور کامل از فرمت‌های متنی، PDF، Word و Excel پشتیبانی می‌کند. این تغییرات کاملاً با اصل Open/Closed (باز برای توسعه، بسته برای تغییر) در SOLID همخوانی دارد، زیرا برای افزودن فرمت جدید (مثلاً PowerPoint در آینده)، فقط کافی است یک کلاس جدید اضافه و در DI ثبت کنید، بدون اینکه کدهای موجود را تغییر دهید. +
+
+ +
+ +
+

👨‍💻 توسعه‌دهنده: هادی خزاعی اصل

+

🏢 شرکت: فن آوران ساحر علم

+

📅 تاریخ تهیه مستند: چهارشنبه ۸ مهر ۱۴۰۵

+

+ 📄 مستند فنی افزودن پشتیبانی Office Files به xAiApi - تمامی حقوق محفوظ است +

+
+ +
+ + \ No newline at end of file diff --git a/Documents/Docs/Developer/6.html b/Documents/Docs/Developer/6.html new file mode 100644 index 0000000..0a33611 --- /dev/null +++ b/Documents/Docs/Developer/6.html @@ -0,0 +1,1346 @@ + + + + + +ارتقای سرویس استخراج محتوا - پشتیبانی از تصویر، صوت و PDF Fallback - فن آوران ساحر علم + + + +
+ +
+

🎨 ارتقای سرویس استخراج محتوا

+
پشتیبانی از تصویر، صوت و Fallback هوشمند برای PDF های اسکن‌شده
+
+ +
+
👨‍💻 توسعه‌دهنده: هادی خزاعی اصل
+
🏢 شرکت: فن آوران ساحر علم
+
📅 تاریخ: شنبه ۱۲ مهر ۱۴۰۵
+
📦 پروژه: xAiApi
+
+ +
+ +
+

📑 فهرست مطالب

+
    +
  1. تحلیل وضعیت فعلی و شکاف‌ها
    2. +
  2. استراتژی معماری جدید
    3. +
  3. گام ۱: بازطراحی Interface با خروجی چندگانه
    4. +
  4. گام ۲: پیاده‌سازی ImageContentExtractor
    5. +
  5. گام ۳: پیاده‌سازی AudioContentExtractor
    6. +
  6. گام ۴: ارتقای PdfExtractor با Fallback تصویری
    7. +
  7. گام ۵: یکپارچه‌سازی در XAIServiceBase
    8. +
  8. گام ۶: ثبت در DI و پیکربندی
    9. +
  9. جریان کامل پردازش
    10. +
  10. خلاصه تغییرات و پکیج‌ها
    11. +
+
+ + +
+

🔍 گام ۱: تحلیل وضعیت فعلی و شکاف‌ها

+ +

۱.۱ وضعیت فعلی Extractor ها:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Extractorوضعیتنوع خروجیشکاف
XPlainTextFileContentExtractor✅ موجودText-
XDocxFileContentExtractor✅ موجودTextعدم پشتیبانی از تصاویر داخل DOCX
XExcelFileContentExtractor✅ موجودText (Tabular)-
XPdfFileContentExtractor⚠️ ناقصTextعدم Fallback برای PDF های اسکن‌شده
Image Extractor❌ موجود نیست-نیاز به پیاده‌سازی (Vision API)
Audio Extractor❌ موجود نیست-نیاز به پیاده‌سازی (Whisper)
+ +

۱.۲ مشکل اساسی در IXFileContentExtractor فعلی:

+
+ ⚠️ محدودیت طراحی فعلی: +

Interface فعلی فقط یک string برمی‌گرداند. این برای متن کافی است اما برای تصویر و صوت نیاز به ساختار داده‌ای غنی‌تر داریم که بتواند چندین نوع محتوا (Text + Images + Audio) را همزمان حمل کند.

+
+ +
// ❌ Interface فعلی - محدود به string
+public interface IXFileContentExtractor
+{
+    bool CanExtract(string mimeType);
+    Task<string> ExtractAsync(Stream fileStream, string mimeType, ...);
+}
+
+ + +
+

🎯 گام ۲: استراتژی معماری جدید

+ +
+
+

📦 خروجی چندگانه (Multi-Content)

+
    +
  • ایجاد XFileExtractionResult
    • +
  • شامل: Text, Images[], AudioTranscript
    • +
  • سازگار با AIContent
    • +
+
+
+

🖼️ پشتیبانی تصویر (Vision)

+
    +
  • ارسال به عنوان DataContent
    • +
  • سازگار با Gemini, GPT-4V, Qwen-VL
    • +
  • OCR محلی به عنوان Fallback
    • +
+
+
+

🎙️ پشتیبانی صوت (Whisper)

+
    +
  • تبدیل Speech-to-Text
    • +
  • استفاده از Whisper محلی یا API
    • +
  • خروجی به صورت متن
    • +
+
+
+

📄 PDF Fallback هوشمند

+
    +
  • استخراج متن ابتدا
    • +
  • اگر متن خالی/کم‌حجم → تبدیل به تصویر
    • +
  • ارسال صفحات به عنوان Vision
    • +
+
+
+ +
+ ✅ مزیت کلیدی: این طراحی Backward Compatible است. کدهای موجود که از ExtractAsync استفاده می‌کنند، بدون تغییر کار خواهند کرد. +
+
+ + +
+

🔧 گام ۳: بازطراحی Interface با خروجی چندگانه

+ +
+ NEW + xAiApi/Models/XFileExtractionResult.cs +
+ +
using System;
+using System.Collections.Generic;
+using System.Linq;
+using xCommons.Extensions;
+using Microsoft.Extensions.AI;
+
+namespace xAiApi.Models
+{
+    /// <summary>
+    /// نتیجه استخراج محتوا از فایل - می‌تواند شامل متن، تصاویر و صوت باشد ...
+    /// </summary>
+    public class XFileExtractionResult
+    {
+        /// <summary>
+        /// محتوای متنی استخراج شده ...
+        /// </summary>
+        public string Text { get; set; } = string.Empty;
+
+        /// <summary>
+        /// تصاویر استخراج شده (برای Vision Models) ...
+        /// </summary>
+        public IList<XExtractedImage> Images { get; set; } = [];
+
+        /// <summary>
+        /// متن تبدیل شده از صوت ...
+        /// </summary>
+        public string AudioTranscript { get; set; } = string.Empty;
+
+        /// <summary>
+        /// نام فایل اصلی ...
+        /// </summary>
+        public string FileName { get; set; }
+
+        /// <summary>
+        /// MIME Type فایل ...
+        /// </summary>
+        public string MimeType { get; set; }
+
+        /// <summary>
+        /// آیا استخراج با Fallback تصویری انجام شده است ...
+        /// </summary>
+        public bool UsedImageFallback { get; set; } = false;
+
+        /// <summary>
+        /// پیام خطا (در صورت بروز مشکل) ...
+        /// </summary>
+        public string ErrorMessage { get; set; }
+
+        /// <summary>
+        /// آیا نتیجه معتبر است ...
+        /// </summary>
+        public bool IsValid()
+        {
+            return !Text.IsNullOrEmpty() ||
+                   Images.HasChild() ||
+                   !AudioTranscript.IsNullOrEmpty();
+        }
+
+        /// <summary>
+        /// تبدیل به لیست AIContent برای ارسال به LLM ...
+        /// </summary>
+        public IList<AIContent> ToAIContents()
+        {
+            var result = new List<AIContent>();
+
+            // ۱. افزودن متن اصلی
+            if (!Text.IsNullOrEmpty())
+            {
+                result.Add(new TextContent(Text));
+            }
+
+            // ۲. افزودن تصاویر (برای Vision Models)
+            if (Images.HasChild())
+            {
+                foreach (var img in Images)
+                {
+                    result.Add(new DataContent(img.Bytes, img.MimeType));
+                }
+            }
+
+            // ۳. افزودن transcript صوت
+            if (!AudioTranscript.IsNullOrEmpty())
+            {
+                result.Add(new TextContent($"[Audio Transcript]\n{AudioTranscript}"));
+            }
+
+            return result;
+        }
+    }
+
+    /// <summary>
+    /// تصویر استخراج شده از فایل ...
+    /// </summary>
+    public class XExtractedImage
+    {
+        /// <summary>
+        /// داده بایت تصویر ...
+        /// </summary>
+        public byte[] Bytes { get; set; }
+
+        /// <summary>
+        /// MIME Type تصویر ...
+        /// </summary>
+        public string MimeType { get; set; } = "image/png";
+
+        /// <summary>
+        /// شماره صفحه (برای PDF) ...
+        /// </summary>
+        public int? PageNumber { get; set; }
+
+        /// <summary>
+        /// توضیح تصویر ...
+        /// </summary>
+        public string Description { get; set; }
+    }
+}
+ +
+ MODIFY + xAiApi/Interfaces/Extractors/IXFileContentExtractor.cs +
+ +
using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+using xAiApi.Models;
+
+namespace xAiApi.Interfaces.Extractors
+{
+    /// <summary>
+    /// سرویس استخراج محتوای فایل - نسخه ارتقا یافته با پشتیبانی چندوجهی ...
+    /// </summary>
+    public interface IXFileContentExtractor
+    {
+        /// <summary>
+        /// بررسی پشتیبانی از MIME Type مشخص ...
+        /// </summary>
+        bool CanExtract(string mimeType);
+
+        /// <summary>
+        /// استخراج محتوای فایل با خروجی غنی (متن + تصویر + صوت) ...
+        /// </summary>
+        Task<XFileExtractionResult> ExtractRichAsync(
+            Stream fileStream,
+            string fileName,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        );
+
+        /// <summary>
+        /// استخراج محتوای متنی (سازگار با نسخه قبل) ...
+        /// </summary>
+        Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        );
+    }
+}
+
+ + +
+

🖼️ گام ۴: پیاده‌سازی ImageContentExtractor

+

این Extractor تصاویر را برای مدل‌های Vision آماده می‌کند. دو حالت پشتیبانی می‌شود:

+
    +
  • حالت ۱ (Vision API): ارسال مستقیم به عنوان DataContent به مدل‌های چندوجهی
    • +
  • حالت ۲ (OCR Fallback): استخراج متن با Tesseract برای مدل‌های متنی
    • +
+ +
+ NEW + xAiApi/Interfaces/Extractors/IXImageFileContentExtractor.cs +
+ +
namespace xAiApi.Interfaces.Extractors
+{
+    /// <summary>
+    /// Extractor اختصاصی برای فایل‌های تصویری ...
+    /// </summary>
+    public interface IXImageFileContentExtractor : IXFileContentExtractor
+    { }
+}
+ +
+ NEW + xAiApi/Providers/Extractors/XImageFileContentExtractor.cs +
+ +
using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using xAiApi.Models;
+using xAiApi.Interfaces.Extractors;
+
+namespace xAiApi.Providers.Extractors
+{
+    /// <summary>
+    /// استخراج محتوا از فایل‌های تصویری - پشتیبانی از Vision و OCR ...
+    /// </summary>
+    public class XImageFileContentExtractor : IXImageFileContentExtractor
+    {
+        private static readonly string[] SupportedMimeTypes =
+        [
+            "image/png",
+            "image/jpeg",
+            "image/jpg",
+            "image/gif",
+            "image/webp",
+            "image/bmp"
+        ];
+
+        /// <summary>
+        /// آیا OCR فعال است (برای مدل‌های غیر Vision) ...
+        /// </summary>
+        private readonly bool _enableOcrFallback;
+
+        public XImageFileContentExtractor(bool enableOcrFallback = true)
+        {
+            _enableOcrFallback = enableOcrFallback;
+        }
+
+        public bool CanExtract(string mimeType)
+        {
+            return SupportedMimeTypes.Contains(
+                mimeType?.ToLowerInvariant() ?? string.Empty
+            );
+        }
+
+        public async Task<XFileExtractionResult> ExtractRichAsync(
+            Stream fileStream,
+            string fileName,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = new XFileExtractionResult
+            {
+                FileName = fileName,
+                MimeType = mimeType
+            };
+
+            // خواندن بایت‌های تصویر
+            using var memoryStream = new MemoryStream();
+            await fileStream.CopyToAsync(memoryStream, cancellationToken);
+            var imageBytes = memoryStream.ToArray();
+
+            // افزودن به عنوان تصویر برای Vision Models
+            result.Images.Add(new XExtractedImage
+            {
+                Bytes = imageBytes,
+                MimeType = mimeType,
+                Description = $"Attached image: {fileName}"
+            });
+
+            // OCR Fallback برای مدل‌های متنی
+            if (_enableOcrFallback)
+            {
+                try
+                {
+                    var ocrText = await PerformOcrAsync(imageBytes, cancellationToken);
+                    if (!string.IsNullOrWhiteSpace(ocrText))
+                    {
+                        result.Text = ocrText;
+                    }
+                }
+                catch
+                {
+                    // OCR failed, but image is still available for Vision
+                }
+            }
+
+            return result;
+        }
+
+        public async Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = await ExtractRichAsync(
+                fileStream, 
+                "image", 
+                mimeType, 
+                cancellationToken
+            );
+            return result.Text;
+        }
+
+        /// <summary>
+        /// انجام OCR روی تصویر - با استفاده از Tesseract ...
+        /// </summary>
+        private async Task<string> PerformOcrAsync(
+            byte[] imageBytes,
+            CancellationToken cancellationToken
+        )
+        {
+            // پیاده‌سازی با Tesseract یا هر کتابخانه OCR دیگر
+            // در حال حاضر یک placeholder است
+            return await Task.FromResult(string.Empty);
+        }
+    }
+}
+ +
+ ⚠️ پکیج‌های NuGet پیشنهادی برای OCR: +
    +
  • Tesseract - کتابخانه قدرتمند و متن‌باز
    • +
  • Microsoft.Azure.CognitiveServices.Vision.ComputerVision - اگر از Azure استفاده می‌کنید
    • +
+
+
+ + +
+

🎙️ گام ۵: پیاده‌سازی AudioContentExtractor

+

این Extractor فایل‌های صوتی را با استفاده از Whisper به متن تبدیل می‌کند.

+ +
+ NEW + xAiApi/Interfaces/Extractors/IXAudioFileContentExtractor.cs +
+ +
namespace xAiApi.Interfaces.Extractors
+{
+    /// <summary>
+    /// Extractor اختصاصی برای فایل‌های صوتی ...
+    /// </summary>
+    public interface IXAudioFileContentExtractor : IXFileContentExtractor
+    { }
+}
+ +
+ NEW + xAiApi/Providers/Extractors/XAudioFileContentExtractor.cs +
+ +
using System;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using xAiApi.Models;
+using xAiApi.Interfaces.Extractors;
+// Install-Package Whisper.net
+// Install-Package Whisper.net.Runtime
+using Whisper.net;
+
+namespace xAiApi.Providers.Extractors
+{
+    /// <summary>
+    /// استخراج متن از فایل‌های صوتی با استفاده از Whisper ...
+    /// </summary>
+    public class XAudioFileContentExtractor : IXAudioFileContentExtractor
+    {
+        private static readonly string[] SupportedMimeTypes =
+        [
+            "audio/mpeg",
+            "audio/mp3",
+            "audio/wav",
+            "audio/wave",
+            "audio/ogg",
+            "audio/m4a",
+            "audio/mp4",
+            "audio/webm"
+        ];
+
+        private readonly string _whisperModelPath;
+        private readonly string _language;
+
+        public XAudioFileContentExtractor(
+            string whisperModelPath = "Models/ggml-base.bin",
+            string language = "fa" // فارسی پیش‌فرض
+        )
+        {
+            _whisperModelPath = whisperModelPath;
+            _language = language;
+        }
+
+        public bool CanExtract(string mimeType)
+        {
+            return SupportedMimeTypes.Contains(
+                mimeType?.ToLowerInvariant() ?? string.Empty
+            );
+        }
+
+        public async Task<XFileExtractionResult> ExtractRichAsync(
+            Stream fileStream,
+            string fileName,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = new XFileExtractionResult
+            {
+                FileName = fileName,
+                MimeType = mimeType
+            };
+
+            try
+            {
+                // کپی به MemoryStream برای Whisper
+                using var memoryStream = new MemoryStream();
+                await fileStream.CopyToAsync(memoryStream, cancellationToken);
+                memoryStream.Position = 0;
+
+                // تبدیل به فرمت WAV 16kHz (مورد نیاز Whisper)
+                using var wavStream = await ConvertToWavAsync(memoryStream, cancellationToken);
+
+                // انجام Speech-to-Text با Whisper
+                using var factory = WhisperFactory.FromPath(_whisperModelPath);
+                using var processor = factory.CreateBuilder()
+                    .WithLanguage(_language)
+                    .Build();
+
+                var segments = new System.Text.StringBuilder();
+                await foreach (var segment in processor.ProcessAsync(wavStream, cancellationToken))
+                {
+                    segments.Append(segment.Text);
+                }
+
+                result.AudioTranscript = segments.ToString().Trim();
+                result.Text = result.AudioTranscript;
+            }
+            catch (Exception ex)
+            {
+                result.ErrorMessage = $"Audio extraction failed: {ex.Message}";
+            }
+
+            return result;
+        }
+
+        public async Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = await ExtractRichAsync(
+                fileStream, 
+                "audio", 
+                mimeType, 
+                cancellationToken
+            );
+            return result.AudioTranscript;
+        }
+
+        /// <summary>
+        /// تبدیل فرمت صوتی به WAV 16kHz (فرمت مورد نیاز Whisper) ...
+        /// </summary>
+        private async Task<Stream> ConvertToWavAsync(
+            Stream inputStream,
+            CancellationToken cancellationToken
+        )
+        {
+            // پیاده‌سازی با NAudio یا ffmpeg
+            // در حال حاضر یک placeholder است
+            return await Task.FromResult(inputStream);
+        }
+    }
+}
+ +
+ 💡 پکیج‌های NuGet مورد نیاز: +
    +
  • Whisper.net - پیاده‌سازی C# از Whisper
    • +
  • Whisper.net.Runtime - Runtime مورد نیاز
    • +
  • NAudio - برای تبدیل فرمت صوتی
    • +
+

دانلود مدل Whisper: از HuggingFace مدل ggml-base.bin یا ggml-small.bin را دانلود کنید.

+
+
+ + +
+

📄 گام ۶: ارتقای PdfExtractor با Fallback تصویری

+

این مهم‌ترین بخش است. اگر PDF متن نداشت (اسکن‌شده بود)، صفحات را به تصویر تبدیل می‌کنیم.

+ +
+ MODIFY + xAiApi/Providers/Extractors/XPdfFileContentExtractor.cs +
+ +
using System;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using UglyToad.PdfPig;
+using UglyToad.PdfPig.Rendering;
+using xAiApi.Models;
+using xAiApi.Interfaces.Extractors;
+
+namespace xAiApi.Providers
+{
+    /// <summary>
+    /// استخراج محتوا از PDF - با Fallback هوشمند برای PDF های اسکن‌شده ...
+    /// </summary>
+    public class XPdfFileContentExtractor : IXPdfFileContentExtractor
+    {
+        private static readonly string[] SupportedMimeTypes = ["application/pdf"];
+
+        /// <summary>
+        /// حداقل تعداد کاراکتر برای تشخیص PDF متنی ...
+        /// </summary>
+        private const int MinTextLengthThreshold = 50;
+
+        /// <summary>
+        /// حداکثر تعداد صفحات برای تبدیل به تصویر ...
+        /// </summary>
+        private const int MaxPagesForImageFallback = 10;
+
+        public bool CanExtract(string mimeType)
+        {
+            return SupportedMimeTypes.Contains(
+                mimeType?.ToLowerInvariant() ?? string.Empty
+            );
+        }
+
+        public async Task<XFileExtractionResult> ExtractRichAsync(
+            Stream fileStream,
+            string fileName,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = new XFileExtractionResult
+            {
+                FileName = fileName,
+                MimeType = mimeType
+            };
+
+            // کپی به MemoryStream
+            using var memoryStream = new MemoryStream();
+            await fileStream.CopyToAsync(memoryStream, cancellationToken);
+            memoryStream.Position = 0;
+
+            // مرحله ۱: تلاش برای استخراج متن
+            var extractedText = await ExtractTextAsync(memoryStream, cancellationToken);
+            
+            // مرحله ۲: بررسی کیفیت متن استخراج شده
+            if (IsTextSufficient(extractedText))
+            {
+                // PDF متنی است
+                result.Text = extractedText;
+            }
+            else
+            {
+                // PDF اسکن‌شده است - Fallback به تصویر
+                memoryStream.Position = 0;
+                result = await FallbackToImageExtractionAsync(
+                    memoryStream, 
+                    fileName, 
+                    mimeType,
+                    cancellationToken
+                );
+                result.UsedImageFallback = true;
+            }
+
+            return result;
+        }
+
+        public async Task<string> ExtractAsync(
+            Stream fileStream,
+            string mimeType,
+            CancellationToken cancellationToken = default
+        )
+        {
+            var result = await ExtractRichAsync(
+                fileStream, 
+                "document.pdf", 
+                mimeType, 
+                cancellationToken
+            );
+            return result.Text;
+        }
+
+        /// <summary>
+        /// استخراج متن از PDF ...
+        /// </summary>
+        private async Task<string> ExtractTextAsync(
+            Stream pdfStream,
+            CancellationToken cancellationToken
+        )
+        {
+            return await Task.Run(() =>
+            {
+                var sb = new StringBuilder();
+                using var document = PdfDocument.Open(pdfStream);
+                
+                foreach (var page in document.GetPages())
+                {
+                    var text = page.Text?.Trim();
+                    if (!string.IsNullOrEmpty(text))
+                    {
+                        sb.AppendLine(text);
+                        sb.AppendLine();
+                    }
+                }
+                
+                return sb.ToString();
+            }, cancellationToken);
+        }
+
+        /// <summary>
+        /// بررسی آیا متن استخراج شده کافی است ...
+        /// </summary>
+        private bool IsTextSufficient(string text)
+        {
+            if (string.IsNullOrWhiteSpace(text))
+            {
+                return false;
+            }
+
+            // حذف فاصله‌ها و بررسی طول
+            var cleanText = new string(text.Where(c => !char.IsWhiteSpace(c)).ToArray());
+            return cleanText.Length >= MinTextLengthThreshold;
+        }
+
+        /// <summary>
+        /// Fallback: تبدیل صفحات PDF به تصویر ...
+        /// </summary>
+        private async Task<XFileExtractionResult> FallbackToImageExtractionAsync(
+            Stream pdfStream,
+            string fileName,
+            string mimeType,
+            CancellationToken cancellationToken
+        )
+        {
+            var result = new XFileExtractionResult
+            {
+                FileName = fileName,
+                MimeType = mimeType,
+                UsedImageFallback = true
+            };
+
+            await Task.Run(() =>
+            {
+                using var document = PdfDocument.Open(pdfStream);
+                using var renderer = new PdfPigRenderer(document);
+
+                var pageCount = Math.Min(
+                    document.NumberOfPages, 
+                    MaxPagesForImageFallback
+                );
+
+                for (int i = 0; i < pageCount; i++)
+                {
+                    var page = document.GetPage(i);
+                    var bitmap = renderer.Render(page);
+
+                    // تبدیل Bitmap به PNG
+                    using var memoryStream = new MemoryStream();
+                    bitmap.Save(memoryStream, System.Drawing.Imaging.ImageFormat.Png);
+                    var imageBytes = memoryStream.ToArray();
+
+                    result.Images.Add(new XExtractedImage
+                    {
+                        Bytes = imageBytes,
+                        MimeType = "image/png",
+                        PageNumber = i + 1,
+                        Description = $"Page {i + 1} of {fileName}"
+                    });
+                }
+            }, cancellationToken);
+
+            return result;
+        }
+    }
+}
+ +
+ ⚠️ پکیج‌های NuGet اضافی برای PDF به تصویر: +
    +
  • UglyToad.PdfPig - قبلاً نصب شده
    • +
  • System.Drawing.Common - برای کار با Bitmap
    • +
+

نکته: PdfPigRenderer در نسخه‌های جدید PdfPig وجود دارد. اگر نبود، از Docnet یا PdfiumViewer استفاده کنید.

+
+
+ + +
+

🔗 گام ۷: یکپارچه‌سازی در XAIServiceBase

+ +
+ MODIFY + xAiApi/Providers/XAIServiceBase.cs +
+ +

اصلاح متد AskAsync - بخش پردازش فایل‌ها:

+ +
// در متد AskAsync، بعد از آپلود فایل‌ها و AddReference:
+
+// ✅ استخراج محتوای فایل‌ها با خروجی غنی
+IList<AIContent> fileContents = null;
+if (uploadedFiles.HasChild())
+{
+    fileContents = new List<AIContent>();
+    
+    foreach (var uploadedFile in uploadedFiles)
+    {
+        try
+        {
+            // دریافت Stream فایل از FileProvider
+            var fileDescriptor = await fileProvider.GetFileDescriptor(
+                id: uploadedFile.Id,
+                cancellationToken: cancellationToken
+            );
+            
+            if (fileDescriptor == null || fileDescriptor.Stream == null)
+            {
+                continue;
+            }
+            
+            // استخراج محتوا با استفاده از Composite Extractor
+            var extractionResult = await fileContentExtractor.ExtractRichAsync(
+                fileStream: fileDescriptor.Stream,
+                fileName: uploadedFile.FileName ?? uploadedFile.Name,
+                mimeType: fileDescriptor.MIMEType,
+                cancellationToken: cancellationToken
+            );
+            
+            if (extractionResult.IsValid())
+            {
+                // تبدیل به AIContent
+                var contents = extractionResult.ToAIContents();
+                fileContents.AddRange(contents);
+                
+                // ذخیره Metadata برای ردیابی
+                logger.LogInformation(
+                    "File {FileName} extracted successfully. Text: {HasText}, Images: {ImageCount}, Fallback: {UsedFallback}",
+                    uploadedFile.FileName,
+                    !string.IsNullOrEmpty(extractionResult.Text),
+                    extractionResult.Images.Count,
+                    extractionResult.UsedImageFallback
+                );
+            }
+            
+            fileDescriptor.Stream.Dispose();
+        }
+        catch (Exception ex)
+        {
+            logger.LogError(ex, "Failed to extract content from file: {FileName}", uploadedFile.FileName);
+        }
+    }
+}
+
+// تبدیل به ChatMessage با محتوای فایل‌ها
+var promptChatMessage = promptMessage.ToChatMessages(fileContents);
+ +
+ MODIFY + xAiModels/Extensions/XAiModelsExtensions.cs +
+ +

اصلاح متد ToChatMessages برای پشتیبانی از محتوای چندگانه:

+ +
public static ChatMessage ToChatMessages(
+    this XAiMessageDto source,
+    IList<AIContent> additionalContents = null
+)
+{
+    ChatMessage result = null;
+    
+    if (!source.IsNullOrDefault())
+    {
+        var contents = new List<AIContent>();
+        
+        // ۱. افزودن متن اصلی پیام
+        if (!string.IsNullOrWhiteSpace(source.Content))
+        {
+            contents.Add(new TextContent(source.Content));
+        }
+        
+        // ۲. ✅ افزودن محتوای فایل‌های ضمیمه (متن + تصویر + صوت)
+        if (additionalContents != null && additionalContents.HasChild())
+        {
+            contents.AddRange(additionalContents);
+        }
+        
+        result = new ChatMessage
+        {
+            AuthorName = 
+                source.Role == XAiChatRole.User &&
+                !source.Owner.IsNullOrDefault()
+                ? source.Owner.GetFullname()
+                : string.Empty,
+            Role = source.Role.ToChatRole(),
+            MessageId = source.Id.ToString(),
+            Contents = contents
+        };
+    }
+    
+    return result;
+}
+
+ + +
+

🔌 گام ۸: ثبت در DI و پیکربندی

+ +
+ MODIFY + xAiApi/Startup.cs +
+ +
public void ConfigureServices(IServiceCollection services)
+{
+    // ... (سایر ثبت‌ها)
+
+    // ✅ ثبت Extractor های فایل
+    services.AddSingleton<IXFileContentExtractor, XPlainTextFileContentExtractor>();
+    services.AddSingleton<IXFileContentExtractor, XDocxFileContentExtractor>();
+    services.AddSingleton<IXFileContentExtractor, XExcelFileContentExtractor>();
+    
+    // ✅ Extractor های جدید
+    services.AddSingleton<IXFileContentExtractor>(sp => 
+        new XImageFileContentExtractor(enableOcrFallback: true));
+    services.AddSingleton<IXFileContentExtractor>(sp => 
+        new XAudioFileContentExtractor(
+            whisperModelPath: "Models/ggml-base.bin",
+            language: "fa"
+        ));
+    services.AddSingleton<IXFileContentExtractor, XPdfFileContentExtractor>();
+    
+    // ✅ ثبت Composite Extractor
+    services.AddSingleton<IXFileContentExtractor, XFileContentExtractor>();
+
+    // ... (بقیه کدها)
+}
+ +
+ 💡 پیکربندی در appsettings.json: +
{
+  "AiApiConfiguration": {
+    "FileExtraction": {
+      "EnableOcrFallback": true,
+      "WhisperModelPath": "Models/ggml-base.bin",
+      "WhisperLanguage": "fa",
+      "MaxPdfPagesForImageFallback": 10,
+      "MinTextLengthThreshold": 50
+    }
+  }
+}
+
+
+ + +
+

🔄 گام ۹: جریان کامل پردازش

+ +
+ 📤 آپلود فایل + → + 💾 ذخیره در FileService + → + 🔍 تشخیص MIME Type + → + 🎯 انتخاب Extractor مناسب + → + 📊 استخراج محتوا (Text/Image/Audio) + → + 🔄 Fallback در صورت نیاز + → + 🎨 تبدیل به AIContent + → + 🤖 ارسال به LLM +
+ +

مثال‌های کاربردی:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
نوع فایلاستراتژی پردازشخروجی
📄 PDF متنیاستخراج متن با PdfPigText
📄 PDF اسکن‌شدهتبدیل صفحات به PNGImages[] (Vision)
🖼️ تصویر (PNG/JPG)ارسال مستقیم + OCRImage + Text (OCR)
🎙️ صوت (MP3/WAV)Whisper Speech-to-TextAudioTranscript
📝 Word (DOCX)استخراج متن با OpenXmlText
📊 Excel (XLSX)استخراج جدول با ExcelDataReaderText (Tabular)
+
+ + +
+

📋 گام ۱۰: خلاصه تغییرات و پکیج‌ها

+ +

فایل‌های جدید:

+ + + + + + + + + + + + + + + + + + + + + + + + + +
فایلتوضیح
xAiApi/Models/XFileExtractionResult.csمدل خروجی غنی با پشتیبانی از Text + Images + Audio
xAiApi/Interfaces/Extractors/IXImageFileContentExtractor.csInterface برای Image Extractor
xAiApi/Interfaces/Extractors/IXAudioFileContentExtractor.csInterface برای Audio Extractor
xAiApi/Providers/Extractors/XImageFileContentExtractor.csپیاده‌سازی Image Extractor با Vision + OCR
xAiApi/Providers/Extractors/XAudioFileContentExtractor.csپیاده‌سازی Audio Extractor با Whisper
+ +

فایل‌های اصلاح‌شده:

+ + + + + + + + + + + + + + + + + + + + + + + + + +
فایلتغییر
xAiApi/Interfaces/Extractors/IXFileContentExtractor.csافزودن متد ExtractRichAsync
xAiApi/Providers/Extractors/XPdfFileContentExtractor.csافزودن Fallback تصویری برای PDF های اسکن‌شده
xAiApi/Providers/XAIServiceBase.csاستفاده از ExtractRichAsync و پردازش چندوجهی
xAiModels/Extensions/XAiModelsExtensions.csپشتیبانی از additionalContents در ToChatMessages
xAiApi/Startup.csثبت Extractor های جدید در DI
+ +

پکیج‌های NuGet مورد نیاز:

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
پکیجکاربردوضعیت
UglyToad.PdfPigاستخراج متن و تصویر از PDF✅ نصب شده
DocumentFormat.OpenXmlاستخراج متن از DOCX✅ نصب شده
ExcelDataReaderاستخراج متن از Excel✅ نصب شده
System.Drawing.Commonکار با Bitmap برای PDF به تصویر❌ نیاز به نصب
Whisper.netSpeech-to-Text محلی❌ نیاز به نصب
Whisper.net.RuntimeRuntime برای Whisper❌ نیاز به نصب
NAudioتبدیل فرمت صوتی❌ نیاز به نصب
Tesseract (اختیاری)OCR برای تصاویر⚠️ اختیاری
+ +
+ ✅ دستاوردهای این ارتقا: +
    +
  • 🎨 پشتیبانی چندوجهی کامل: متن، تصویر، صوت
    • +
  • 📄 PDF هوشمند: تشخیص خودکار PDF متنی/اسکن‌شده
    • +
  • 🔄 Fallback خودکار: تبدیل به تصویر در صورت عدم وجود متن
    • +
  • 🎙️ Speech-to-Text محلی: بدون نیاز به API خارجی
    • +
  • 🖼️ Vision API Ready: سازگار با Gemini, GPT-4V, Qwen-VL
    • +
  • 🔌 Backward Compatible: کدهای موجود بدون تغییر کار می‌کنند
    • +
+
+
+ +
+ +
+

👨‍💻 توسعه‌دهنده: هادی خزاعی اصل

+

🏢 شرکت: فن آوران ساحر علم

+

📅 تاریخ: شنبه ۱۲ مهر ۱۴۰۵

+

+ 🎨 ارتقای سرویس استخراج محتوا - پشتیبانی از تصویر، صوت و PDF Fallback +

+
+ +
+ + \ No newline at end of file diff --git a/Documents/GeneratedContext/xDashboard-xAiApi-20261002_203559.txt b/Documents/GeneratedContext/xDashboard-xAiApi-20261002_203559.txt new file mode 100644 index 0000000..6604ea0 --- /dev/null +++ b/Documents/GeneratedContext/xDashboard-xAiApi-20261002_203559.txt @@ -0,0 +1,5010 @@ + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Program.cs + +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; + +namespace xAiApi +{ + public class Program + { + public static void Main(string[] args) + { + CreateHostBuilder(args).Build().Run(); + } + + public static IHostBuilder CreateHostBuilder(string[] args) => + Host.CreateDefaultBuilder(args) + .ConfigureWebHostDefaults(webBuilder => + { + webBuilder.UseStartup(); + }); + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Startup.cs + +using System; +using System.IO; +using xAiApi.DI; +using System.Linq; +using xAiModels.DI; +using xTagService.DI; +using System.Net.Http; +using xAiApi.Database; +using xHttpService.DI; +using xPushService.DI; +using xFileService.DI; +using xDataService.DI; +using Newtonsoft.Json; +using xAiApi.Providers; +using System.Reflection; +using xAiApi.Extensions; +using xStringService.DI; +using xAiApi.Interfaces; +using xStorageService.DI; +using xIdentityService.DI; +using xDataService.Models; +using xCommons.Extensions; +using xPushService.Helpers; +using xCommons.Configurations; +using System.Collections.Generic; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.Hosting; +using Microsoft.Extensions.Hosting; +using xAiApi.Interfaces.Extractors; +using Newtonsoft.Json.Serialization; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using IdentityModel.AspNetCore.OAuth2Introspection; + +namespace xAiApi +{ + public class Startup + { + // + #region Props ... + public IConfiguration Configuration { get; } + private readonly List databases; + private readonly XAiApiDatabaseDescriptor apiDatabaseDescriptor; + #endregion + + // + #region Constructor(s) ... + public Startup(IConfiguration configuration) + { + // + Configuration = configuration; + + // + // Create an Api Descriptor Class Instance ... + apiDatabaseDescriptor = new XAiApiDatabaseDescriptor(); + databases = new List + { + apiDatabaseDescriptor + }; + } + #endregion + + public void ConfigureServices(IServiceCollection services) + { + // + // Register Validation Provider and all XCommons Module Services ... + services.AddXCommons(); + + // + // Register App Configuration ... + services.AddXAppConfiguration(Configuration); + var appConfiguration = services.GetRegisteredService(); + + // + // Register Allowed Origins ... + services.AddXCors(appConfiguration.AllowedOrigins); + + // + // Register Swagger ... + var xmlFile = $"{Assembly.GetExecutingAssembly().GetName().Name}.xml"; + var xmlPath = Path.Combine(AppContext.BaseDirectory, xmlFile); + services.AddXSwagger(Configuration, xmlFilePath: xmlPath); + + // + // Register HttpService ... + services.AddXHttpService(Configuration); + + // + // Add Api V1 Versioning ... + services.AddXApiV1Versioning(); + + // + // OAuthIntrospectin Http Client Handler ... + // this is for handling SSLErrors ... + services.AddHttpClient(OAuth2IntrospectionDefaults.BackChannelHttpClientName) + .ConfigurePrimaryHttpMessageHandler(() => + { + return new HttpClientHandler + { + ServerCertificateCustomValidationCallback = (sender, certificate, chain, sslPolicyErrors) => true + }; + }); + + // + // Register Authorizations ... + services.AddXAuthorization(); + + // + services.AddControllers() + .AddNewtonsoftJson(x => + { + x.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore; + x.SerializerSettings.ContractResolver = new CamelCasePropertyNamesContractResolver(); + }); + + // + var lifeTime = ServiceLifetime.Scoped; + var repositoryType = xDataService.Constants.XRepositoryType.EF; + + // + // Register XIdentity Service ... + services.AddXIdentityService( + lifeTime: lifeTime, + configuration: Configuration + ); + + // + // Storage Service ... + services.AddXStorageService(Configuration); + + // + // Push Service ... + services.AddXPushService( + lifeTime: lifeTime, + config: Configuration + ); + + // + // Preparing DataBases ... + if (databases.HasChild()) + { + // + databases + .ToList() + .ForEach(database => + { + // + // Register DataBase ... + services.AddXDatabase( + lifeTime: lifeTime, + descriptor: database, + configuration: Configuration + ); + }); + } + + // + // Register String Service ... + services.AddXStringService( + lifeTime: lifeTime, + repositoryType: repositoryType + ); + + // + // Register Tag Service ... + services.AddXTagService( + lifeTime: lifeTime, + repositoryType: repositoryType + ); + + // + // Register File Service ... + services.AddXFileService( + lifeTime: lifeTime, + repositoryType: repositoryType + ); + + // + // Register AiApi Data Service ... + services.AddXAiDataService( + lifeTime: lifeTime, + repositoryType: repositoryType + ); + + // + // Register XAiApi Configuration ... + services.AddXAiApiConfiguration(Configuration); + + // + // Register File Content Extractors ... + // services.AddSingleton(); + // services.AddSingleton(); + services.AddSingleton(); + + // + // Register Default Ai Service ... + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + } + + public void Configure(IApplicationBuilder app, IWebHostEnvironment env) + { + // + // Configure the HTTP request pipeline. + var isDevelopmentEnvironment = false; + if (env.IsDevelopment()) + { + isDevelopmentEnvironment = true; + app.UseDeveloperExceptionPage(); + } + + // + // Use Swagger Middleware ... + app.UseXSwagger(); + + // + // Force Https Redirection ... + app.UseHttpsRedirection(); + + // + // Using Cors ... + app.UseXCors(); + + // + // Using Routing ... + app.UseRouting(); + + // + // Use xIdentityService ... + app.UseXIdentityService(); + + // + // Use Authorization ... + app.UseAuthorization(); + + // + app.UseEndpoints(endpoints => + { + endpoints.MapControllers(); + }); + + // + // Use Storage Service ... + app.UseXStorageService(); + + // + // Use PushService ... + var pushHelper = new XPushServiceHelper(); + app.UseXPushService(pushHelper); + + // + // Using DataBase ... + if (databases.HasChild()) + { + // + databases + .ToList() + .ForEach(database => + { + // + app.UseXDatabase( + descriptor: database, + isDevelopmentEnvironment: isDevelopmentEnvironment + ); + }); + } + + // + // Using String Service ... + app.UseXStringService( + isDevelopmentEnvironment: isDevelopmentEnvironment + ); + + // + // Using XTag Service ... + app.UseXTagService( + isDevelopmentEnvironment: isDevelopmentEnvironment + ); + + // + // Using File Service ... + app.UseXFileService( + isDevelopmentEnvironment: isDevelopmentEnvironment + ); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Configurations\XAiApiConfiguration.cs + +using xAiModels.Models; + +namespace xAiApi.Configurations +{ + public class XAiApiConfiguration + { + /// + /// Allowed Models to Use ... + /// + public XAiModelDescriptor[] Models { get; set; } = []; + + /// + /// Allowed Prompts to Use ... + /// + public XAiPromptDescriptor[] Prompts { get; set; } = []; + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Constants\ConfigurationNodeNames.cs + +namespace xAiApi.Constants +{ + public partial struct ConfigurationNodeNames + { + public const string AI_API_NODE = "AiApiConfiguration"; + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Constants\XAiApiConstants.cs + +namespace xAiApi.Constants +{ + public struct XAiApiConstants + { + // + // Database Descriptor ... + public const string XAiApiDbIdentifier = "xAiApiDb"; + public const string XAiApiMigrationsAssemblyIdentifier = "xAiApi"; + + // + public const string XDILogTag = "XAiApi"; + public const string XAiApiIntroductionPromptName = "Introduction"; + + // + public const string XAiDefaultModelName = "Gemma"; + public const string XAiEmbeddingModelName = "Embed"; + public const string XAiDefaultThinkingModelName = "Qwen"; + public const string XAiEmbeddingPromptName = "Embedding"; + + // + public const string XOpenAINoKey = "no-key-required"; + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\DefaultAiController.cs + +using xAiApi.Interfaces; +using xCommons.Providers; +using xCommons.Configurations; +using Microsoft.AspNetCore.Mvc; +using xIdentityService.Interfaces; +using Microsoft.Extensions.Logging; +using Microsoft.AspNetCore.Authorization; + +namespace xAiApi.Controllers +{ + [Authorize] + [Route("[controller]")] + public class DefaultAiController : XAiServiceControllerBase + { + public DefaultAiController( + ILogger logger, + IXDefaultAiService aiService, + XAppConfiguration appConfiguration, + IXIdentityProvider identityProvider, + XValidationProvider validationProvider + ) : base( + logger, + aiService, + appConfiguration, + identityProvider, + validationProvider + ) + { } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\DefaultEmbeddingController.cs + +using xAiApi.Interfaces; +using xCommons.Providers; +using xCommons.Configurations; +using Microsoft.AspNetCore.Mvc; +using xIdentityService.Interfaces; +using Microsoft.Extensions.Logging; +using Microsoft.AspNetCore.Authorization; + +namespace xAiApi.Controllers +{ + [Authorize] + [Route("[controller]")] + public class DefaultEmbeddingController : XAiEmbeddingServiceControllerBase + { + public DefaultEmbeddingController( + ILogger logger, + IXDefaultEmbeddingService aiService, + XAppConfiguration appConfiguration, + IXIdentityProvider identityProvider, + XValidationProvider validationProvider + ) : base( + logger, + aiService, + appConfiguration, + identityProvider, + validationProvider + ) + { } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\DefaultThinkingAiController.cs + +using xAiApi.Interfaces; +using xCommons.Providers; +using xCommons.Configurations; +using xIdentityService.Interfaces; +using Microsoft.Extensions.Logging; + +namespace xAiApi.Controllers +{ + public class DefaultThinkingAiController : XAiServiceControllerBase + { + public DefaultThinkingAiController( + ILogger logger, + IXDefaultThinkingAiService aiService, + XAppConfiguration appConfiguration, + IXIdentityProvider identityProvider, + XValidationProvider validationProvider + ) : base( + logger, + aiService, + appConfiguration, + identityProvider, + validationProvider + ) + { } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\StartupController.cs + +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.Logging; +using xCommons.Configurations; +using xCommons.Controllers; +using xCommons.Providers; + +namespace xAiApi.Controllers +{ + /// + /// Runing when application start ... + /// + [Route("")] + [AllowAnonymous] + public class StartupController : XBaseController + { + + public StartupController( + ILogger logger, + XAppConfiguration appConfiguration, + XValidationProvider validationProvider + ) : base( + logger, + appConfiguration, + validationProvider + ) + { } + + /// + /// Show Configured Welcome Message + /// + /// + [HttpGet("")] + [AllowAnonymous] + public virtual ActionResult Index() + { + // + var controllerName = GetControllerName(); + var message = $"{AppConfiguration.WelcomeMessage}"; + + // + return Ok(message); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\XAiEmbeddingServiceControllerBase.cs + +using System; +using System.Threading; +using xAiApi.Interfaces; +using xCommons.Providers; +using System.Threading.Tasks; +using xCommons.Configurations; +using Microsoft.AspNetCore.Mvc; +using System.Collections.Generic; +using xIdentityService.Interfaces; +using Microsoft.Extensions.Logging; +using Microsoft.AspNetCore.Authorization; + +namespace xAiApi.Controllers +{ + [Authorize] + [Route("[controller]")] + public abstract class XAiEmbeddingServiceControllerBase : XAiServiceControllerBase, IXAiEmbeddingServiceControllerBase + where TAiService : IXAiEmbeddingServiceBase + { + private readonly TAiService aiService; + + protected XAiEmbeddingServiceControllerBase( + ILogger logger, + TAiService aiService, + XAppConfiguration appConfiguration, + IXIdentityProvider identityProvider, + XValidationProvider validationProvider + ) : base( + logger, + aiService, + appConfiguration, + identityProvider, + validationProvider + ) + { + this.aiService = aiService; + } + + /// + /// Request For Embedding Specified String ... + /// + /// + /// + /// + [AllowAnonymous] + [HttpGet("Embedding")] + public async Task> Embedding( + [FromQuery] string content, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + ValidationProvider.NotEmpty(content); + + // + var result = await aiService.Embedding( + content: content, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Request For Embedding Specified Batch ... + /// + /// + /// + /// + [AllowAnonymous] + [HttpGet("Embeddings")] + public async Task>> Embeddings( + [FromQuery] string[] batch, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + ValidationProvider.NotZeroChilds(batch); + + // + var result = await aiService.Embeddings( + batch: batch, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Controllers\XAiServiceControllerBase.cs + +using System; +using xModels.Dtos; +using System.Text.Json; +using System.Threading; +using xAiModels.Models; +using xAiApi.Interfaces; +using xCommons.Providers; +using xCommons.Extensions; +using xAiModels.Extensions; +using xAiModels.Models.Dtos; +using xExceptions.Constants; +using System.Threading.Tasks; +using xCommons.Configurations; +using Microsoft.AspNetCore.Mvc; +using Microsoft.AspNetCore.Http; +using System.Collections.Generic; +using xIdentityService.Interfaces; +using Microsoft.Extensions.Logging; +using xIdentityService.Controllers; +using Microsoft.AspNetCore.Authorization; + +namespace xAiApi.Controllers +{ + [Authorize] + [Route("[controller]")] + public abstract class XAiServiceControllerBase : XBaseIdentityApiV1Controller, IXAiServiceControllerBase + where TAiService : IXAiServiceBase + { + private readonly TAiService aiService; + + protected XAiServiceControllerBase( + ILogger logger, + TAiService aiService, + XAppConfiguration appConfiguration, + IXIdentityProvider identityProvider, + XValidationProvider validationProvider + ) : base( + logger, + appConfiguration, + identityProvider, + validationProvider + ) + { + this.aiService = aiService; + } + + // + #region Actions ... + /// + /// Ask Question and Get Response ... + /// + /// + /// + /// + [AllowAnonymous] + [HttpGet("AskText")] + public virtual async Task> AskText( + [FromQuery] string question, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + ValidationProvider.NotEmpty(question); + + // + var result = await aiService.AskTextAsync( + prompt: question, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [AllowAnonymous] + [HttpGet("AskTextStream")] + public virtual async Task AskTextStream( + [FromQuery] string question, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + ValidationProvider.NotEmpty(question); + + // + // Configure SSE Response Headers ... + Response.Headers["X-Accel-Buffering"] = "no"; + Response.Headers["Connection"] = "keep-alive"; + Response.Headers["Content-Type"] = "text/event-stream"; + Response.Headers["Cache-Control"] = "no-cache, no-store"; + + // + // Disable response buffering ... + var responseBodyFeature = HttpContext.Features + .Get(); + responseBodyFeature?.DisableBuffering(); + + // + // Stream Response ... + var jsonOptions = new JsonSerializerOptions + { + Encoder = System.Text.Encodings.Web.JavaScriptEncoder.UnsafeRelaxedJsonEscaping + }; + + // + var enumerable = aiService.AskTextAsEnumerable( + prompt: question, + cancellationToken: cancellationToken + ); + + // + await foreach (var chunk in enumerable) + { + // + // Skip empty chunks ... + if (string.IsNullOrEmpty(chunk)) + { + continue; + } + + // + // Build SSE message ... + // Format: data: \n\n + // var payload = JsonSerializer.Serialize( + // new { text = chunk }, + // jsonOptions + // ); + var payload = chunk; + + // + // Write SSE Event ... + await Response.WriteAsync($"{payload}\n\n", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + + // + await Response.Body.FlushAsync(cancellationToken); + } + catch (Exception) + { + await Response.Body.FlushAsync(); + } + } + #endregion + + // + #region Loaders ... + /// + /// Retrieve AI Projects ... + /// + /// + /// + /// + [HttpGet("Projects")] + public async Task>> LoadProjects( + [FromQuery] bool includeDefaults = false, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.LoadProjects( + ownerId: userInfo.UserId, + connectionId: connectionId, + includeDefaults: includeDefaults, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Retrieve Specified AI Project's Conversations ... + /// + /// + /// + /// + [HttpGet("Projects/{projectId}/Conversations")] + public async Task>> LoadConversations( + [FromRoute] Guid projectId, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (projectId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.LoadConversations( + projectId: projectId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Retrieve Specified AI Conversation's Messages ... + /// + /// + /// + /// + [HttpGet("Conversations/{conversationId}/Messages")] + public async Task>> LoadMessages( + [FromRoute] Guid conversationId, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (conversationId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + var result = await aiService.LoadMessages( + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + #endregion + + // + #region Queries ... + /// + /// Query Projects ... + /// + /// + /// + /// + /// + /// + [HttpGet("Projects/Query")] + public async Task>> QueryProjects( + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.QueryProjects( + query: query, + ownerId: userInfo.UserId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Query Conversations ... + /// + /// + /// + /// + /// + [HttpGet("Projects/{projectId}/Conversations/Query")] + public async Task>> QueryConversations( + [FromRoute] Guid projectId, + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.QueryConversations( + query: query, + projectId: projectId, + ownerId: userInfo.UserId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Query Messages ... + /// + /// + /// + /// + /// + [HttpGet("Conversations/{conversationId}/Messages/Query")] + public async Task>> QueryMessages( + [FromRoute] Guid conversationId, + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.QueryMessages( + query: query, + ownerId: userInfo.UserId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + #endregion + + // + #region Creators ... + /// + /// Create an AI Project ... + /// + /// + /// + /// + [HttpPost("Projects")] + public async Task> CreateProject( + [FromBody] XAiProjectDto item, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (item.IsNullOrDefault()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + // Update Required Params ... + item.OwnerId = userInfo.UserId; + if (!item.IsValidForAddOrUpdate()) + { + XException.InvalidData.Throw(); + } + + // + var result = await aiService.CreateProject( + item: item, + ownerId: userInfo.UserId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Create an AI Conversation ... + /// + /// + /// + /// + /// + [HttpPost("Projects/{projectId}/Conversations")] + public async Task> CreateConversation( + [FromRoute] Guid projectId, + [FromBody] XAiConversationDto item, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (projectId.IsDefaultGuid() || + item.IsNullOrDefault()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + // Update Required Params ... + item.OwnerId = userInfo.UserId; + if (!item.IsValidForAddOrUpdate()) + { + XException.InvalidData.Throw(); + } + + // + var result = await aiService.CreateConversation( + item: item, + projectId: projectId, + ownerId: userInfo.UserId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + #endregion + + // + #region Data Actions ... + /// + /// Ask Question and Get Response ... + /// + /// + /// + /// + [HttpPost("Ask")] + public async Task> Ask( + [FromBody] XAiResponseRequest request, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (!request.IsValid()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + // TODO: Reading Fiels Form Collection and Attach it ... + var result = await aiService.AskAsync( + files: null, + prompt: request.Prompt, + ownerId: userInfo.UserId, + connectionId: connectionId, + projectId: request.ProjectId, + cancellationToken: cancellationToken, + conversationId: request.ConversationId + ); + + // + return Ok(result); + } + catch (Exception ex) + { + // + var result = GetExceptionActionResult(ex); + return result; + } + } + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [HttpPost("AskStream")] + public async Task AskStream( + [FromBody] XAiResponseRequest request, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (!request.IsValid()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + // Configure SSE Response Headers ... + Response.Headers["X-Accel-Buffering"] = "no"; + Response.Headers["Connection"] = "keep-alive"; + Response.Headers["Content-Type"] = "text/event-stream"; + Response.Headers["Cache-Control"] = "no-cache, no-store"; + + // + // Disable response buffering ... + var responseBodyFeature = HttpContext.Features + .Get(); + responseBodyFeature?.DisableBuffering(); + + // + // TODO: Reading Fiels Form Collection and Attach it ... + var enumerable = aiService.AskAsEnumerable( + files: null, + prompt: request.Prompt, + ownerId: userInfo.UserId, + connectionId: connectionId, + projectId: request.ProjectId, + cancellationToken: cancellationToken, + conversationId: request.ConversationId + ); + + // + await foreach (var resp in enumerable) + { + // + // Skip Empty Responses ... + if (resp.IsNullOrDefault()) + { + continue; + } + + // + var payload = resp.ToJSON(camelCase: true); + + // + // Write SSE Event ... + await Response.WriteAsync($"{payload}", cancellationToken); + await Response.Body.FlushAsync(cancellationToken); + } + + // + await Response.Body.FlushAsync(cancellationToken); + } + catch (Exception) + { + await Response.Body.FlushAsync(); + } + } + + /// + /// Ask Question with Files and Get Response ... + /// + /// + /// + /// + [HttpPost("AskWithFiles")] + [Consumes("multipart/form-data")] + public async Task> AskWithFiles( + [FromForm] XAiResponseRequest request, + [FromForm] IFormFileCollection files, + CancellationToken cancellationToken = default + ) + { + // + try + { + // + // Validate ... + if (!request.IsValid()) + { + XException.InvalidArgs.Throw(); + } + + // + var userInfo = await GetUserInfo(); + var connectionId = GetConnectionId(); + + // + var result = await aiService.AskAsync( + files: files, + userInfo: userInfo, + prompt: request.Prompt, + ownerId: userInfo.UserId, + connectionId: connectionId, + projectId: request.ProjectId, + cancellationToken: cancellationToken, + conversationId: request.ConversationId + ); + + // + return Ok(result); + } + catch (Exception ex) + { + var result = GetExceptionActionResult(ex); + return result; + } + } + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Database\XAiApiDatabaseDescriptor.cs + +using xAiApi.Constants; +using xDataService.Interfaces; +using xDataService.Models; + +namespace xAiApi.Database +{ + /// + /// Ai Api Database Descriptor ... + /// + public class XAiApiDatabaseDescriptor : XDatabaseDescriptor, IXDatabaseDescriptor + { + public XAiApiDatabaseDescriptor() : base( + name: XAiApiConstants.XAiApiDbIdentifier, + repositories: [] + ) + { + OptionsBuilder = b => b.MigrationsAssembly(XAiApiConstants.XAiApiMigrationsAssemblyIdentifier); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Database\XAiApiDbContext.cs + +using Microsoft.EntityFrameworkCore; +using xAiApi.Constants; +using xAiModels.Providers; +using xDataService.Configuration; +using xDataService.Db; +using xDataService.Providers; +using xFileService.Providers; +using xStringService.Providers; +using xTagService.Providers; + +namespace xAiApi.Database +{ + /// + /// Db Context of AiApi ... + /// + public class XAiApiDbContext : XDbContext + { + // + #region DbSets ... + #endregion + + // + #region Navigation Property Entities ... + #endregion + + // + #region Constructor ... + public XAiApiDbContext( + DbContextOptions options, + XDataServiceConfiguration config, + XEntityRegisterarHandlerService dynamicEntityHandlers + ) : base( + config: config, + options: options, + database: XAiApiConstants.XAiApiDbIdentifier, + dynamicEntityHandlers: dynamicEntityHandlers, + dynamicEntityRegisterarsNames: [ + nameof(XTagEntityRegisterar), + nameof(XFileEntityRegisterar), + nameof(XStringEntityRegisterar), + nameof(XAiMessageEntityRegisterar), + nameof(XAiProjectEntityRegisterar), + nameof(XAiConversationEntityRegisterar), + ] + ) + { } + #endregion + + // + #region Overrides ... + // + public override void OnXModelCreating(ModelBuilder modelBuilder) + { } + + // + public override void OnXConfiguring(DbContextOptionsBuilder optionsBuilder) + { } + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\DI\XDIHelperExtension.cs + +using System; +using xAiApi.Constants; +using xAiApi.Extensions; +using xCommons.Extensions; +using xAiApi.Configurations; +using xExceptions.Constants; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; + +namespace xAiApi.DI +{ + public static partial class XDIHelperExtension + { + /// + /// Extract AI Configuration ... + /// + /// + /// + public static XAiApiConfiguration GetXAiApiConfiguration(this IConfiguration source) + { + // + XAiApiConfiguration result = null; + + // + var section = source.GetSection(ConfigurationNodeNames.AI_API_NODE); + if (!section.IsNull()) + { + result = section.Get(); + } + + // + return result; + } + + /// + /// Register AiApi Configuration ... + /// + /// + /// + public static void AddXAiApiConfiguration( + this IServiceCollection source, + IConfiguration configuration + ) + { + // + var apiConfig = configuration.GetXAiApiConfiguration(); + source.AddXAiApiConfiguration(apiConfig); + } + + /// + /// Register AiApi Configuration ... + /// + /// + /// + public static void AddXAiApiConfiguration( + this IServiceCollection source, + XAiApiConfiguration configuration + ) + { + // + if (!configuration.IsValid()) + { + XException.InvalidConfiguration.Throw(); + } + + // + source.AddSingleton(configuration); + } + + // + #region Private ... + /// + /// a LogTag for Service ... + /// + private static string XLogTag = XAiApiConstants.XDILogTag; + + /// + /// print a log in Console ... + /// + /// + private static void Log(string message) + { + Console.WriteLine($"{XLogTag} => {message}"); + } + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Extensions\XModelsExtensions.cs + +using System.Linq; +using xAiApi.Constants; +using xAiModels.Models; +using xCommons.Extensions; +using xAiModels.Extensions; +using xAiApi.Configurations; +using System.Collections.Generic; + +namespace xAiApi.Extensions +{ + public static class XModelsExtensions + { + /// + /// Validate Configurations ... + /// + /// + /// + public static bool IsValid(this XAiApiConfiguration source) + { + // + var result = + !source.IsNullOrDefault() && + source.Models.HasChild() && + source.Models.All(m => m.IsValid()) && + source.Prompts.All(m => m.IsValid()) && + source.Prompts.Any(p => p.Name + .Equals( + XAiApiConstants.XAiApiIntroductionPromptName, + System.StringComparison.InvariantCultureIgnoreCase)); + + // + return result; + } + + /// + /// Extract Default Model Descriptor ... + /// + /// + /// + public static XAiModelDescriptor GetDefaultModel(this XAiApiConfiguration source) + { + // + XAiModelDescriptor result = null; + + // + if (source.IsValid()) + { + result = source.Models[0]; + } + + // + return result; + } + + /// + /// Retrieve Specified Model from Config ... + /// + /// + /// + /// + public static XAiModelDescriptor GetModel( + this XAiApiConfiguration source, + string name + ) + { + // + XAiModelDescriptor result = null; + + // + if (source.IsValid()) + { + // + if (name.IsNullOrEmpty()) + { + result = source.GetDefaultModel(); + } + else + { + result = source.Models + .FirstOrDefault(m => m.Name + .Equals(name, System.StringComparison.InvariantCultureIgnoreCase)); + } + } + + // + return result; + } + + /// + /// Retrieve Introduction Prompt ... + /// + /// + /// + public static XAiPromptDescriptor GetIntroductionPrompt(this XAiApiConfiguration source) + { + // + XAiPromptDescriptor result = null; + + // + if (source.IsValid()) + { + // + result = source.Prompts + .FirstOrDefault(p => p.Name + .Equals( + XAiApiConstants.XAiApiIntroductionPromptName, + System.StringComparison.InvariantCultureIgnoreCase)); + } + + // + return result; + } + + /// + /// Select Specified Prompts and Parse it's Template + /// and Generate Message ... + /// + /// + /// + /// + /// + public static string GetPrompt( + this XAiApiConfiguration source, + string name = null, + IDictionary @params = null + ) + { + // + var result = string.Empty; + + // + // Normalize ... + if (name.IsNullOrEmpty()) + { + name = XAiApiConstants.XAiApiIntroductionPromptName; + } + + // + if (source.IsValid()) + { + // + var prompt = source.Prompts + .FirstOrDefault(p => p.Name + .Equals( + name, + System.StringComparison.InvariantCultureIgnoreCase + ) + ); + if (!prompt.IsNullOrDefault()) + { + // + result = prompt.Template; + + // + // Handle Params ... + if (!@params.IsNullOrDefault() && @params.HasChild()) + { + // + // Select Params ... + var existsParams = prompt.Params + .Where(p => @params.Any(pr => pr.Key == p.Name)) + .Distinct() + .ToList(); + + // + // Replace Selected Params ... + existsParams + .ForEach(pk => + { + result = result.Replace(pk.Param, @params[pk.Name]); + }); + } + } + } + + // + return result; + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Extensions\XProgramExtensions.cs + +using System.Collections.Generic; +using Microsoft.AspNetCore.Authorization; +using Microsoft.AspNetCore.Builder; +using Microsoft.AspNetCore.HttpOverrides; +using Microsoft.Extensions.DependencyInjection; +using xCommons.Authorization; +using xCommons.Extensions; +using xIdentityHelper; + +namespace xAiApi.Extensions +{ + public static class XProgramExtensions + { + /// + /// Register Authorization Policies + /// + /// + /// + public static void AddXAuthorization( + this IServiceCollection services, + IDictionary policies = null + ) + { + // + services.AddAuthorization(options => + { + // + if (!policies.IsNull()) + { + // + // Fill Policies ... + using (var policiesEnumerator = policies.GetEnumerator()) + { + while (policiesEnumerator.MoveNext()) + { + var policyDescriptor = policiesEnumerator.Current; + options.AddPolicy(policyDescriptor.Key, policyDescriptor.Value); + } + } + } + + // + var xPolicies = XAuthorizationHelper.GetXAuthorizationPolicies(); + if (!xPolicies.IsNull()) + { + // + // Fill Policies ... + using (var policiesEnumerator = xPolicies.GetEnumerator()) + { + while (policiesEnumerator.MoveNext()) + { + var policyDescriptor = policiesEnumerator.Current; + options.AddPolicy(policyDescriptor.Key, policyDescriptor.Value); + } + } + } + }); + + // + services.AddSingleton(); + } + + /// + /// Use Forward Headers Options for resolving behind a proxy issues ... + /// + /// + /// + public static void UseXForwardOptions(this IApplicationBuilder source) + { + // + var forwardOptions = new ForwardedHeadersOptions + { + ForwardedHeaders = ForwardedHeaders.XForwardedFor | ForwardedHeaders.XForwardedProto, + RequireHeaderSymmetry = false + }; + + // + forwardOptions.KnownNetworks.Clear(); + forwardOptions.KnownProxies.Clear(); + + // + // ref: https://github.com/aspnet/Docs/issues/2384 + source.UseForwardedHeaders(forwardOptions); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXAiEmbeddingServiceBase.cs + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Extensions.AI; +using OllamaSharp; + +namespace xAiApi.Interfaces +{ + /// + /// Base Embedding Provider Service ... + /// + public interface IXAiEmbeddingServiceBase : IXAiServiceBase + { + /// + /// Embedding Specified String ... + /// + /// + /// + /// + Task Embedding( + string content, + CancellationToken cancellationToken = default + ); + + /// + /// Embedding Specified Batch ... + /// + /// + /// + /// + Task> Embeddings( + string[] batch, + CancellationToken cancellationToken = default + ); + + /// + /// Embedding Generator ... + /// + /// + IEmbeddingGenerator> GetEmbeddingGenerator(); + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXAiEmbeddingServiceControllerBase.cs + +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; + +namespace xAiApi.Interfaces +{ + public interface IXAiEmbeddingServiceControllerBase : IXAiServiceControllerBase + { + /// + /// Request For Embedding Specified String ... + /// + /// + /// + /// + [HttpGet("Embedding")] + Task> Embedding( + [FromQuery] string content, + CancellationToken cancellationToken = default + ); + + /// + /// Request For Embedding Specified Batch ... + /// + /// + /// + /// + [HttpGet("Embeddings")] + Task>> Embeddings( + [FromQuery] string[] batch, + CancellationToken cancellationToken = default + ); + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXAiServiceBase.cs + +using System; +using xModels.Dtos; +using System.Net.Http; +using System.Threading; +using xAiModels.Models; +using xAiModels.Models.Dtos; +using System.Threading.Tasks; +using xIdentityModels.Models; +using Microsoft.Extensions.AI; +using Microsoft.AspNetCore.Http; +using System.Collections.Generic; + +namespace xAiApi.Interfaces +{ + /// + /// Describe a Base Service Actions of any Ai Based Services ... + /// + public interface IXAiServiceBase : IDisposable + { + /// + /// Chat Options ... + /// + ChatOptions Options { get; } + + /// + /// Descriptor of Models which used in Service ... + /// + XAiModelDescriptor Descriptor { get; } + + // + #region Actions ... + /// + /// Base Method for Communicate with LLM ... + /// + /// + /// + /// + Task AskTextAsync( + string prompt, + CancellationToken cancellationToken = default + ); + + /// + /// Base Method for Communicate with LLM ... + /// + /// + /// + /// + IAsyncEnumerable AskTextAsEnumerable( + string prompt, + CancellationToken cancellationToken = default + ); + + /// + /// Get LLM Client instance for Communicating with LLM ... + /// + /// + IChatClient GetClient(); + + /// + /// Create Custom HttpClient for Communicating with LLM API ... + /// + /// + /// + HttpClient GetHttpClient(string url = null); + + /// + /// Prepare a Message History List for LLM Communication based on Prompt ... + /// + /// + /// + /// + IList GetHistory( + string prompt = null, + bool forcePrompt = true + ); + + /// + /// Prepare a Message History List for LLM Communication based on Prompt ... + /// + /// + /// + /// + IList GetHistory( + ChatMessage message = null, + bool forceMessage = true + ); + #endregion + + // + #region Loaders ... + /// + /// Load AI Projects ... + /// + /// + /// + /// + /// + /// + Task> LoadProjects( + string ownerId, + string connectionId = null, + bool includeDefaults = false, + CancellationToken cancellationToken = default + ); + + /// + /// Load Conversations of Specified AI Project ... + /// + /// + /// + /// + /// + Task> LoadConversations( + Guid projectId, + string connectionId = null, + CancellationToken cancellationToken = default + ); + + /// + /// Load Messages of Specified Conversation ... + /// + /// + /// + /// + Task> LoadMessages( + Guid conversationId, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Queries ... + /// + /// Query Projects ... + /// + /// + /// + /// + /// + Task> QueryProjects( + string ownerId, + XQuery query, + CancellationToken cancellationToken = default + ); + + /// + /// Query Conversations ... + /// + /// + /// + /// + /// + /// + Task> QueryConversations( + string ownerId, + Guid projectId, + XQuery query, + CancellationToken cancellationToken = default + ); + + /// + /// Query Messages ... + /// + /// + /// + /// + /// + /// + Task> QueryMessages( + string ownerId, + Guid conversationId, + XQuery query, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Creators ... + /// + /// Create an AI Project ... + /// + /// + /// + /// + /// + /// + Task CreateProject( + string ownerId, + XAiProjectDto item, + string connectionId = null, + CancellationToken cancellationToken = default + ); + + /// + /// Create an AI Conversation ... + /// + /// + /// + /// + /// + /// + /// + Task CreateConversation( + string ownerId, + Guid projectId, + XAiConversationDto item, + string connectionId = null, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Data Actions ... + /// + /// Ask a Question as Prompt and Retrieve Response ... + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + Task AskAsync( + string prompt, + string ownerId, + Guid projectId, + Guid conversationId, + string connectionId = null, + IFormFileCollection files = null, + XUserClaimsInfoDto userInfo = null, + CancellationToken cancellationToken = default + ); + + /// + /// Ask a Question as Prompt and Retrieve Response as Async Enumerable ... + /// + /// + /// + /// + /// + /// + /// + /// + /// + IAsyncEnumerable AskAsEnumerable( + string prompt, + string ownerId, + Guid projectId, + Guid conversationId, + string connectionId = null, + IFormFileCollection files = null, + CancellationToken cancellationToken = default + ); + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXAiServiceControllerBase.cs + +using System; +using xModels.Dtos; +using System.Threading; +using xAiModels.Models; +using xAiModels.Models.Dtos; +using System.Threading.Tasks; +using Microsoft.AspNetCore.Mvc; +using System.Collections.Generic; +using Microsoft.AspNetCore.Http; + +namespace xAiApi.Interfaces +{ + public interface IXAiServiceControllerBase + { + // + #region Actions ... + /// + /// Ask Question and Get Response ... + /// + /// + /// + /// + [HttpGet("AskText")] + Task> AskText( + [FromQuery] string question, + CancellationToken cancellationToken = default + ); + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [HttpGet("AskTextStream")] + Task AskTextStream( + [FromQuery] string question, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Loaders ... + /// + /// Retrieve AI Projects ... + /// + /// + /// + /// + [HttpGet("Projects")] + Task>> LoadProjects( + [FromQuery] bool includeDefaults = false, + CancellationToken cancellationToken = default + ); + + /// + /// Retrieve Specified AI Project's Conversations ... + /// + /// + /// + /// + [HttpGet("Projects/{projectId}/Conversations")] + Task>> LoadConversations( + [FromRoute] Guid projectId, + CancellationToken cancellationToken = default + ); + + /// + /// Retrieve Specified AI Conversation's Messages ... + /// + /// + /// + /// + [HttpGet("Conversations/{conversationId}/Messages")] + Task>> LoadMessages( + [FromRoute] Guid conversationId, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Queries ... + /// + /// Query Projects ... + /// + /// + /// + /// + /// + /// + [HttpGet("Projects/Query")] + Task>> QueryProjects( + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ); + + /// + /// Query Conversations ... + /// + /// + /// + /// + /// + [HttpGet("Projects/{projectId}/Conversations/Query")] + Task>> QueryConversations( + [FromRoute] Guid projectId, + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ); + + /// + /// Query Messages ... + /// + /// + /// + /// + /// + [HttpGet("Conversations/{conversationId}/Messages/Query")] + Task>> QueryMessages( + [FromRoute] Guid conversationId, + [FromQuery] XQuery query, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Creators ... + /// + /// Create an AI Project ... + /// + /// + /// + /// + [HttpPost("Projects")] + Task> CreateProject( + [FromBody] XAiProjectDto item, + CancellationToken cancellationToken = default + ); + + /// + /// Create an AI Conversation ... + /// + /// + /// + /// + /// + [HttpPost("Projects/{projectId}/Conversations")] + Task> CreateConversation( + [FromRoute] Guid projectId, + [FromBody] XAiConversationDto item, + CancellationToken cancellationToken = default + ); + #endregion + + // + #region Data Actions ... + /// + /// Ask Question and Get Response ... + /// + /// + /// + /// + [HttpPost("Ask")] + Task> Ask( + [FromBody] XAiResponseRequest request, + CancellationToken cancellationToken = default + ); + + /// + /// Ask Question and Get Response Stream ... + /// + /// + /// + /// + [HttpPost("AskStream")] + Task AskStream( + [FromBody] XAiResponseRequest request, + CancellationToken cancellationToken = default + ); + + /// + /// Ask Question with Files and Get Response ... + /// + /// + /// + /// + [HttpPost("AskWithFiles")] + [Consumes("multipart/form-data")] + Task> AskWithFiles( + [FromForm] XAiResponseRequest request, + [FromForm] IFormFileCollection files, + CancellationToken cancellationToken = default + ); + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXDefaultAiService.cs + +namespace xAiApi.Interfaces +{ + public interface IXDefaultAiService : IXAiServiceBase + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXDefaultEmbeddingService.cs + +namespace xAiApi.Interfaces +{ + public interface IXDefaultEmbeddingService : IXAiEmbeddingServiceBase + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\IXDefaultThinkingAiService.cs + +namespace xAiApi.Interfaces +{ + public interface IXDefaultThinkingAiService: IXAiServiceBase + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\Extractors\IXDocxFileContentExtractor.cs + +namespace xAiApi.Interfaces.Extractors +{ + public interface IXDocxFileContentExtractor : IXFileContentExtractor + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\Extractors\IXExcelFileContentExtractor.cs + +namespace xAiApi.Interfaces.Extractors +{ + public interface IXExcelFileContentExtractor : IXFileContentExtractor + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\Extractors\IXFileContentExtractor.cs + +using System.IO; +using System.Threading; +using System.Threading.Tasks; + +namespace xAiApi.Interfaces.Extractors +{ + /// + /// a Service for Extracting File Content ... + /// + public interface IXFileContentExtractor + { + /// + /// Check if this extractor supports the specified MIME type ... + /// + bool CanExtract(string mimeType); + + /// + /// Extract text content from file stream ... + /// + Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ); + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\Extractors\IXPdfFileContentExtractor.cs + +namespace xAiApi.Interfaces.Extractors +{ + public interface IXPdfFileContentExtractor : IXFileContentExtractor + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Interfaces\Extractors\IXPlainTextFileContentExtractor.cs + +namespace xAiApi.Interfaces.Extractors +{ + public interface IXPlainTextFileContentExtractor : IXFileContentExtractor + { } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Migrations\20260930203317_InitialMigrationAiApi.cs + +using System; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace xAiApi.Migrations +{ + public partial class InitialMigrationAiApi : Migration + { + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "AiConversations", + columns: table => new + { + Id = table.Column(nullable: false), + Deleted = table.Column(nullable: false), + OwnerId = table.Column(maxLength: 255, nullable: false), + Title = table.Column(maxLength: 255, nullable: false), + CreatedOn = table.Column(nullable: false), + UpdatedAt = table.Column(nullable: false), + ProjectId = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AiConversations", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "AiMessages", + columns: table => new + { + Id = table.Column(nullable: false), + Deleted = table.Column(nullable: false), + OwnerId = table.Column(maxLength: 255, nullable: false), + Content = table.Column(nullable: false), + Role = table.Column(nullable: false), + CreatedOn = table.Column(nullable: false), + UpdatedAt = table.Column(nullable: false), + ConversationId = table.Column(maxLength: 255, nullable: false), + MetaDatas = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_AiMessages", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "AiProjects", + columns: table => new + { + Id = table.Column(nullable: false), + Deleted = table.Column(nullable: false), + OwnerId = table.Column(maxLength: 255, nullable: false), + Title = table.Column(maxLength: 255, nullable: false), + Description = table.Column(maxLength: 1000, nullable: true), + Prompt = table.Column(nullable: true), + CreatedOn = table.Column(nullable: false), + UpdatedAt = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_AiProjects", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Files", + columns: table => new + { + Id = table.Column(nullable: false), + Deleted = table.Column(nullable: false), + Type = table.Column(nullable: false), + OwnerId = table.Column(maxLength: 255, nullable: false), + Name = table.Column(maxLength: 255, nullable: false), + Path = table.Column(nullable: false), + Thumb = table.Column(nullable: true), + ThumbPath = table.Column(nullable: true), + UploadedOn = table.Column(nullable: false), + FileName = table.Column(nullable: true), + References = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_Files", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Strings", + columns: table => new + { + Id = table.Column(nullable: false) + .Annotation("Sqlite:Autoincrement", true), + Deleted = table.Column(nullable: false), + Language = table.Column(maxLength: 10, nullable: false), + ResourceTitle = table.Column(maxLength: 255, nullable: false), + TranslatedValue = table.Column(nullable: false) + }, + constraints: table => + { + table.PrimaryKey("PK_Strings", x => x.Id); + }); + + migrationBuilder.CreateTable( + name: "Tags", + columns: table => new + { + Id = table.Column(nullable: false) + .Annotation("Sqlite:Autoincrement", true), + Deleted = table.Column(nullable: false), + Tag = table.Column(maxLength: 255, nullable: false), + References = table.Column(nullable: true) + }, + constraints: table => + { + table.PrimaryKey("PK_Tags", x => x.Id); + }); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropTable( + name: "AiConversations"); + + migrationBuilder.DropTable( + name: "AiMessages"); + + migrationBuilder.DropTable( + name: "AiProjects"); + + migrationBuilder.DropTable( + name: "Files"); + + migrationBuilder.DropTable( + name: "Strings"); + + migrationBuilder.DropTable( + name: "Tags"); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Migrations\20260930203317_InitialMigrationAiApi.Designer.cs + +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using xAiApi.Database; + +namespace xAiApi.Migrations +{ + [DbContext(typeof(XAiApiDbContext))] + [Migration("20260930203317_InitialMigrationAiApi")] + partial class InitialMigrationAiApi + { + protected override void BuildTargetModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.14"); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("ProjectId") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiConversations"); + }); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Content") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ConversationId") + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("MetaDatas") + .HasColumnType("TEXT"); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiMessages"); + }); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiProject", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("Description") + .HasColumnType("TEXT") + .HasMaxLength(1000); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Prompt") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiProjects"); + }); + + modelBuilder.Entity("xFileService.Models.Entities.XFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Path") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("References") + .HasColumnType("TEXT"); + + b.Property("Thumb") + .HasColumnType("TEXT"); + + b.Property("ThumbPath") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("UploadedOn") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("xStringService.Models.Entities.XString", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("Language") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(10); + + b.Property("ResourceTitle") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("TranslatedValue") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Strings"); + }); + + modelBuilder.Entity("xTagService.Models.Entities.XTag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("References") + .HasColumnType("TEXT"); + + b.Property("Tag") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.HasKey("Id"); + + b.ToTable("Tags"); + }); +#pragma warning restore 612, 618 + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Migrations\XAiApiDbContextModelSnapshot.cs + +// +using System; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; +using xAiApi.Database; + +namespace xAiApi.Migrations +{ + [DbContext(typeof(XAiApiDbContext))] + partial class XAiApiDbContextModelSnapshot : ModelSnapshot + { + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder + .HasAnnotation("ProductVersion", "3.1.14"); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiConversation", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("ProjectId") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiConversations"); + }); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiMessage", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Content") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("ConversationId") + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("MetaDatas") + .HasColumnType("TEXT"); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Role") + .HasColumnType("INTEGER"); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiMessages"); + }); + + modelBuilder.Entity("xAiModels.Models.Entities.XAiProject", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("CreatedOn") + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("Description") + .HasColumnType("TEXT") + .HasMaxLength(1000); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Prompt") + .HasColumnType("TEXT"); + + b.Property("Title") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("UpdatedAt") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("AiProjects"); + }); + + modelBuilder.Entity("xFileService.Models.Entities.XFile", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("TEXT"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("FileName") + .HasColumnType("TEXT"); + + b.Property("Name") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("OwnerId") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("Path") + .IsRequired() + .HasColumnType("TEXT"); + + b.Property("References") + .HasColumnType("TEXT"); + + b.Property("Thumb") + .HasColumnType("TEXT"); + + b.Property("ThumbPath") + .HasColumnType("TEXT"); + + b.Property("Type") + .HasColumnType("INTEGER"); + + b.Property("UploadedOn") + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Files"); + }); + + modelBuilder.Entity("xStringService.Models.Entities.XString", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("Language") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(10); + + b.Property("ResourceTitle") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.Property("TranslatedValue") + .IsRequired() + .HasColumnType("TEXT"); + + b.HasKey("Id"); + + b.ToTable("Strings"); + }); + + modelBuilder.Entity("xTagService.Models.Entities.XTag", b => + { + b.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER"); + + b.Property("Deleted") + .HasColumnType("INTEGER"); + + b.Property("References") + .HasColumnType("TEXT"); + + b.Property("Tag") + .IsRequired() + .HasColumnType("TEXT") + .HasMaxLength(255); + + b.HasKey("Id"); + + b.ToTable("Tags"); + }); +#pragma warning restore 612, 618 + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\XAiEmbeddingServiceBase.cs + +using System; +using OpenAI; +using System.Linq; +using OllamaSharp; +using System.Threading; +using xAiApi.Constants; +using xAiModels.Models; +using xAiApi.Extensions; +using xAiApi.Interfaces; +using xAiService.Helpers; +using System.ClientModel; +using xCommons.Extensions; +using xAiModels.Constants; +using xAiModels.Extensions; +using xAiModels.Interfaces; +using xAiApi.Configurations; +using xExceptions.Constants; +using System.Threading.Tasks; +using xFileService.Interfaces; +using Microsoft.Extensions.AI; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using xAiApi.Interfaces.Extractors; +using System.ClientModel.Primitives; +using xCommons.Providers; + +namespace xAiApi.Providers +{ + /// + /// Base Embedding Provider Service ... + /// + public abstract class XAiEmbeddingServiceBase : XAIServiceBase, IXAiEmbeddingServiceBase + { + private readonly IXFileProvider fileProvider; + private readonly XAiModelDescriptor descriptor; + private readonly ILogger logger; + + protected XAiEmbeddingServiceBase( + IXFileProvider fileProvider, + IXAiDataProvider dataProvider, + XAiApiConfiguration configuration, + XValidationProvider validationProvider, + ILogger logger, + IXFileContentExtractor fileContentExtractor, + string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName + ) : base( + fileProvider, + dataProvider, + logger, + configuration, + validationProvider, + fileContentExtractor, + null + ) + { + // + // Prepare Embedding Model Descriptor ... + descriptor = configuration.GetModel(embeddingModelName); + if (!descriptor.IsValid()) + { + XException.InvalidConfiguration.Throw(); + } + + this.fileProvider = fileProvider; + + // + this.logger = logger; + } + + /// + /// Embedding Specified String ... + /// + /// + /// + /// + public virtual async Task Embedding( + string content, + CancellationToken cancellationToken = default + ) + { + // + // Validate ... + if (content.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + using var generator = GetEmbeddingGenerator(); + + // + GeneratedEmbeddings> response = null; + response = await generator.GenerateAsync( + values: [content], + cancellationToken: cancellationToken + ); + + // + if (response.IsNullOrDefault()) + { + XException.ActionFailed.Throw(); + } + + // + // Generae Result ... + var result = response + .First()? + .Vector + .ToArray() + ?? []; + if (result.Length == 0) + { + XException.ActionFailed.Throw(); + } + + // + // Normalize ... + VectorHelper.Normalize(result); + + // + return result; + } + + /// + /// Embedding Specified Batch ... + /// + /// + /// + /// + public virtual async Task> Embeddings( + string[] batch, + CancellationToken cancellationToken = default + ) + { + // + // Validate ... + if (!batch.HasChild() || + !batch.All(x => !x.IsNullOrEmpty()) + ) + { + XException.InvalidArgs.Throw(); + } + + // + using var generator = GetEmbeddingGenerator(); + + // + GeneratedEmbeddings> response = null; + response = await generator.GenerateAsync( + values: [.. batch], + cancellationToken: cancellationToken + ); + + // + if (response.IsNullOrDefault()) + { + XException.ActionFailed.Throw(); + } + + // + var result = response + .Select(e => e.Vector.ToArray()) + .ToList(); + + // + // Normalize ... + foreach (var vector in result) + { + VectorHelper.Normalize(vector); + } + + // + return result; + } + + /// + /// Embedding Generator ... + /// + /// + public IEmbeddingGenerator> GetEmbeddingGenerator() + { + // + IEmbeddingGenerator> result = null; + + // + if (!descriptor.IsValid()) + { + XException.InvalidData.Throw("Invalid Embedding Model ..."); + } + + // + switch (descriptor.Provider) + { + // + case XAiModelProviderType.OpenAI: + // + var openAiClient = new OpenAIClient( + new ApiKeyCredential(descriptor.ApiKey.IsNullOrEmpty() ? XAiApiConstants.XOpenAINoKey : descriptor.ApiKey), + new OpenAIClientOptions + { + Endpoint = new Uri(descriptor.Url), + Transport = new HttpClientPipelineTransport(GetHttpClient(descriptor.Url)) + } + ); + result = openAiClient + .GetEmbeddingClient(descriptor.LLM) + .AsIEmbeddingGenerator(); + break; + + // + case XAiModelProviderType.Ollama: + // + var ollamaClient = new OllamaApiClient(new Uri(descriptor.Url)); + result = ollamaClient; + break; + } + + // + if (result.IsNull()) + { + XException.InvalidData.Throw("Invalid Embedding Model ..."); + } + + // + return result; + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\XAIServiceBase.cs + +using System; +using OpenAI; +using OllamaSharp; +using System.Linq; +using xModels.Dtos; +using System.Net.Http; +using System.Threading; +using xAiApi.Constants; +using xAiModels.Models; +using xAiApi.Interfaces; +using xAiApi.Extensions; +using System.ClientModel; +using xCommons.Providers; +using xCommons.Extensions; +using xAiModels.Constants; +using xAiModels.Interfaces; +using xAiModels.Extensions; +using xAiService.Constants; +using xAiApi.Configurations; +using xExceptions.Constants; +using xAiModels.Models.Dtos; +using xAiService.Extensions; +using System.Threading.Tasks; +using xIdentityModels.Models; +using xFileService.Interfaces; +using Microsoft.Extensions.AI; +using Microsoft.AspNetCore.Http; +using System.Collections.Generic; +using Microsoft.Extensions.Logging; +using xAiApi.Interfaces.Extractors; +using System.ClientModel.Primitives; +using System.Runtime.CompilerServices; + +namespace xAiApi.Providers +{ + /// + /// Base Ai Service ... + /// + public abstract class XAIServiceBase : IXAiServiceBase + { + /// + /// Chat Options ... + /// + public ChatOptions Options { get; } + + /// + /// Descriptor of Models which used in Service ... + /// + public XAiModelDescriptor Descriptor { get; } + + private readonly IXFileProvider fileProvider; + private readonly IXAiDataProvider dataProvider; + private readonly ILogger logger; + private readonly XAiApiConfiguration configuration; + private readonly XValidationProvider validationProvider; + private readonly IXFileContentExtractor fileContentExtractor; + + protected XAIServiceBase( + IXFileProvider fileProvider, + IXAiDataProvider dataProvider, + ILogger logger, + XAiApiConfiguration configuration, + XValidationProvider validationProvider, + IXFileContentExtractor fileContentExtractor, + string model = null + ) + { + // + this.logger = logger; + this.fileProvider = fileProvider; + this.dataProvider = dataProvider; + this.configuration = configuration; + this.validationProvider = validationProvider; + this.fileContentExtractor = fileContentExtractor; + + // + // Prepare Embedding Model Descriptor ... + Descriptor = configuration.GetModel(model); + if (!Descriptor.IsValid()) + { + XException.InvalidConfiguration.Throw(); + } + + // + // Prepare Chat Options based On Tools ... + Options = new ChatOptions(); + } + + // + #region Actions ... + /// + /// Base Method for Communicate with LLM ... + /// + /// + /// + /// + public virtual async Task AskTextAsync( + string prompt, + CancellationToken cancellationToken = default + ) + { + // + // Check Prompt Exists ... + if (prompt.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + using var client = GetClient(); + var history = GetHistory(prompt); + var response = await client.GetResponseAsync( + messages: history, + cancellationToken: cancellationToken + ); + + // + // Validate Response ... + if (!response.IsValid()) + { + // + // Dispose Client ... + client.Dispose(); + XException.ActionFailed.Throw(); + } + + // + // Retrieve Response Text ... + var result = response.Text; + + // + return result; + } + + /// + /// Base Method for Communicate with LLM ... + /// + /// + /// + /// + public virtual async IAsyncEnumerable AskTextAsEnumerable( + string prompt, + [EnumeratorCancellation] + CancellationToken cancellationToken = default + ) + { + // + // Check Prompt Exists ... + if (prompt.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + using var client = GetClient(); + var history = GetHistory(prompt); + var enumerable = client.GetStreamingResponseAsync( + options: null, + messages: history + ); + + // + await foreach (var res in enumerable) + { + // + // Cancellation Token ... + if (cancellationToken.IsCancellationRequested) + { + yield break; + } + + // + yield return res.Text; + } + } + + /// + /// Get LLM Client instance for Communicating with LLM ... + /// + /// + public virtual IChatClient GetClient() + { + // + // Try to Initialize LLm ... + var model = Descriptor.LLM; + var apiKey = Descriptor.ApiKey; + var url = new Uri(Descriptor.Url); + var httpClient = GetHttpClient(Descriptor.Url); + + // + IChatClient result = null; + switch (Descriptor.Provider) + { + // + case XAiModelProviderType.Ollama: + // + // Instance Ollama Client using Custom HttpCLient ... + var ollamaClient = new OllamaApiClient(httpClient, model); + result = + new ChatClientBuilder(ollamaClient) + .UseFunctionInvocation() + .Build(); + break; + + // + case XAiModelProviderType.OpenAI: + // + // Instance Ollama Client using Custom HttpCLient ... + var openAiClient = new OpenAIClient( + new ApiKeyCredential(apiKey.IsNullOrEmpty() ? XAiApiConstants.XOpenAINoKey : apiKey), + new OpenAIClientOptions + { + Endpoint = url, + Transport = new HttpClientPipelineTransport(httpClient) + } + ); + result = + new ChatClientBuilder( + openAiClient + .GetChatClient(model) + .AsIChatClient()) + .UseFunctionInvocation() + .Build(); + break; + + // + case XAiModelProviderType.DeepSeek: + break; + + // + case XAiModelProviderType.HuggingFace: + break; + + // + default: + break; + } + + // + if (result.IsNullOrDefault()) + { + XException.InvalidData.Throw(); + } + + // + return result; + } + + /// + /// Create Custom HttpClient for Communicating with LLM API ... + /// + /// + /// + public virtual HttpClient GetHttpClient(string url = null) + { + // + var result = new HttpClient + { + // + // Disable timeout completely (not recommended for production) + Timeout = Timeout.InfiniteTimeSpan, + + // + BaseAddress = url.IsNullOrEmpty() + ? null + : new Uri(url), + + // + // OR set a very long timeout (e.g., 30 minutes for large file uploads) + // Timeout = TimeSpan.FromMinutes(30) + }; + + // + return result; + } + + /// + /// Prepare a Message History List for LLM Communication based on Prompt ... + /// + /// + /// + /// + public virtual IList GetHistory( + string prompt = null, + bool forcePrompt = true + ) + { + // + // Validate Args ... + if (prompt.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + // User Prompt ... + var promptMessage = new ChatMessage( + ChatRole.User, + prompt + ); + + // + // Preparing Introduction Prompt ... + var introductionPrompt = new ChatMessage( + ChatRole.System, + configuration.GetPrompt() + ); + + // + List result = + [ + introductionPrompt, + ]; + if (forcePrompt && + prompt is not null && + !string.IsNullOrWhiteSpace(prompt)) + { + result.Add(promptMessage); + } + + // + return result; + } + + /// + /// Prepare a Message History List for LLM Communication based on Prompt ... + /// + /// + /// + /// + public virtual IList GetHistory( + ChatMessage message = null, + bool forceMessage = true + ) + { + // + // Preparing Introduction Prompt ... + var introductionPrompt = new ChatMessage( + ChatRole.System, + configuration.GetPrompt(XAiApiConstants.XAiApiIntroductionPromptName) + ); + + // + List result = + [ + introductionPrompt, + ]; + if (forceMessage && + !message.IsNullOrDefault()) + { + result.Add(message); + } + + // + return result; + } + #endregion + + // + #region Loaders ... + /// + /// Load AI Projects ... + /// + /// + /// + /// + /// + /// + public async Task> LoadProjects( + string ownerId, + string connectionId = null, + bool includeDefaults = false, + CancellationToken cancellationToken = default + ) + { + // + // Validate Args ... + if (ownerId.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + var result = await dataProvider.LoadProjects( + ownerId: ownerId, + fillConversations: false, + connectionId: connectionId, + fillConversationMessages: false, + cancellationToken: cancellationToken + ); + + // + // Handle Defaults Including ... + if (includeDefaults) + { + // + // Here we have to fill Conversations of Default Project ... + var defaultProjectId = result.GetDafualtProject(ownerId); + var defaultProject = result.FirstOrDefault(x => x.Id == defaultProjectId); + result = result.Where(x => x.Id != defaultProjectId); + + // + // Reading Conversations of Specified Default Project ... + var defaultProjectConversations = await dataProvider.LoadConversations( + fillMessages: false, + connectionId: connectionId, + projectId: defaultProjectId, + cancellationToken: cancellationToken + ); + var defaultConversationId = defaultProjectConversations.GetDefaultConversation( + ownerId: ownerId, + projectId: defaultProjectId + ); + var defaultConversation = defaultProjectConversations.First(x => x.Id == defaultConversationId); + defaultProjectConversations = defaultProjectConversations.Where(x => x.Id != defaultConversationId); + + // + // Reading Default Conversations Messages ... + var defaultConversationMessages = await dataProvider.LoadMessages( + cancellationToken: cancellationToken, + conversationId: defaultConversationId + ); + + // + defaultConversation.Messages = [.. defaultConversationMessages.OrderByDescending(m => m.CreatedOn)]; + defaultProjectConversations = [defaultConversation, .. defaultProjectConversations]; + defaultProject.Conversations = [.. defaultProjectConversations]; + result = [defaultProject, .. result]; + } + + // + return result; + } + + /// + /// Load Conversations of Specified AI Project ... + /// + /// + /// + /// + /// + public async Task> LoadConversations( + Guid projectId, + string connectionId = null, + CancellationToken cancellationToken = default + ) + { + // + // Validate Args ... + if (projectId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + var result = await dataProvider.LoadConversations( + fillMessages: false, + projectId: projectId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return result; + } + + /// + /// Load Messages of Specified Conversation ... + /// + /// + /// + /// + public async Task> LoadMessages( + Guid conversationId, + CancellationToken cancellationToken = default + ) + { + // + // Validate Args ... + if (conversationId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + var result = await dataProvider.LoadMessages( + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + return result; + } + #endregion + + // + #region Queries ... + /// + /// Query Projects ... + /// + /// + /// + /// + /// + public async Task> QueryProjects( + string ownerId, + XQuery query, + CancellationToken cancellationToken = default + ) + { + // + // Validate ... + if (ownerId.IsNullOrEmpty()) + { + XException.InvalidArgs.Throw(); + } + + // + var result = await dataProvider.QueryProjects( + query: query, + ownerId: ownerId, + cancellationToken: cancellationToken + ); + + // + return result; + } + + /// + /// Query Conversations ... + /// + /// + /// + /// + /// + /// + public async Task> QueryConversations( + string ownerId, + Guid projectId, + XQuery query, + CancellationToken cancellationToken = default + ) + { + // + // Validate ... + if (ownerId.IsNullOrEmpty() || + projectId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + // Retrieve Project ... + var project = await dataProvider.GetProject( + id: projectId, + fillConversations: false, + fillConversationMessages: false, + cancellationToken: cancellationToken + ); + if (project.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Validate Owner ... + if (ownerId != project.OwnerId) + { + XException.NotAllowed.Throw(); + } + + // + var result = await dataProvider.QueryConversations( + query: query, + ownerId: ownerId, + projectId: projectId, + cancellationToken: cancellationToken + ); + + // + return result; + } + + /// + /// Query Messages ... + /// + /// + /// + /// + /// + /// + public async Task> QueryMessages( + string ownerId, + Guid conversationId, + XQuery query, + CancellationToken cancellationToken = default + ) + { + // + // Validate ... + if (ownerId.IsNullOrEmpty() || + conversationId.IsDefaultGuid()) + { + XException.InvalidArgs.Throw(); + } + + // + // Retrieve Conversation ... + var conversation = await dataProvider.GetConversation( + id: conversationId, + fillMessages: false, + cancellationToken: cancellationToken + ); + if (conversation.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Validate Owner ... + if (ownerId != conversation.OwnerId) + { + XException.NotAllowed.Throw(); + } + + // + var result = await dataProvider.QueryMessages( + query: query, + ownerId: ownerId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + return result; + } + #endregion + + // + #region Creators ... + /// + /// Create an AI Project ... + /// + /// + /// + /// + /// + /// + public async Task CreateProject( + string ownerId, + XAiProjectDto item, + string connectionId = null, + CancellationToken cancellationToken = default + ) + { + // + var result = await dataProvider.CreateProject( + item: item, + ownerId: ownerId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return result; + } + + /// + /// Create an AI Conversation ... + /// + /// + /// + /// + /// + /// + /// + public async Task CreateConversation( + string ownerId, + Guid projectId, + XAiConversationDto item, + string connectionId = null, + CancellationToken cancellationToken = default + ) + { + // + var result = await dataProvider.CreateConversation( + item: item, + ownerId: ownerId, + projectId: projectId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + return result; + } + #endregion + + // + #region Data Actions ... + /// + /// Ask a Question as Prompt and Retrieve Response ... + /// + /// + /// + /// + /// + /// + /// + /// + /// + /// + public async Task AskAsync( + string prompt, + string ownerId, + Guid projectId, + Guid conversationId, + string connectionId = null, + IFormFileCollection files = null, + XUserClaimsInfoDto userInfo = null, + CancellationToken cancellationToken = default + ) + { + // + // Validate Args ... + validationProvider + .GroupValidationBuilder() + .AddNotEmpty(prompt) + .AddNotEmpty(ownerId) + .ValidateGroup(); + + // + // Check Project ... + XAiProjectDto project = null; + if (projectId.IsDefaultGuid()) + { + // + project = await dataProvider.GetDefaultProject( + ownerId: ownerId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + projectId = project.Id; + } + else + { + // + project = await dataProvider.GetProject( + id: projectId, + fillConversations: false, + fillConversationMessages: false, + cancellationToken: cancellationToken + ); + } + if (project.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Check Conversation ... + XAiConversationDto conversation = null; + if (conversationId.IsDefaultGuid()) + { + // + conversation = await dataProvider.GetDefaultConversation( + ownerId: ownerId, + projectId: projectId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + conversationId = conversation.Id; + } + else + { + // + conversation = await dataProvider.GetConversation( + id: conversationId, + fillMessages: false, + cancellationToken: cancellationToken + ); + } + if (conversation.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Reading Messages ... + var messages = await dataProvider.LoadMessages( + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + // Upload Files ... + IEnumerable uploadedFiles = null; + if (files.HasChild()) + { + // + // Validate User Info ... + if (userInfo.IsNullOrDefault()) + { + XException.InvalidArgs.Throw(); + } + + // + // Upload Files ... + uploadedFiles = await fileProvider.Upload( + files: files, + userInfo: userInfo, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + } + + // + // Handle Memory ... + IList history = await PrepareMemory( + project: project, + messages: messages, + cancellationToken: cancellationToken + ); + + // + // Prepare and Add Prompt Message ... + var promptMessage = new XAiMessageDto + { + Content = prompt, + OwnerId = ownerId, + Role = XAiChatRole.User, + CreatedOn = DateTime.UtcNow, + ConversationId = conversationId, + ConversationTitle = conversation.Title + }; + + // + // Attached Uploaded File Id's as Meta Data ... + if (uploadedFiles.HasChild()) + { + // + var fileIds = uploadedFiles.Select(f => f.Id).ToList(); + promptMessage.MetaDatas = new Dictionary + { + { XAiServiceConstants.XMetaDataFileIdsIdentifier, fileIds } + }.ToJSON(); + } + + // + // Save Prompt Message ... + promptMessage = await dataProvider.AddMessage( + item: promptMessage, + connectionId: connectionId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + // Since Prompt Message Has Id now, we can Attach Uploaded Files to it ... + IList fileContents = null; + if (uploadedFiles.HasChild()) + { + // + foreach (var uf in uploadedFiles) + { + // + await fileProvider.AddReference( + id: uf.Id, + userInfo: userInfo, + connectionId: connectionId, + forProvidedId: promptMessage.Id, + cancellationToken: cancellationToken, + providedFor: XAiServiceConstants.XAiMessageFilesIdentifier + ); + } + + // + // TODO: Extract File Contents ... + } + + // + // Converts to ChatMessage ... + var promptChatMessage = promptMessage + .ToChatMessages(fileContents); + + // + // Ask Questions From LLM ... + var answer = await AskLLMAsync( + history: history, + prompt: promptChatMessage, + cancellationToken: cancellationToken + ); + + // + // Prepare Answer AI Message ... + var result = new XAiMessageDto + { + Content = answer, + OwnerId = ownerId, + CreatedOn = DateTime.UtcNow, + Role = XAiChatRole.Assistant, + ConversationId = conversationId, + ConversationTitle = conversation.Title + }; + result = await dataProvider.AddMessage( + item: result, + connectionId: connectionId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + return result; + } + + /// + /// Ask a Question as Prompt and Retrieve Response as Async Enumerable ... + /// + /// + /// + /// + /// + /// + /// + /// + /// + public async IAsyncEnumerable AskAsEnumerable( + string prompt, + string ownerId, + Guid projectId, + Guid conversationId, + string connectionId = null, + IFormFileCollection files = null, + [EnumeratorCancellation] + CancellationToken cancellationToken = default + ) + { + // + // Validate Args ... + validationProvider + .GroupValidationBuilder() + .AddNotEmpty(prompt) + .AddNotEmpty(ownerId) + .ValidateGroup(); + + // + // Check Project ... + XAiProjectDto project = null; + if (projectId.IsDefaultGuid()) + { + // + project = await dataProvider.GetDefaultProject( + ownerId: ownerId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + projectId = project.Id; + } + else + { + // + project = await dataProvider.GetProject( + id: projectId, + fillConversations: false, + fillConversationMessages: false, + cancellationToken: cancellationToken + ); + } + if (project.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Check Conversation ... + XAiConversationDto conversation = null; + if (conversationId.IsDefaultGuid()) + { + // + conversation = await dataProvider.GetDefaultConversation( + ownerId: ownerId, + projectId: projectId, + connectionId: connectionId, + cancellationToken: cancellationToken + ); + + // + conversationId = conversation.Id; + } + else + { + // + conversation = await dataProvider.GetConversation( + id: conversationId, + fillMessages: false, + cancellationToken: cancellationToken + ); + } + if (conversation.IsNullOrDefault()) + { + XException.NotFound.Throw(); + } + + // + // Reading Messages ... + var messages = await dataProvider.LoadMessages( + conversationId: conversationId, + cancellationToken: cancellationToken + ); + + // + // TODO: Parse Files ... + // for Attached them into Prompt Message ... + + // + // Handle Memory ... + IList history = await PrepareMemory( + project: project, + messages: messages, + cancellationToken: cancellationToken + ); + + // + // Prepare and Add Prompt Message ... + var promptMessage = new XAiMessageDto + { + Content = prompt, + OwnerId = ownerId, + Role = XAiChatRole.User, + CreatedOn = DateTime.UtcNow, + ConversationId = conversationId, + }; + promptMessage = await dataProvider.AddMessage( + item: promptMessage, + connectionId: connectionId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + promptMessage = await dataProvider.Enrich( + source: promptMessage, + cancellationToken: cancellationToken + ); + + // + // Attach Uploaded Files ... + var promptChatMessage = promptMessage + .ToChatMessages(); + history.Add(promptChatMessage); + + // + // Ask Questions From LLM ... + using var client = GetClient(); + var enumerable = client.GetStreamingResponseAsync( + options: null, + messages: history + ); + + // + long sequence = 1; + var answer = string.Empty; + await foreach (var res in enumerable) + { + // + // Cancellation Token ... + if (cancellationToken.IsCancellationRequested) + { + yield break; + } + + // + var resp = new XAiResonseMessage + { + OwnerId = ownerId, + Content = res.Text, + Sequence = sequence, + Role = XAiChatRole.System, + RespondTo = promptMessage.Id, + RespondedOn = DateTime.UtcNow, + }; + + // + sequence++; + answer += res.Text; + + // + yield return resp; + } + + // + // Prepare Answer AI Message ... + var result = new XAiMessageDto + { + Content = answer, + OwnerId = ownerId, + Role = XAiChatRole.System, + CreatedOn = DateTime.UtcNow, + ConversationId = conversationId + }; + result = await dataProvider.AddMessage( + item: result, + connectionId: connectionId, + conversationId: conversationId, + cancellationToken: cancellationToken + ); + } + #endregion + + /// + /// Base Dispose Action ... + /// + /// + public virtual void Dispose() + { } + + // + #region Private ... + /// + /// Implementing Memory Strategy here ... + /// + /// + /// + /// + /// + private async Task> PrepareMemory( + XAiProjectDto project, + IEnumerable messages, + CancellationToken cancellationToken = default + ) + { + // + var result = new List(); + + // + // Retrieve Introduction Prompt ... + var introductionPrompt = configuration.GetPrompt(XAiApiConstants.XAiApiIntroductionPromptName); + if (introductionPrompt.IsNullOrEmpty()) + { + XException.InvalidData.Throw(); + } + + // + var introductionMessage = new ChatMessage( + role: ChatRole.System, + content: introductionPrompt + ); + + // + result.Add(introductionMessage); + + // + if (!project.IsNullOrDefault() && + !project.Prompt.IsNullOrEmpty()) + { + // + // TODO: Handle Project Files Here ... + var projectMessage = new ChatMessage( + role: ChatRole.System, + contents: [new TextContent(project.Prompt)] + ); + result.Add(projectMessage); + } + + // + if (messages.HasChild()) + { + // + // TODO: Here we can Implement Memory Management based on + // Last nth of Messages, or Summarization of Messages, etc ... + result = [..result, ..messages + .OrderBy(m => m.CreatedOn) + .Select(m => m.ToChatMessages())]; + } + + // + return result; + } + + /// + /// Ask Quetion from LLM ... + /// + /// + /// + /// + /// + private async Task AskLLMAsync( + ChatMessage prompt, + IList history, + CancellationToken cancellationToken = default + ) + { + // + if (prompt.IsNullOrDefault()) + { + XException.InvalidArgs.Throw(); + } + + // + history.Add(prompt); + using var client = GetClient(); + var response = await client.GetResponseAsync( + messages: history, + cancellationToken: cancellationToken + ); + + // + // Validate Response ... + if (!response.IsValid()) + { + // + // Dispose Client ... + client.Dispose(); + XException.ActionFailed.Throw(); + } + + // + // Retrieve Response Text ... + var result = response.Text; + + // + return result; + } + #endregion + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\XDefaultAiService.cs + +using xAiApi.Constants; +using xAiApi.Interfaces; +using xCommons.Providers; +using xAiModels.Interfaces; +using xAiApi.Configurations; +using xFileService.Interfaces; +using Microsoft.Extensions.Logging; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers +{ + public class XDefaultAiService : XAIServiceBase, IXDefaultAiService + { + public XDefaultAiService( + IXFileProvider fileProvider, + IXAiDataProvider dataProvider, + ILogger logger, + XAiApiConfiguration configuration, + XValidationProvider validationProvider, + IXFileContentExtractor fileContentExtractor + ) : base( + fileProvider, + dataProvider, + logger, + configuration, + validationProvider, + fileContentExtractor, + XAiApiConstants.XAiDefaultModelName + ) + { } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\XDefaultEmbeddingService.cs + +using xAiApi.Constants; +using xAiApi.Interfaces; +using xAiModels.Interfaces; +using xAiApi.Configurations; +using xFileService.Interfaces; +using Microsoft.Extensions.Logging; +using xAiApi.Interfaces.Extractors; +using xCommons.Providers; + +namespace xAiApi.Providers +{ + public class XDefaultEmbeddingService : XAiEmbeddingServiceBase, IXDefaultEmbeddingService + { + public XDefaultEmbeddingService( + IXFileProvider fileProvider, + IXAiDataProvider dataProvider, + XAiApiConfiguration configuration, + XValidationProvider validationProvider, + ILogger logger, + IXFileContentExtractor fileContentExtractor + ) : base( + fileProvider, + dataProvider, + configuration, + validationProvider, + logger, + fileContentExtractor, + XAiApiConstants.XAiEmbeddingModelName + ) + { } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\XDefaultThinkingAiService.cs + +using xAiApi.Constants; +using xAiApi.Interfaces; +using xCommons.Providers; +using xAiModels.Interfaces; +using xAiApi.Configurations; +using Microsoft.Extensions.AI; +using xFileService.Interfaces; +using Microsoft.Extensions.Logging; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers +{ + public class XDefaultThinkingAiService : XAIServiceBase, IXDefaultThinkingAiService + { + public XDefaultThinkingAiService( + IXFileProvider fileProvider, + IXAiDataProvider dataProvider, + ILogger logger, + XAiApiConfiguration configuration, + XValidationProvider validationProvider, + IXFileContentExtractor fileContentExtractor, + ReasoningEffort effort = ReasoningEffort.Medium, + ReasoningOutput output = ReasoningOutput.Full + ) : base( + fileProvider, + dataProvider, + logger, + configuration, + validationProvider, + fileContentExtractor, + XAiApiConstants.XAiDefaultThinkingModelName + ) + { + // + // Preparing Reasoning ... + Options.Reasoning = new ReasoningOptions + { + Output = output, + Effort = effort, + }; + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\Extractors\XDocxFileContentExtractor.cs + +using System; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using xAiApi.Interfaces.Extractors; +using DocumentFormat.OpenXml.Drawing; +using DocumentFormat.OpenXml.Packaging; + +namespace xAiApi.Providers.Extractors +{ + /// + /// Extracts text content from DOCX files using DocumentFormat.OpenXml ... + /// + public class XDocxFileContentExtractor : IXDocxFileContentExtractor + { + /// + /// Supported MIME Types ... + /// + private static readonly string[] SupportedMimeTypes = + [ + "application/vnd.openxmlformats-officedocument.wordprocessingml.document" + ]; + + /// + /// Check if this extractor supports the specified MIME type ... + /// + public bool CanExtract(string mimeType) + { + // + return SupportedMimeTypes.Contains( + mimeType?.ToLowerInvariant() ?? string.Empty + ); + } + + /// + /// Extract text content from file stream ... + /// + public async Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ) + { + // + using var memoryStream = new MemoryStream(); + await fileStream.CopyToAsync(memoryStream, cancellationToken); + memoryStream.Position = 0; + + // + var stringBuilder = new StringBuilder(); + + // + using (var wordDocument = WordprocessingDocument.Open(memoryStream, false)) + { + // + var body = wordDocument.MainDocumentPart?.Document?.Body; + if (body != null) + { + // + var paragraphs = body.Elements(); + foreach (var paragraph in paragraphs) + { + // + var text = paragraph.InnerText?.Trim(); + if (!string.IsNullOrEmpty(text)) + { + stringBuilder.AppendLine(text); + } + } + } + } + + // + return stringBuilder.ToString(); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\Extractors\XExcelFileContentExtractor.cs + +using System; +using System.IO; +using System.Data; +using System.Linq; +using System.Text; +using ExcelDataReader; +using System.Threading; +using System.Threading.Tasks; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers.Extractors +{ + public class XExcelFileContentExtractor : IXExcelFileContentExtractor + { + /// + /// Supported MIME Types ... + /// + private static readonly string[] SupportedMimeTypes = + [ + "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet", // .xlsx + "application/vnd.ms-excel" // .xls + ]; + + /// + /// Check if this extractor supports the specified MIME type ... + /// + public bool CanExtract(string mimeType) + { + // + return SupportedMimeTypes.Contains( + mimeType?.ToLowerInvariant() ?? string.Empty + ); + } + + /// + /// Extract text content from file stream ... + /// + public async Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ) + { + // + Encoding.RegisterProvider(CodePagesEncodingProvider.Instance); + + // + using var memoryStream = new MemoryStream(); + await fileStream.CopyToAsync(memoryStream, cancellationToken); + memoryStream.Position = 0; + + // + var stringBuilder = new StringBuilder(); + + // + using (var reader = ExcelReaderFactory.CreateReader(memoryStream)) + { + // + var result = reader.AsDataSet(); + foreach (DataTable table in result.Tables) + { + // + stringBuilder.AppendLine($"--- Sheet: {table.TableName} ---"); + + // + var headers = new string[table.Columns.Count]; + for (int i = 0; i < table.Columns.Count; i++) + { + headers[i] = table.Columns[i].ColumnName; + } + stringBuilder.AppendLine(string.Join(" | ", headers)); + stringBuilder.AppendLine(new string('-', 50)); + + // + foreach (DataRow row in table.Rows) + { + // + var rowValues = new string[row.ItemArray.Length]; + for (int i = 0; i < row.ItemArray.Length; i++) + { + rowValues[i] = row.ItemArray[i]?.ToString()?.Trim() ?? string.Empty; + } + stringBuilder.AppendLine(string.Join(" | ", rowValues)); + } + stringBuilder.AppendLine(); + } + } + + // + return stringBuilder.ToString(); + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\Extractors\XFileContentExtractor.cs + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Reflection; +using xCommons.Extensions; +using xExceptions.Constants; +using System.Threading.Tasks; +using System.Collections.Generic; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers +{ + /// + /// Composite extractor that delegates to appropriate extractor + /// based on MIME type ... + /// + public class XFileContentExtractor : IXFileContentExtractor + { + // + private readonly IList extractors = []; + + public XFileContentExtractor() + { + // + var path = Path.Combine(AppContext.BaseDirectory); + var files = Directory.GetFiles(path, "*.dll"); + foreach (var dll in files) + { + // + // Retrieve DLL Assembly ... + var assembly = Assembly.LoadFrom(dll); + + // + // Extract Registerar Types ... + var extractorTypes = assembly + .GetExportedTypes() + .Where(t => + !t.IsAbstract && + typeof(IXFileContentExtractor).IsAssignableFrom(t)); + foreach (var extractorType in extractorTypes) + { + // + // Ignore XFileContentExtractor ... + if (extractorType.FullName.Contains(nameof(XFileContentExtractor))) + { + continue; + } + + // + // Create Instance of Service ... + var instance = (IXFileContentExtractor)Activator.CreateInstance(extractorType); + if (!instance.IsNull()) + { + // + // Add Registerar Instance to Collection ... + extractors.Add(instance); + } + } + } + } + + /// + /// Check if this extractor supports the specified MIME type ... + /// + public bool CanExtract(string mimeType) + { + return extractors.Any(e => e.CanExtract(mimeType)); + } + + /// + /// Extract text content from file stream ... + /// + public async Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ) + { + // + var extractor = extractors + .FirstOrDefault(e => e.CanExtract(mimeType)); + if (extractor.IsNull()) + { + // + XException.NotAllowed.Throw( + $"Unsupported file type: {mimeType}" + ); + } + + // + var result = await extractor + .ExtractAsync( + mimeType: mimeType, + fileStream: fileStream, + cancellationToken: cancellationToken + ); + + // + return result; + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\Extractors\XPdfFileContentExtractor.cs + +using System; +using System.IO; +using System.Linq; +using System.Text; +using UglyToad.PdfPig; +using System.Threading; +using System.Threading.Tasks; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers +{ + /// + /// Extracts content from PDF files using PdfPig ... + /// + public class XPdfFileContentExtractor : IXPdfFileContentExtractor + { + /// + /// Supported MIME Types ... + /// + private static readonly string[] SupportedMimeTypes = + [ + "application/pdf" + ]; + + /// + /// Check if this extractor supports the specified MIME type ... + /// + public bool CanExtract(string mimeType) + { + // + return SupportedMimeTypes.Contains( + mimeType?.ToLowerInvariant() ?? string.Empty + ); + } + + /// + /// Extract text content from file stream ... + /// + public async Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ) + { + // + var result = await Task.Run(() => + { + // + var sb = new StringBuilder(); + using var document = PdfDocument.Open(fileStream); + foreach (var page in document.GetPages()) + { + // + var text = page.Text; + + // + sb.AppendLine(text); + sb.AppendLine(); + } + + // + return sb.ToString(); + }, cancellationToken); + + // + return result; + } + } +} + +### FILE: C:\Users\SaherElm\Documents\Projects\xSaherelmWorkspace\xAiApi\Providers\Extractors\XPlainTextFileContentExtractor.cs + +using System; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using xAiApi.Interfaces.Extractors; + +namespace xAiApi.Providers +{ + /// + /// Extracts content from plain text files (txt, md, csv, json, xml) ... + /// + public class XPlainTextFileContentExtractor : IXPlainTextFileContentExtractor + { + /// + /// Supported MIME Types ... + /// + private static readonly string[] SupportedMimeTypes = + [ + "text/plain", + "text/markdown", + "text/csv", + "text/html", + "text/xml", + "application/json", + "application/xml" + ]; + + /// + /// Check if this extractor supports the specified MIME type ... + /// + public bool CanExtract(string mimeType) + { + // + return SupportedMimeTypes.Contains( + mimeType?.ToLowerInvariant() ?? string.Empty + ); + } + + /// + /// Extract text content from file stream ... + /// + public async Task ExtractAsync( + Stream fileStream, + string mimeType, + CancellationToken cancellationToken = default + ) + { + // + using var reader = new StreamReader(fileStream); + return await reader.ReadToEndAsync(); + } + } +} diff --git a/Modules/xAiModels b/Modules/xAiModels index 755eafa..26df2c7 160000 --- a/Modules/xAiModels +++ b/Modules/xAiModels @@ -1 +1 @@ -Subproject commit 755eafa7d2b8c1dfcd2d0da3da9f34d736c211c3 +Subproject commit 26df2c7ebcfb6fc6cbb456d84f1d2c9cd46d0410 diff --git a/xAiApi b/xAiApi index 4521a53..c30d93b 160000 --- a/xAiApi +++ b/xAiApi @@ -1 +1 @@ -Subproject commit 4521a537348bedb070283fb6212f37b2ae8b4c95 +Subproject commit c30d93be0b3b62b5d6547e7f1ab1cc0decf1ff5c