last ...

Complete Audi Transcript using NAudio and Whisper ...
add support for OCR ...
2026-10-04 18:10:44 +03:30 · 2026-10-04 11:42:27 +03:30 · 2026-10-03 22:26:03 +03:30
9 changed files with 328 additions and 62 deletions
@@ -1,3 +1,4 @@
+using xAiApi.Models;
 using xAiModels.Models;

 namespace xAiApi.Configurations
@@ -13,5 +14,15 @@ namespace xAiApi.Configurations
        /// Allowed Prompts to Use ...
        /// </summary>
        public XAiPromptDescriptor[] Prompts { get; set; } = [];
+
+        /// <summary>
+        /// OCR Configuration ...
+        /// </summary>
+        public XAiOCRDescriptor OCR { get; set; }
+
+        /// <summary>
+        /// Transcript Configuration ...
+        /// </summary>
+        public XAiTransciptDescriptor Transcript { get; set; }
    }
 }
@@ -0,0 +1,27 @@
+using Tesseract;
+
+namespace xAiApi.Models
+{
+    public class XAiOCRDescriptor
+    {
+        /// <summary>
+        /// Provides Trained Data Path ...
+        /// </summary>
+        public string DataPath { get; set; }
+
+        /// <summary>
+        /// The Engine for Doing OCR ...
+        /// </summary>
+        public EngineMode EngineMode { get; set; }
+
+        /// <summary>
+        /// Enable OCR Fallback using Model ...
+        /// </summary>
+        public bool EnableOcrFallback { get; set; }
+
+        /// <summary>
+        /// Default OCR Language ...
+        /// </summary>
+        public string DefaultLanguage { get; set; }
+    }
+}
@@ -0,0 +1,8 @@
+namespace xAiApi.Models
+{
+    public class XAiTransciptDescriptor
+    {
+        public string Language { get; set; }
+        public string ModelsPath { get; set; }
+    }
+}
@@ -21,7 +21,7 @@
      "commandName": "Project",
      "launchBrowser": true,
      "launchUrl": "weatherforecast",
-      "applicationUrl": "https://localhost:5001;http://localhost:5000",
+      "applicationUrl": "https://localhost:6001;http://localhost:6000",
      "environmentVariables": {
        "ASPNETCORE_ENVIRONMENT": "Development"
      }
@@ -2,11 +2,15 @@ using System;
 using System.IO;
 using System.Linq;
 using Whisper.net;
+using NAudio.Wave;
 using System.Threading;
 using xAiModels.Models;
+using xCommons.Extensions;
+using xAiApi.Configurations;
+using xExceptions.Constants;
 using System.Threading.Tasks;
+using NAudio.MediaFoundation;
 using xAiApi.Interfaces.Extractors;
-using DocumentFormat.OpenXml.EMMA;

 namespace xAiApi.Providers.Extractors
 {
@@ -24,29 +28,30 @@ namespace xAiApi.Providers.Extractors
            "audio/mp3",
            "audio/wav",
            "audio/wave",
+            "audio/x-wav",
            "audio/ogg",
            "audio/m4a",
            "audio/mp4",
+            "audio/aac",
+            "audio/x-m4a",
            "audio/webm"
        ];

-        private readonly string language;
-        private readonly string whisperModelPath;
-
-        public XAudioFileContentExtractor() : this(
-                language: "fa",
-                whisperModelPath: "Models/ggml-base.bin"
-        )
-        { }
+        private readonly XAiApiConfiguration configuration;

        public XAudioFileContentExtractor(
-            string whisperModelPath = "Models/ggml-base.bin",
-            string language = "fa"
+            XAiApiConfiguration configuration
        )
        {
            //
-            this.language = language;
-            this.whisperModelPath = whisperModelPath;
+            if (configuration.Transcript.IsNullOrDefault() ||
+                configuration.Transcript.ModelsPath.IsNullOrEmpty())
+            {
+                XException.InvalidConfiguration.Throw();
+            }
+
+            //
+            this.configuration = configuration;
        }

        /// <summary>
@@ -113,14 +118,15 @@ namespace xAiApi.Providers.Extractors

                //
                using var wavStream = await ConvertToWavAsync(
-                    memoryStream,
-                    cancellationToken
+                    mimeType: mimeType,
+                    inputStream: memoryStream,
+                    cancellationToken: cancellationToken
                );

                //
-                using var factory = WhisperFactory.FromPath(whisperModelPath);
+                using var factory = WhisperFactory.FromPath(configuration.Transcript.ModelsPath);
                using var processor = factory.CreateBuilder()
-                    .WithLanguage(language)
+                    .WithLanguage(configuration.Transcript.Language)
                    .Build();

                //
@@ -143,17 +149,186 @@ namespace xAiApi.Providers.Extractors
            return result;
        }

+        //
+        #region Private ...
        /// <summary>
-        /// Audio Format Converting ...
+        /// Audio Format Converting to WAV 16kHz Mono 16-bit PCM ...
        /// </summary>
        private async Task<Stream> ConvertToWavAsync(
            Stream inputStream,
-            CancellationToken cancellationToken
+            string mimeType,
+            CancellationToken cancellationToken = default
        )
        {
            //
-            // TODO: Complete this ...
-            return await Task.FromResult(inputStream);
+            var result = await Task.Run(() =>
+            {
+                //
+                var memoryStream = new MemoryStream();
+                inputStream.CopyTo(memoryStream);
+                memoryStream.Position = 0;
+
+                //
+                var targetFormat = new WaveFormat(16000, 16, 1);
+                string tempFilePath = null;
+
+                //
+                try
+                {
+                    //
+                    WaveStream reader = GetAudioReader(memoryStream, mimeType, ref tempFilePath);
+                    if (reader == null)
+                    {
+                        //
+                        XException.NotAllowed.Throw(
+                            $"Unsupported audio format: {mimeType}"
+                        );
+                    }
+
+                    //
+                    var needsConversion =
+                        reader.WaveFormat.SampleRate != 16000 ||
+                        reader.WaveFormat.Channels != 1 ||
+                        reader.WaveFormat.BitsPerSample != 16 ||
+                        reader.WaveFormat.Encoding != WaveFormatEncoding.Pcm;
+                    var outputStream = new MemoryStream();
+                    if (!needsConversion)
+                    {
+                        //
+                        reader.CopyTo(outputStream);
+                        reader.Dispose();
+                    }
+                    else
+                    {
+                        //
+                        MediaFoundationApi.Startup();
+                        using var resampler = new MediaFoundationResampler(
+                            reader,
+                            targetFormat
+                        );
+                        resampler.ResamplerQuality = 60;
+
+                        //
+                        WaveFileWriter.WriteWavFileToStream(outputStream, resampler);
+                        reader.Dispose();
+                    }
+
+                    //
+                    outputStream.Position = 0;
+                    return outputStream;
+                }
+                finally
+                {
+                    //
+                    if (!string.IsNullOrEmpty(tempFilePath) &&
+                        File.Exists(tempFilePath))
+                    {
+                        //
+                        try
+                        {
+                            File.Delete(tempFilePath);
+                        }
+                        catch { }
+                    }
+
+                    //
+                    memoryStream.Dispose();
+                }
+            }, cancellationToken);
+
+            //
+            return result;
        }
+
+        /// <summary>
+        /// Get appropriate WaveStream reader based on MIME type ...
+        /// </summary>
+        /// <param name="stream">Input stream (MemoryStream)</param>
+        /// <param name="mimeType">MIME type of audio file</param>
+        /// <param name="tempFilePath">Path to temp file (if created)</param>
+        /// <returns>WaveStream reader or null if unsupported</returns>
+        private WaveStream GetAudioReader(
+            MemoryStream stream,
+            string mimeType,
+            ref string tempFilePath
+        )
+        {
+            //
+            try
+            {
+                //
+                var normalizedMime = mimeType?.ToLowerInvariant() ?? string.Empty;
+                switch (normalizedMime)
+                {
+                    //
+                    case "audio/wav":
+                    case "audio/wave":
+                    case "audio/x-wav":
+                        return new WaveFileReader(stream);
+
+                    //
+                    case "audio/mpeg":
+                    case "audio/mp3":
+                        return new Mp3FileReader(stream);
+
+                    //
+                    case "audio/m4a":
+                    case "audio/mp4":
+                    case "audio/aac":
+                    case "audio/ogg":
+                    case "audio/webm":
+                    case "audio/x-m4a":
+                        //
+                        var extension = normalizedMime switch
+                        {
+                            "audio/m4a" or "audio/mp4" or "audio/aac" or "audio/x-m4a" => ".m4a",
+                            "audio/ogg" => ".ogg",
+                            "audio/webm" => ".webm",
+                            _ => ".tmp"
+                        };
+                        tempFilePath = Path.Combine(
+                            Path.GetTempPath(),
+                            $"audio_{Guid.NewGuid()}{extension}"
+                        );
+
+                        //
+                        stream.Position = 0;
+                        using (var fileStream = File.Create(tempFilePath))
+                        {
+                            stream.CopyTo(fileStream);
+                        }
+
+                        //
+                        MediaFoundationApi.Startup();
+                        return new MediaFoundationReader(tempFilePath);
+
+                    default:
+                        //
+                        tempFilePath = Path.Combine(
+                            Path.GetTempPath(),
+                            $"audio_{Guid.NewGuid()}.tmp"
+                        );
+                        stream.Position = 0;
+                        using (var fileStream = File.Create(tempFilePath))
+                        {
+                            stream.CopyTo(fileStream);
+                        }
+                        try
+                        {
+                            MediaFoundationApi.Startup();
+                            return new MediaFoundationReader(tempFilePath);
+                        }
+                        catch
+                        {
+                            return null;
+                        }
+                }
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+        #endregion
    }
 }
@@ -5,6 +5,10 @@ using System.Threading;
 using xAiModels.Models;
 using System.Threading.Tasks;
 using xAiApi.Interfaces.Extractors;
+using xAiApi.Configurations;
+using xCommons.Extensions;
+using xExceptions.Constants;
+using Tesseract;

 namespace xAiApi.Providers.Extractors
 {
@@ -24,16 +28,25 @@ namespace xAiApi.Providers.Extractors
        ];

        /// <summary>
-        /// آیا OCR فعال است (برای مدل‌های غیر Vision) ...
+        /// AI Api Configuration ...
        /// </summary>
-        private readonly bool _enableOcrFallback;
+        private readonly XAiApiConfiguration configuration;

-        public XImageFileContentExtractor() : this(enableOcrFallback: true)
-        { }
-
-        public XImageFileContentExtractor(bool enableOcrFallback = true)
+        public XImageFileContentExtractor(
+            XAiApiConfiguration configuration
+        )
        {
-            _enableOcrFallback = enableOcrFallback;
+            //
+            if (configuration.IsNullOrDefault() ||
+                configuration.OCR.IsNullOrDefault() ||
+                configuration.OCR.DataPath.IsNullOrEmpty() ||
+                configuration.OCR.DefaultLanguage.IsNullOrEmpty())
+            {
+                XException.InvalidConfiguration.Throw();
+            }
+
+            //
+            this.configuration = configuration;
        }

        /// <summary>
@@ -103,8 +116,9 @@ namespace xAiApi.Providers.Extractors
                Description = $"Attached image: {fileName}"
            });

-            // OCR Fallback برای مدل‌های متنی
-            if (_enableOcrFallback)
+            //
+            // OCR Fallback ...
+            if (configuration.OCR.EnableOcrFallback)
            {
                //
                try
@@ -135,8 +149,44 @@ namespace xAiApi.Providers.Extractors
        )
        {
            //
-            // TODO: Complete this ...
-            return await Task.FromResult(string.Empty);
+            return await Task.Run(() =>
+            {
+                //
+                // Validate Path ...
+                if (!Directory.Exists(configuration.OCR.DataPath))
+                {
+                    //
+                    throw new DirectoryNotFoundException(
+                        $"TessData directory not found at: {configuration.OCR.DataPath}. " +
+                        "Please download traineddata files from https://github.com/tesseract-ocr/tessdata"
+                    );
+                }
+            
+                //
+                using var engine = new TesseractEngine(
+                    datapath: configuration.OCR.DataPath,
+                    language: configuration.OCR.DefaultLanguage,
+                    engineMode: configuration.OCR.EngineMode
+                );
+
+                //
+                using var pix = Pix.LoadFromMemory(imageBytes);
+
+                //
+                cancellationToken.ThrowIfCancellationRequested();
+
+                //
+                using var page = engine.Process(pix);
+
+                //
+                var text = page.GetText();
+
+                //
+                text = text?.Trim();
+
+                //
+                return text ?? string.Empty;
+            }, cancellationToken);
        }
    }
 }
@@ -199,14 +199,6 @@ namespace xAiApi

            //
            services.AddSingleton<IXFileContentExtractor, XFileContentExtractor>();
-            // services.AddSingleton(sp =>
-            // {
-            //     // 
-            //     var extractors = sp.GetServices<IXFileContentExtractor>();
-            //     return new XFileContentExtractor(extractors);
-            // });
-            // services.AddSingleton<IXFileContentExtractor>(sp =>
-            //     sp.GetRequiredService<XFileContentExtractor>());

            //
            // Register Default Ai Service ...
@@ -78,12 +78,22 @@
        "Description": "",
        "Params": []
      }
-    ]
+    ],
+    "OCR": {
+      "DataPath": "Models",
+      "EngineMode": "LstmOnly",
+      "EnableOcrFallback": true,
+      "DefaultLanguage": "fas+eng"
+    },
+    "Transcript": {
+      "Language": "fa",
+      "ModelsPath": "Models/ggml-base.bin"
+    }
  },
  "Logging": {
    "LogLevel": {
-      "Default": "Information",
      "Microsoft": "Warning",
+      "Default": "Information",
      "Microsoft.Hosting.Lifetime": "Information"
    }
  },
@@ -27,13 +27,6 @@
    <None Include="../../Resources/Images/favicon.png" Link="icon.png" Pack="true" PackagePath="\icon.png" />
  </ItemGroup>

-  <!-- Native Libraries -->
-  <!-- <ItemGroup>
-    <None Include="runtimes/linux-x64/native/libpdfium.so">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup> -->
-
  <!-- Local Modules -->
  <ItemGroup>
  </ItemGroup>
@@ -53,22 +46,22 @@

  <!-- Dependencies -->
  <ItemGroup>
-    <PackageReference Include="DocumentFormat.OpenXml" Version="3.5.1" />
-    <PackageReference Include="ExcelDataReader" Version="3.9.0" />
-    <PackageReference Include="ExcelDataReader.DataSet" Version="3.9.0" />
    <PackageReference Include="NAudio" Version="2.4.0" />
+    <PackageReference Include="Tesseract" Version="5.2.0" />
+    <PackageReference Include="Whisper.net" Version="1.9.1" />
    <PackageReference Include="OllamaSharp" Version="5.4.25" />
    <PackageReference Include="PdfiumViewer" Version="2.13.0" />
-    <PackageReference Include="PdfiumViewer.Native.x86.no_v8-no_xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86.v8-xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86_64.no_v8-no_xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86_64.v8-xfa" Version="2018.4.8.256" />
-    <PackageReference Include="Tesseract" Version="5.2.0" />
-    <PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" />
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.11" />
-    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="5.0.0" />
-    <PackageReference Include="Whisper.net" Version="1.9.1" />
+    <PackageReference Include="ExcelDataReader" Version="3.9.0" />
    <PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
+    <PackageReference Include="DocumentFormat.OpenXml" Version="3.5.1" />
+    <PackageReference Include="ExcelDataReader.DataSet" Version="3.9.0" />
+    <PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" />
+    <PackageReference Include="PdfiumViewer.Native.x86.v8-xfa" Version="2018.4.8.256" />
+    <PackageReference Include="PdfiumViewer.Native.x86_64.v8-xfa" Version="2018.4.8.256" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.11" />
+    <PackageReference Include="PdfiumViewer.Native.x86.no_v8-no_xfa" Version="2018.4.8.256" />
+    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="5.0.0" />
+    <PackageReference Include="PdfiumViewer.Native.x86_64.no_v8-no_xfa" Version="2018.4.8.256" />
  </ItemGroup>

  <!-- For XML Documentation Support -->
Author	SHA1	Message	Date
saherelm	20f1c7fb2f	last ...	2026-10-04 18:10:44 +03:30
saherelm	e6e5369b95	Complete Audi Transcript using NAudio and Whisper ...	2026-10-04 11:42:27 +03:30
saherelm	70c2d5a649	add support for OCR ...	2026-10-03 22:26:03 +03:30