Complete Audi Transcript using NAudio and Whisper ...

2026-10-04 11:42:27 +03:30
parent 70c2d5a649
commit e6e5369b95
5 changed files with 225 additions and 40 deletions
@@ -19,5 +19,10 @@ namespace xAiApi.Configurations
        /// OCR Configuration ...
        /// </summary>
        public XAiOCRDescriptor OCR { get; set; }
+
+        /// <summary>
+        /// Transcript Configuration ...
+        /// </summary>
+        public XAiTransciptDescriptor Transcript { get; set; }
    }
 }
@@ -0,0 +1,8 @@
+namespace xAiApi.Models
+{
+    public class XAiTransciptDescriptor
+    {
+        public string Language { get; set; }
+        public string ModelsPath { get; set; }
+    }
+}
@@ -2,11 +2,15 @@ using System;
 using System.IO;
 using System.Linq;
 using Whisper.net;
+using NAudio.Wave;
 using System.Threading;
 using xAiModels.Models;
+using xCommons.Extensions;
+using xAiApi.Configurations;
+using xExceptions.Constants;
 using System.Threading.Tasks;
+using NAudio.MediaFoundation;
 using xAiApi.Interfaces.Extractors;
-using DocumentFormat.OpenXml.EMMA;

 namespace xAiApi.Providers.Extractors
 {
@@ -24,29 +28,30 @@ namespace xAiApi.Providers.Extractors
            "audio/mp3",
            "audio/wav",
            "audio/wave",
+            "audio/x-wav",
            "audio/ogg",
            "audio/m4a",
            "audio/mp4",
+            "audio/aac",
+            "audio/x-m4a",
            "audio/webm"
        ];

-        private readonly string language;
-        private readonly string whisperModelPath;
-
-        public XAudioFileContentExtractor() : this(
-                language: "fa",
-                whisperModelPath: "Models/ggml-base.bin"
-        )
-        { }
+        private readonly XAiApiConfiguration configuration;

        public XAudioFileContentExtractor(
-            string whisperModelPath = "Models/ggml-base.bin",
-            string language = "fa"
+            XAiApiConfiguration configuration
        )
        {
            //
-            this.language = language;
-            this.whisperModelPath = whisperModelPath;
+            if (configuration.Transcript.IsNullOrDefault() ||
+                configuration.Transcript.ModelsPath.IsNullOrEmpty())
+            {
+                XException.InvalidConfiguration.Throw();
+            }
+
+            //
+            this.configuration = configuration;
        }

        /// <summary>
@@ -113,14 +118,15 @@ namespace xAiApi.Providers.Extractors

                //
                using var wavStream = await ConvertToWavAsync(
-                    memoryStream,
-                    cancellationToken
+                    mimeType: mimeType,
+                    inputStream: memoryStream,
+                    cancellationToken: cancellationToken
                );

                //
-                using var factory = WhisperFactory.FromPath(whisperModelPath);
+                using var factory = WhisperFactory.FromPath(configuration.Transcript.ModelsPath);
                using var processor = factory.CreateBuilder()
-                    .WithLanguage(language)
+                    .WithLanguage(configuration.Transcript.Language)
                    .Build();

                //
@@ -143,17 +149,186 @@ namespace xAiApi.Providers.Extractors
            return result;
        }

+        //
+        #region Private ...
        /// <summary>
-        /// Audio Format Converting ...
+        /// Audio Format Converting to WAV 16kHz Mono 16-bit PCM ...
        /// </summary>
        private async Task<Stream> ConvertToWavAsync(
            Stream inputStream,
-            CancellationToken cancellationToken
+            string mimeType,
+            CancellationToken cancellationToken = default
        )
        {
            //
-            // TODO: Complete this ...
-            return await Task.FromResult(inputStream);
+            var result = await Task.Run(() =>
+            {
+                //
+                var memoryStream = new MemoryStream();
+                inputStream.CopyTo(memoryStream);
+                memoryStream.Position = 0;
+
+                //
+                var targetFormat = new WaveFormat(16000, 16, 1);
+                string tempFilePath = null;
+
+                //
+                try
+                {
+                    //
+                    WaveStream reader = GetAudioReader(memoryStream, mimeType, ref tempFilePath);
+                    if (reader == null)
+                    {
+                        //
+                        XException.NotAllowed.Throw(
+                            $"Unsupported audio format: {mimeType}"
+                        );
+                    }
+
+                    //
+                    var needsConversion =
+                        reader.WaveFormat.SampleRate != 16000 ||
+                        reader.WaveFormat.Channels != 1 ||
+                        reader.WaveFormat.BitsPerSample != 16 ||
+                        reader.WaveFormat.Encoding != WaveFormatEncoding.Pcm;
+                    var outputStream = new MemoryStream();
+                    if (!needsConversion)
+                    {
+                        //
+                        reader.CopyTo(outputStream);
+                        reader.Dispose();
+                    }
+                    else
+                    {
+                        //
+                        MediaFoundationApi.Startup();
+                        using var resampler = new MediaFoundationResampler(
+                            reader,
+                            targetFormat
+                        );
+                        resampler.ResamplerQuality = 60;
+
+                        //
+                        WaveFileWriter.WriteWavFileToStream(outputStream, resampler);
+                        reader.Dispose();
+                    }
+
+                    //
+                    outputStream.Position = 0;
+                    return outputStream;
+                }
+                finally
+                {
+                    //
+                    if (!string.IsNullOrEmpty(tempFilePath) &&
+                        File.Exists(tempFilePath))
+                    {
+                        //
+                        try
+                        {
+                            File.Delete(tempFilePath);
+                        }
+                        catch { }
+                    }
+
+                    //
+                    memoryStream.Dispose();
+                }
+            }, cancellationToken);
+
+            //
+            return result;
        }
+
+        /// <summary>
+        /// Get appropriate WaveStream reader based on MIME type ...
+        /// </summary>
+        /// <param name="stream">Input stream (MemoryStream)</param>
+        /// <param name="mimeType">MIME type of audio file</param>
+        /// <param name="tempFilePath">Path to temp file (if created)</param>
+        /// <returns>WaveStream reader or null if unsupported</returns>
+        private WaveStream GetAudioReader(
+            MemoryStream stream,
+            string mimeType,
+            ref string tempFilePath
+        )
+        {
+            //
+            try
+            {
+                //
+                var normalizedMime = mimeType?.ToLowerInvariant() ?? string.Empty;
+                switch (normalizedMime)
+                {
+                    //
+                    case "audio/wav":
+                    case "audio/wave":
+                    case "audio/x-wav":
+                        return new WaveFileReader(stream);
+
+                    //
+                    case "audio/mpeg":
+                    case "audio/mp3":
+                        return new Mp3FileReader(stream);
+
+                    //
+                    case "audio/m4a":
+                    case "audio/mp4":
+                    case "audio/aac":
+                    case "audio/ogg":
+                    case "audio/webm":
+                    case "audio/x-m4a":
+                        //
+                        var extension = normalizedMime switch
+                        {
+                            "audio/m4a" or "audio/mp4" or "audio/aac" or "audio/x-m4a" => ".m4a",
+                            "audio/ogg" => ".ogg",
+                            "audio/webm" => ".webm",
+                            _ => ".tmp"
+                        };
+                        tempFilePath = Path.Combine(
+                            Path.GetTempPath(),
+                            $"audio_{Guid.NewGuid()}{extension}"
+                        );
+
+                        //
+                        stream.Position = 0;
+                        using (var fileStream = File.Create(tempFilePath))
+                        {
+                            stream.CopyTo(fileStream);
+                        }
+
+                        //
+                        MediaFoundationApi.Startup();
+                        return new MediaFoundationReader(tempFilePath);
+
+                    default:
+                        //
+                        tempFilePath = Path.Combine(
+                            Path.GetTempPath(),
+                            $"audio_{Guid.NewGuid()}.tmp"
+                        );
+                        stream.Position = 0;
+                        using (var fileStream = File.Create(tempFilePath))
+                        {
+                            stream.CopyTo(fileStream);
+                        }
+                        try
+                        {
+                            MediaFoundationApi.Startup();
+                            return new MediaFoundationReader(tempFilePath);
+                        }
+                        catch
+                        {
+                            return null;
+                        }
+                }
+            }
+            catch (Exception)
+            {
+                return null;
+            }
+        }
+        #endregion
    }
 }
@@ -84,6 +84,10 @@
      "EngineMode": "LstmOnly",
      "EnableOcrFallback": true,
      "DefaultLanguage": "fas+eng"
+    },
+    "Transcript": {
+      "Language": "fa",
+      "ModelsPath": "Models/ggml-base.bin"
    }
  },
  "Logging": {
@@ -27,13 +27,6 @@
    <None Include="../../Resources/Images/favicon.png" Link="icon.png" Pack="true" PackagePath="\icon.png" />
  </ItemGroup>

-  <!-- Native Libraries -->
-  <!-- <ItemGroup>
-    <None Include="runtimes/linux-x64/native/libpdfium.so">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </None>
-  </ItemGroup> -->
-
  <!-- Local Modules -->
  <ItemGroup>
  </ItemGroup>
@@ -53,22 +46,22 @@

  <!-- Dependencies -->
  <ItemGroup>
-    <PackageReference Include="DocumentFormat.OpenXml" Version="3.5.1" />
-    <PackageReference Include="ExcelDataReader" Version="3.9.0" />
-    <PackageReference Include="ExcelDataReader.DataSet" Version="3.9.0" />
    <PackageReference Include="NAudio" Version="2.4.0" />
+    <PackageReference Include="Tesseract" Version="5.2.0" />
+    <PackageReference Include="Whisper.net" Version="1.9.1" />
    <PackageReference Include="OllamaSharp" Version="5.4.25" />
    <PackageReference Include="PdfiumViewer" Version="2.13.0" />
-    <PackageReference Include="PdfiumViewer.Native.x86.no_v8-no_xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86.v8-xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86_64.no_v8-no_xfa" Version="2018.4.8.256" />
-    <PackageReference Include="PdfiumViewer.Native.x86_64.v8-xfa" Version="2018.4.8.256" />
-    <PackageReference Include="Tesseract" Version="5.2.0" />
-    <PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" />
-    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.11" />
-    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="5.0.0" />
-    <PackageReference Include="Whisper.net" Version="1.9.1" />
+    <PackageReference Include="ExcelDataReader" Version="3.9.0" />
    <PackageReference Include="Whisper.net.Runtime" Version="1.9.1" />
+    <PackageReference Include="DocumentFormat.OpenXml" Version="3.5.1" />
+    <PackageReference Include="ExcelDataReader.DataSet" Version="3.9.0" />
+    <PackageReference Include="UglyToad.PdfPig" Version="1.7.0-custom-5" />
+    <PackageReference Include="PdfiumViewer.Native.x86.v8-xfa" Version="2018.4.8.256" />
+    <PackageReference Include="PdfiumViewer.Native.x86_64.v8-xfa" Version="2018.4.8.256" />
+    <PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="3.1.11" />
+    <PackageReference Include="PdfiumViewer.Native.x86.no_v8-no_xfa" Version="2018.4.8.256" />
+    <PackageReference Include="IdentityModel.AspNetCore.OAuth2Introspection" Version="5.0.0" />
+    <PackageReference Include="PdfiumViewer.Native.x86_64.no_v8-no_xfa" Version="2018.4.8.256" />
  </ItemGroup>

  <!-- For XML Documentation Support -->