xSaherElmAiApi/Providers/Extractors/XAudioFileContentExtractor.cs

using System;
using System.IO;
using System.Linq;
using Whisper.net;
using NAudio.Wave;
using System.Threading;
using xAiModels.Models;
using xCommons.Extensions;
using xAiApi.Configurations;
using xExceptions.Constants;
using System.Threading.Tasks;
using NAudio.MediaFoundation;
using xAiApi.Interfaces.Extractors;

namespace xAiApi.Providers.Extractors
{
    /// <summary>
    /// Extracts text content from Audio files ...
    /// </summary>    
    public class XAudioFileContentExtractor : IXAudioFileContentExtractor
    {
        /// <summary>
        /// Supported MIME Types ...
        /// </summary>
        private static readonly string[] SupportedMimeTypes =
        [
            "audio/mpeg",
            "audio/mp3",
            "audio/wav",
            "audio/wave",
            "audio/x-wav",
            "audio/ogg",
            "audio/m4a",
            "audio/mp4",
            "audio/aac",
            "audio/x-m4a",
            "audio/webm"
        ];

        private readonly XAiApiConfiguration configuration;

        public XAudioFileContentExtractor(
            XAiApiConfiguration configuration
        )
        {
            //
            if (configuration.Transcript.IsNullOrDefault() ||
                configuration.Transcript.ModelsPath.IsNullOrEmpty())
            {
                XException.InvalidConfiguration.Throw();
            }

            //
            this.configuration = configuration;
        }

        /// <summary>
        /// Check if this extractor supports the specified MIME type ...
        /// </summary>
        public bool CanExtract(string mimeType)
        {
            //
            return SupportedMimeTypes.Contains(
                mimeType?.ToLowerInvariant() ?? string.Empty
            );
        }

        /// <summary>
        /// Extract text content from file stream ...
        /// </summary>
        public async Task<string> ExtractAsync(
            Stream fileStream,
            string mimeType,
            CancellationToken cancellationToken = default
        )
        {
            //
            var result = await ExtractRichAsync(
                fileStream,
                "audio",
                mimeType,
                cancellationToken
            );

            //
            return result.AudioTranscript;
        }

        /// <summary>
        /// Extract content from stream as Rich Result ...
        /// </summary>
        /// <param name="fileStream"></param>
        /// <param name="fileName"></param>
        /// <param name="mimeType"></param>
        /// <param name="cancellationToken"></param>
        /// <returns></returns>
        public async Task<XFileExtractionResult> ExtractRichAsync(
            Stream fileStream,
            string fileName,
            string mimeType,
            CancellationToken cancellationToken = default
        )
        {
            //
            var result = new XFileExtractionResult
            {
                FileName = fileName,
                MimeType = mimeType
            };

            //
            try
            {
                //
                using var memoryStream = new MemoryStream();
                await fileStream.CopyToAsync(memoryStream, cancellationToken);
                memoryStream.Position = 0;

                //
                using var wavStream = await ConvertToWavAsync(
                    mimeType: mimeType,
                    inputStream: memoryStream,
                    cancellationToken: cancellationToken
                );

                //
                using var factory = WhisperFactory.FromPath(configuration.Transcript.ModelsPath);
                using var processor = factory.CreateBuilder()
                    .WithLanguage(configuration.Transcript.Language)
                    .Build();

                //
                var segments = new System.Text.StringBuilder();
                await foreach (var segment in processor.ProcessAsync(wavStream, cancellationToken))
                {
                    segments.Append(segment.Text);
                }

                //
                result.AudioTranscript = segments.ToString().Trim();
                result.Text = result.AudioTranscript;
            }
            catch (Exception ex)
            {
                result.ErrorMessage = $"Audio extraction failed: {ex.Message}";
            }

            //
            return result;
        }

        //
        #region Private ...
        /// <summary>
        /// Audio Format Converting to WAV 16kHz Mono 16-bit PCM ...
        /// </summary>
        private async Task<Stream> ConvertToWavAsync(
            Stream inputStream,
            string mimeType,
            CancellationToken cancellationToken = default
        )
        {
            //
            var result = await Task.Run(() =>
            {
                //
                var memoryStream = new MemoryStream();
                inputStream.CopyTo(memoryStream);
                memoryStream.Position = 0;

                //
                var targetFormat = new WaveFormat(16000, 16, 1);
                string tempFilePath = null;

                //
                try
                {
                    //
                    WaveStream reader = GetAudioReader(memoryStream, mimeType, ref tempFilePath);
                    if (reader == null)
                    {
                        //
                        XException.NotAllowed.Throw(
                            $"Unsupported audio format: {mimeType}"
                        );
                    }

                    //
                    var needsConversion =
                        reader.WaveFormat.SampleRate != 16000 ||
                        reader.WaveFormat.Channels != 1 ||
                        reader.WaveFormat.BitsPerSample != 16 ||
                        reader.WaveFormat.Encoding != WaveFormatEncoding.Pcm;
                    var outputStream = new MemoryStream();
                    if (!needsConversion)
                    {
                        //
                        reader.CopyTo(outputStream);
                        reader.Dispose();
                    }
                    else
                    {
                        //
                        MediaFoundationApi.Startup();
                        using var resampler = new MediaFoundationResampler(
                            reader,
                            targetFormat
                        );
                        resampler.ResamplerQuality = 60;

                        //
                        WaveFileWriter.WriteWavFileToStream(outputStream, resampler);
                        reader.Dispose();
                    }

                    //
                    outputStream.Position = 0;
                    return outputStream;
                }
                finally
                {
                    //
                    if (!string.IsNullOrEmpty(tempFilePath) &&
                        File.Exists(tempFilePath))
                    {
                        //
                        try
                        {
                            File.Delete(tempFilePath);
                        }
                        catch { }
                    }

                    //
                    memoryStream.Dispose();
                }
            }, cancellationToken);

            //
            return result;
        }

        /// <summary>
        /// Get appropriate WaveStream reader based on MIME type ...
        /// </summary>
        /// <param name="stream">Input stream (MemoryStream)</param>
        /// <param name="mimeType">MIME type of audio file</param>
        /// <param name="tempFilePath">Path to temp file (if created)</param>
        /// <returns>WaveStream reader or null if unsupported</returns>
        private WaveStream GetAudioReader(
            MemoryStream stream,
            string mimeType,
            ref string tempFilePath
        )
        {
            //
            try
            {
                //
                var normalizedMime = mimeType?.ToLowerInvariant() ?? string.Empty;
                switch (normalizedMime)
                {
                    //
                    case "audio/wav":
                    case "audio/wave":
                    case "audio/x-wav":
                        return new WaveFileReader(stream);

                    //
                    case "audio/mpeg":
                    case "audio/mp3":
                        return new Mp3FileReader(stream);

                    //
                    case "audio/m4a":
                    case "audio/mp4":
                    case "audio/aac":
                    case "audio/ogg":
                    case "audio/webm":
                    case "audio/x-m4a":
                        //
                        var extension = normalizedMime switch
                        {
                            "audio/m4a" or "audio/mp4" or "audio/aac" or "audio/x-m4a" => ".m4a",
                            "audio/ogg" => ".ogg",
                            "audio/webm" => ".webm",
                            _ => ".tmp"
                        };
                        tempFilePath = Path.Combine(
                            Path.GetTempPath(),
                            $"audio_{Guid.NewGuid()}{extension}"
                        );

                        //
                        stream.Position = 0;
                        using (var fileStream = File.Create(tempFilePath))
                        {
                            stream.CopyTo(fileStream);
                        }

                        //
                        MediaFoundationApi.Startup();
                        return new MediaFoundationReader(tempFilePath);

                    default:
                        //
                        tempFilePath = Path.Combine(
                            Path.GetTempPath(),
                            $"audio_{Guid.NewGuid()}.tmp"
                        );
                        stream.Position = 0;
                        using (var fileStream = File.Create(tempFilePath))
                        {
                            stream.CopyTo(fileStream);
                        }
                        try
                        {
                            MediaFoundationApi.Startup();
                            return new MediaFoundationReader(tempFilePath);
                        }
                        catch
                        {
                            return null;
                        }
                }
            }
            catch (Exception)
            {
                return null;
            }
        }
        #endregion
    }
}