Compare commits

...
1 Commits
Author SHA1 Message Date
saherelm 4521a53734 fix file content extractor mechanism ... 2026-10-02 00:07:58 +03:30
13 changed files with 157 additions and 83 deletions
@@ -2,7 +2,7 @@ using System.IO;
using System.Threading; using System.Threading;
using System.Threading.Tasks; using System.Threading.Tasks;
namespace xAiApi.Interfaces namespace xAiApi.Interfaces.Extractors
{ {
/// <summary> /// <summary>
/// a Service for Extracting File Content ... /// a Service for Extracting File Content ...
@@ -0,0 +1,5 @@
namespace xAiApi.Interfaces.Extractors
{
public interface IXPdfFileContentExtractor : IXFileContentExtractor
{ }
}
@@ -0,0 +1,5 @@
namespace xAiApi.Interfaces.Extractors
{
public interface IXPlainTextFileContentExtractor : IXFileContentExtractor
{ }
}
@@ -0,0 +1,103 @@
using System;
using System.IO;
using System.Linq;
using System.Threading;
using System.Reflection;
using xCommons.Extensions;
using xExceptions.Constants;
using System.Threading.Tasks;
using System.Collections.Generic;
using xAiApi.Interfaces.Extractors;
namespace xAiApi.Providers
{
/// <summary>
/// Composite extractor that delegates to appropriate extractor
/// based on MIME type ...
/// </summary>
public class XFileContentExtractor : IXFileContentExtractor
{
//
private readonly IList<IXFileContentExtractor> extractors = [];
public XFileContentExtractor()
{
//
var path = Path.Combine(AppContext.BaseDirectory);
var files = Directory.GetFiles(path, "*.dll");
foreach (var dll in files)
{
//
// Retrieve DLL Assembly ...
var assembly = Assembly.LoadFrom(dll);
//
// Extract Registerar Types ...
var extractorTypes = assembly
.GetExportedTypes()
.Where(t =>
!t.IsAbstract &&
typeof(IXFileContentExtractor).IsAssignableFrom(t));
foreach (var extractorType in extractorTypes)
{
//
// Ignore XFileContentExtractor ...
if (extractorType.FullName.Contains(nameof(XFileContentExtractor)))
{
continue;
}
//
// Create Instance of Service ...
var instance = (IXFileContentExtractor)Activator.CreateInstance(extractorType);
if (!instance.IsNull())
{
//
// Add Registerar Instance to Collection ...
extractors.Add(instance);
}
}
}
}
/// <summary>
/// Check if this extractor supports the specified MIME type ...
/// </summary>
public bool CanExtract(string mimeType)
{
return extractors.Any(e => e.CanExtract(mimeType));
}
/// <summary>
/// Extract text content from file stream ...
/// </summary>
public async Task<string> ExtractAsync(
Stream fileStream,
string mimeType,
CancellationToken cancellationToken = default
)
{
//
var extractor = extractors
.FirstOrDefault(e => e.CanExtract(mimeType));
if (extractor.IsNull())
{
//
XException.NotAllowed.Throw(
$"Unsupported file type: {mimeType}"
);
}
//
var result = await extractor
.ExtractAsync(
mimeType: mimeType,
fileStream: fileStream,
cancellationToken: cancellationToken
);
//
return result;
}
}
}
@@ -4,15 +4,15 @@ using System.Linq;
using System.Text; using System.Text;
using UglyToad.PdfPig; using UglyToad.PdfPig;
using System.Threading; using System.Threading;
using xAiApi.Interfaces;
using System.Threading.Tasks; using System.Threading.Tasks;
using xAiApi.Interfaces.Extractors;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
/// <summary> /// <summary>
/// Extracts content from PDF files using PdfPig ... /// Extracts content from PDF files using PdfPig ...
/// </summary> /// </summary>
public class XPdfContentExtractor : IXFileContentExtractor public class XPdfFileContentExtractor : IXPdfFileContentExtractor
{ {
/// <summary> /// <summary>
/// Supported MIME Types ... /// Supported MIME Types ...
@@ -2,15 +2,15 @@ using System;
using System.IO; using System.IO;
using System.Linq; using System.Linq;
using System.Threading; using System.Threading;
using xAiApi.Interfaces;
using System.Threading.Tasks; using System.Threading.Tasks;
using xAiApi.Interfaces.Extractors;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
/// <summary> /// <summary>
/// Extracts content from plain text files (txt, md, csv, json, xml) ... /// Extracts content from plain text files (txt, md, csv, json, xml) ...
/// </summary> /// </summary>
public class XPlainTextContentExtractor : IXFileContentExtractor public class XPlainTextFileContentExtractor : IXPlainTextFileContentExtractor
{ {
/// <summary> /// <summary>
/// Supported MIME Types ... /// Supported MIME Types ...
+4
View File
@@ -27,6 +27,7 @@ using Microsoft.Extensions.AI;
using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Http;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using xAiApi.Interfaces.Extractors;
using System.ClientModel.Primitives; using System.ClientModel.Primitives;
using System.Runtime.CompilerServices; using System.Runtime.CompilerServices;
@@ -52,6 +53,7 @@ namespace xAiApi.Providers
private readonly ILogger<XAIServiceBase> logger; private readonly ILogger<XAIServiceBase> logger;
private readonly XAiApiConfiguration configuration; private readonly XAiApiConfiguration configuration;
private readonly XValidationProvider validationProvider; private readonly XValidationProvider validationProvider;
private readonly IXFileContentExtractor fileContentExtractor;
protected XAIServiceBase( protected XAIServiceBase(
IXFileProvider fileProvider, IXFileProvider fileProvider,
@@ -59,6 +61,7 @@ namespace xAiApi.Providers
ILogger<XAIServiceBase> logger, ILogger<XAIServiceBase> logger,
XAiApiConfiguration configuration, XAiApiConfiguration configuration,
XValidationProvider validationProvider, XValidationProvider validationProvider,
IXFileContentExtractor fileContentExtractor,
string model = null string model = null
) )
{ {
@@ -68,6 +71,7 @@ namespace xAiApi.Providers
this.dataProvider = dataProvider; this.dataProvider = dataProvider;
this.configuration = configuration; this.configuration = configuration;
this.validationProvider = validationProvider; this.validationProvider = validationProvider;
this.fileContentExtractor = fileContentExtractor;
// //
// Prepare Embedding Model Descriptor ... // Prepare Embedding Model Descriptor ...
+13 -1
View File
@@ -20,7 +20,9 @@ using xFileService.Interfaces;
using Microsoft.Extensions.AI; using Microsoft.Extensions.AI;
using System.Collections.Generic; using System.Collections.Generic;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using xAiApi.Interfaces.Extractors;
using System.ClientModel.Primitives; using System.ClientModel.Primitives;
using xCommons.Providers;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
@@ -37,9 +39,19 @@ namespace xAiApi.Providers
IXFileProvider fileProvider, IXFileProvider fileProvider,
IXAiDataProvider dataProvider, IXAiDataProvider dataProvider,
XAiApiConfiguration configuration, XAiApiConfiguration configuration,
XValidationProvider validationProvider,
ILogger<XAiEmbeddingServiceBase> logger, ILogger<XAiEmbeddingServiceBase> logger,
IXFileContentExtractor fileContentExtractor,
string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName
) : base(fileProvider, dataProvider, logger, configuration, null) ) : base(
fileProvider,
dataProvider,
logger,
configuration,
validationProvider,
fileContentExtractor,
null
)
{ {
// //
// Prepare Embedding Model Descriptor ... // Prepare Embedding Model Descriptor ...
@@ -1,68 +0,0 @@
using System.IO;
using System.Linq;
using System.Threading;
using xAiApi.Interfaces;
using xCommons.Extensions;
using xExceptions.Constants;
using System.Threading.Tasks;
using System.Collections.Generic;
namespace xAiApi.Providers
{
/// <summary>
/// Composite extractor that delegates to appropriate extractor
/// based on MIME type ...
/// </summary>
public class XCompositeFileContentExtractor : IXFileContentExtractor
{
//
private readonly IEnumerable<IXFileContentExtractor> extractors;
public XCompositeFileContentExtractor(
IEnumerable<IXFileContentExtractor> extractors
)
{
this.extractors = extractors;
}
/// <summary>
/// Check if this extractor supports the specified MIME type ...
/// </summary>
public bool CanExtract(string mimeType)
{
return extractors.Any(e => e.CanExtract(mimeType));
}
/// <summary>
/// Extract text content from file stream ...
/// </summary>
public async Task<string> ExtractAsync(
Stream fileStream,
string mimeType,
CancellationToken cancellationToken = default
)
{
//
var extractor = extractors
.FirstOrDefault(e => e.CanExtract(mimeType));
if (extractor.IsNull())
{
//
XException.NotAllowed.Throw(
$"Unsupported file type: {mimeType}"
);
}
//
var result = await extractor
.ExtractAsync(
mimeType: mimeType,
fileStream: fileStream,
cancellationToken: cancellationToken
);
//
return result;
}
}
}
+4 -1
View File
@@ -5,6 +5,7 @@ using xAiModels.Interfaces;
using xAiApi.Configurations; using xAiApi.Configurations;
using xFileService.Interfaces; using xFileService.Interfaces;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using xAiApi.Interfaces.Extractors;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
@@ -15,13 +16,15 @@ namespace xAiApi.Providers
IXAiDataProvider dataProvider, IXAiDataProvider dataProvider,
ILogger<XDefaultAiService> logger, ILogger<XDefaultAiService> logger,
XAiApiConfiguration configuration, XAiApiConfiguration configuration,
XValidationProvider validationProvider XValidationProvider validationProvider,
IXFileContentExtractor fileContentExtractor
) : base( ) : base(
fileProvider, fileProvider,
dataProvider, dataProvider,
logger, logger,
configuration, configuration,
validationProvider, validationProvider,
fileContentExtractor,
XAiApiConstants.XAiDefaultModelName XAiApiConstants.XAiDefaultModelName
) )
{ } { }
+7 -1
View File
@@ -4,6 +4,8 @@ using xAiModels.Interfaces;
using xAiApi.Configurations; using xAiApi.Configurations;
using xFileService.Interfaces; using xFileService.Interfaces;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using xAiApi.Interfaces.Extractors;
using xCommons.Providers;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
@@ -13,12 +15,16 @@ namespace xAiApi.Providers
IXFileProvider fileProvider, IXFileProvider fileProvider,
IXAiDataProvider dataProvider, IXAiDataProvider dataProvider,
XAiApiConfiguration configuration, XAiApiConfiguration configuration,
ILogger<XDefaultEmbeddingService> logger XValidationProvider validationProvider,
ILogger<XDefaultEmbeddingService> logger,
IXFileContentExtractor fileContentExtractor
) : base( ) : base(
fileProvider, fileProvider,
dataProvider, dataProvider,
configuration, configuration,
validationProvider,
logger, logger,
fileContentExtractor,
XAiApiConstants.XAiEmbeddingModelName XAiApiConstants.XAiEmbeddingModelName
) )
{ } { }
+3
View File
@@ -6,6 +6,7 @@ using xAiApi.Configurations;
using Microsoft.Extensions.AI; using Microsoft.Extensions.AI;
using xFileService.Interfaces; using xFileService.Interfaces;
using Microsoft.Extensions.Logging; using Microsoft.Extensions.Logging;
using xAiApi.Interfaces.Extractors;
namespace xAiApi.Providers namespace xAiApi.Providers
{ {
@@ -17,6 +18,7 @@ namespace xAiApi.Providers
ILogger<XAIServiceBase> logger, ILogger<XAIServiceBase> logger,
XAiApiConfiguration configuration, XAiApiConfiguration configuration,
XValidationProvider validationProvider, XValidationProvider validationProvider,
IXFileContentExtractor fileContentExtractor,
ReasoningEffort effort = ReasoningEffort.Medium, ReasoningEffort effort = ReasoningEffort.Medium,
ReasoningOutput output = ReasoningOutput.Full ReasoningOutput output = ReasoningOutput.Full
) : base( ) : base(
@@ -25,6 +27,7 @@ namespace xAiApi.Providers
logger, logger,
configuration, configuration,
validationProvider, validationProvider,
fileContentExtractor,
XAiApiConstants.XAiDefaultThinkingModelName XAiApiConstants.XAiDefaultThinkingModelName
) )
{ {
+4 -3
View File
@@ -26,6 +26,7 @@ using System.Collections.Generic;
using Microsoft.AspNetCore.Builder; using Microsoft.AspNetCore.Builder;
using Microsoft.AspNetCore.Hosting; using Microsoft.AspNetCore.Hosting;
using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Hosting;
using xAiApi.Interfaces.Extractors;
using Newtonsoft.Json.Serialization; using Newtonsoft.Json.Serialization;
using Microsoft.Extensions.Configuration; using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.DependencyInjection;
@@ -187,9 +188,9 @@ namespace xAiApi
// //
// Register File Content Extractors ... // Register File Content Extractors ...
services.AddSingleton<IXFileContentExtractor, XPdfContentExtractor>(); // services.AddSingleton<IXFileContentExtractor, XPdfContentExtractor>();
services.AddSingleton<IXFileContentExtractor, XPlainTextContentExtractor>(); // services.AddSingleton<IXFileContentExtractor, XPlainTextContentExtractor>();
services.AddSingleton<IXFileContentExtractor, XCompositeFileContentExtractor>(); services.AddSingleton<IXFileContentExtractor, XFileContentExtractor>();
// //
// Register Default Ai Service ... // Register Default Ai Service ...