Compare commits
1
Commits
f32972b55f
...
4521a53734
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
4521a53734 |
+1
-1
@@ -2,7 +2,7 @@ using System.IO;
|
|||||||
using System.Threading;
|
using System.Threading;
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
|
||||||
namespace xAiApi.Interfaces
|
namespace xAiApi.Interfaces.Extractors
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// a Service for Extracting File Content ...
|
/// a Service for Extracting File Content ...
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
namespace xAiApi.Interfaces.Extractors
|
||||||
|
{
|
||||||
|
public interface IXPdfFileContentExtractor : IXFileContentExtractor
|
||||||
|
{ }
|
||||||
|
}
|
||||||
@@ -0,0 +1,5 @@
|
|||||||
|
namespace xAiApi.Interfaces.Extractors
|
||||||
|
{
|
||||||
|
public interface IXPlainTextFileContentExtractor : IXFileContentExtractor
|
||||||
|
{ }
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
using System.Linq;
|
||||||
|
using System.Threading;
|
||||||
|
using System.Reflection;
|
||||||
|
using xCommons.Extensions;
|
||||||
|
using xExceptions.Constants;
|
||||||
|
using System.Threading.Tasks;
|
||||||
|
using System.Collections.Generic;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
|
||||||
|
namespace xAiApi.Providers
|
||||||
|
{
|
||||||
|
/// <summary>
|
||||||
|
/// Composite extractor that delegates to appropriate extractor
|
||||||
|
/// based on MIME type ...
|
||||||
|
/// </summary>
|
||||||
|
public class XFileContentExtractor : IXFileContentExtractor
|
||||||
|
{
|
||||||
|
//
|
||||||
|
private readonly IList<IXFileContentExtractor> extractors = [];
|
||||||
|
|
||||||
|
public XFileContentExtractor()
|
||||||
|
{
|
||||||
|
//
|
||||||
|
var path = Path.Combine(AppContext.BaseDirectory);
|
||||||
|
var files = Directory.GetFiles(path, "*.dll");
|
||||||
|
foreach (var dll in files)
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Retrieve DLL Assembly ...
|
||||||
|
var assembly = Assembly.LoadFrom(dll);
|
||||||
|
|
||||||
|
//
|
||||||
|
// Extract Registerar Types ...
|
||||||
|
var extractorTypes = assembly
|
||||||
|
.GetExportedTypes()
|
||||||
|
.Where(t =>
|
||||||
|
!t.IsAbstract &&
|
||||||
|
typeof(IXFileContentExtractor).IsAssignableFrom(t));
|
||||||
|
foreach (var extractorType in extractorTypes)
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Ignore XFileContentExtractor ...
|
||||||
|
if (extractorType.FullName.Contains(nameof(XFileContentExtractor)))
|
||||||
|
{
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
// Create Instance of Service ...
|
||||||
|
var instance = (IXFileContentExtractor)Activator.CreateInstance(extractorType);
|
||||||
|
if (!instance.IsNull())
|
||||||
|
{
|
||||||
|
//
|
||||||
|
// Add Registerar Instance to Collection ...
|
||||||
|
extractors.Add(instance);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Check if this extractor supports the specified MIME type ...
|
||||||
|
/// </summary>
|
||||||
|
public bool CanExtract(string mimeType)
|
||||||
|
{
|
||||||
|
return extractors.Any(e => e.CanExtract(mimeType));
|
||||||
|
}
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Extract text content from file stream ...
|
||||||
|
/// </summary>
|
||||||
|
public async Task<string> ExtractAsync(
|
||||||
|
Stream fileStream,
|
||||||
|
string mimeType,
|
||||||
|
CancellationToken cancellationToken = default
|
||||||
|
)
|
||||||
|
{
|
||||||
|
//
|
||||||
|
var extractor = extractors
|
||||||
|
.FirstOrDefault(e => e.CanExtract(mimeType));
|
||||||
|
if (extractor.IsNull())
|
||||||
|
{
|
||||||
|
//
|
||||||
|
XException.NotAllowed.Throw(
|
||||||
|
$"Unsupported file type: {mimeType}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
//
|
||||||
|
var result = await extractor
|
||||||
|
.ExtractAsync(
|
||||||
|
mimeType: mimeType,
|
||||||
|
fileStream: fileStream,
|
||||||
|
cancellationToken: cancellationToken
|
||||||
|
);
|
||||||
|
|
||||||
|
//
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
+2
-2
@@ -4,15 +4,15 @@ using System.Linq;
|
|||||||
using System.Text;
|
using System.Text;
|
||||||
using UglyToad.PdfPig;
|
using UglyToad.PdfPig;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using xAiApi.Interfaces;
|
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Extracts content from PDF files using PdfPig ...
|
/// Extracts content from PDF files using PdfPig ...
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class XPdfContentExtractor : IXFileContentExtractor
|
public class XPdfFileContentExtractor : IXPdfFileContentExtractor
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Supported MIME Types ...
|
/// Supported MIME Types ...
|
||||||
+2
-2
@@ -2,15 +2,15 @@ using System;
|
|||||||
using System.IO;
|
using System.IO;
|
||||||
using System.Linq;
|
using System.Linq;
|
||||||
using System.Threading;
|
using System.Threading;
|
||||||
using xAiApi.Interfaces;
|
|
||||||
using System.Threading.Tasks;
|
using System.Threading.Tasks;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Extracts content from plain text files (txt, md, csv, json, xml) ...
|
/// Extracts content from plain text files (txt, md, csv, json, xml) ...
|
||||||
/// </summary>
|
/// </summary>
|
||||||
public class XPlainTextContentExtractor : IXFileContentExtractor
|
public class XPlainTextFileContentExtractor : IXPlainTextFileContentExtractor
|
||||||
{
|
{
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Supported MIME Types ...
|
/// Supported MIME Types ...
|
||||||
@@ -27,6 +27,7 @@ using Microsoft.Extensions.AI;
|
|||||||
using Microsoft.AspNetCore.Http;
|
using Microsoft.AspNetCore.Http;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
using System.ClientModel.Primitives;
|
using System.ClientModel.Primitives;
|
||||||
using System.Runtime.CompilerServices;
|
using System.Runtime.CompilerServices;
|
||||||
|
|
||||||
@@ -52,6 +53,7 @@ namespace xAiApi.Providers
|
|||||||
private readonly ILogger<XAIServiceBase> logger;
|
private readonly ILogger<XAIServiceBase> logger;
|
||||||
private readonly XAiApiConfiguration configuration;
|
private readonly XAiApiConfiguration configuration;
|
||||||
private readonly XValidationProvider validationProvider;
|
private readonly XValidationProvider validationProvider;
|
||||||
|
private readonly IXFileContentExtractor fileContentExtractor;
|
||||||
|
|
||||||
protected XAIServiceBase(
|
protected XAIServiceBase(
|
||||||
IXFileProvider fileProvider,
|
IXFileProvider fileProvider,
|
||||||
@@ -59,6 +61,7 @@ namespace xAiApi.Providers
|
|||||||
ILogger<XAIServiceBase> logger,
|
ILogger<XAIServiceBase> logger,
|
||||||
XAiApiConfiguration configuration,
|
XAiApiConfiguration configuration,
|
||||||
XValidationProvider validationProvider,
|
XValidationProvider validationProvider,
|
||||||
|
IXFileContentExtractor fileContentExtractor,
|
||||||
string model = null
|
string model = null
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
@@ -68,6 +71,7 @@ namespace xAiApi.Providers
|
|||||||
this.dataProvider = dataProvider;
|
this.dataProvider = dataProvider;
|
||||||
this.configuration = configuration;
|
this.configuration = configuration;
|
||||||
this.validationProvider = validationProvider;
|
this.validationProvider = validationProvider;
|
||||||
|
this.fileContentExtractor = fileContentExtractor;
|
||||||
|
|
||||||
//
|
//
|
||||||
// Prepare Embedding Model Descriptor ...
|
// Prepare Embedding Model Descriptor ...
|
||||||
|
|||||||
@@ -20,7 +20,9 @@ using xFileService.Interfaces;
|
|||||||
using Microsoft.Extensions.AI;
|
using Microsoft.Extensions.AI;
|
||||||
using System.Collections.Generic;
|
using System.Collections.Generic;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
using System.ClientModel.Primitives;
|
using System.ClientModel.Primitives;
|
||||||
|
using xCommons.Providers;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
@@ -37,9 +39,19 @@ namespace xAiApi.Providers
|
|||||||
IXFileProvider fileProvider,
|
IXFileProvider fileProvider,
|
||||||
IXAiDataProvider dataProvider,
|
IXAiDataProvider dataProvider,
|
||||||
XAiApiConfiguration configuration,
|
XAiApiConfiguration configuration,
|
||||||
|
XValidationProvider validationProvider,
|
||||||
ILogger<XAiEmbeddingServiceBase> logger,
|
ILogger<XAiEmbeddingServiceBase> logger,
|
||||||
|
IXFileContentExtractor fileContentExtractor,
|
||||||
string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName
|
string embeddingModelName = XAiApiConstants.XAiEmbeddingModelName
|
||||||
) : base(fileProvider, dataProvider, logger, configuration, null)
|
) : base(
|
||||||
|
fileProvider,
|
||||||
|
dataProvider,
|
||||||
|
logger,
|
||||||
|
configuration,
|
||||||
|
validationProvider,
|
||||||
|
fileContentExtractor,
|
||||||
|
null
|
||||||
|
)
|
||||||
{
|
{
|
||||||
//
|
//
|
||||||
// Prepare Embedding Model Descriptor ...
|
// Prepare Embedding Model Descriptor ...
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
using System.IO;
|
|
||||||
using System.Linq;
|
|
||||||
using System.Threading;
|
|
||||||
using xAiApi.Interfaces;
|
|
||||||
using xCommons.Extensions;
|
|
||||||
using xExceptions.Constants;
|
|
||||||
using System.Threading.Tasks;
|
|
||||||
using System.Collections.Generic;
|
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
|
||||||
{
|
|
||||||
/// <summary>
|
|
||||||
/// Composite extractor that delegates to appropriate extractor
|
|
||||||
/// based on MIME type ...
|
|
||||||
/// </summary>
|
|
||||||
public class XCompositeFileContentExtractor : IXFileContentExtractor
|
|
||||||
{
|
|
||||||
//
|
|
||||||
private readonly IEnumerable<IXFileContentExtractor> extractors;
|
|
||||||
|
|
||||||
public XCompositeFileContentExtractor(
|
|
||||||
IEnumerable<IXFileContentExtractor> extractors
|
|
||||||
)
|
|
||||||
{
|
|
||||||
this.extractors = extractors;
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Check if this extractor supports the specified MIME type ...
|
|
||||||
/// </summary>
|
|
||||||
public bool CanExtract(string mimeType)
|
|
||||||
{
|
|
||||||
return extractors.Any(e => e.CanExtract(mimeType));
|
|
||||||
}
|
|
||||||
|
|
||||||
/// <summary>
|
|
||||||
/// Extract text content from file stream ...
|
|
||||||
/// </summary>
|
|
||||||
public async Task<string> ExtractAsync(
|
|
||||||
Stream fileStream,
|
|
||||||
string mimeType,
|
|
||||||
CancellationToken cancellationToken = default
|
|
||||||
)
|
|
||||||
{
|
|
||||||
//
|
|
||||||
var extractor = extractors
|
|
||||||
.FirstOrDefault(e => e.CanExtract(mimeType));
|
|
||||||
if (extractor.IsNull())
|
|
||||||
{
|
|
||||||
//
|
|
||||||
XException.NotAllowed.Throw(
|
|
||||||
$"Unsupported file type: {mimeType}"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
//
|
|
||||||
var result = await extractor
|
|
||||||
.ExtractAsync(
|
|
||||||
mimeType: mimeType,
|
|
||||||
fileStream: fileStream,
|
|
||||||
cancellationToken: cancellationToken
|
|
||||||
);
|
|
||||||
|
|
||||||
//
|
|
||||||
return result;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -5,6 +5,7 @@ using xAiModels.Interfaces;
|
|||||||
using xAiApi.Configurations;
|
using xAiApi.Configurations;
|
||||||
using xFileService.Interfaces;
|
using xFileService.Interfaces;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
@@ -15,13 +16,15 @@ namespace xAiApi.Providers
|
|||||||
IXAiDataProvider dataProvider,
|
IXAiDataProvider dataProvider,
|
||||||
ILogger<XDefaultAiService> logger,
|
ILogger<XDefaultAiService> logger,
|
||||||
XAiApiConfiguration configuration,
|
XAiApiConfiguration configuration,
|
||||||
XValidationProvider validationProvider
|
XValidationProvider validationProvider,
|
||||||
|
IXFileContentExtractor fileContentExtractor
|
||||||
) : base(
|
) : base(
|
||||||
fileProvider,
|
fileProvider,
|
||||||
dataProvider,
|
dataProvider,
|
||||||
logger,
|
logger,
|
||||||
configuration,
|
configuration,
|
||||||
validationProvider,
|
validationProvider,
|
||||||
|
fileContentExtractor,
|
||||||
XAiApiConstants.XAiDefaultModelName
|
XAiApiConstants.XAiDefaultModelName
|
||||||
)
|
)
|
||||||
{ }
|
{ }
|
||||||
|
|||||||
@@ -4,6 +4,8 @@ using xAiModels.Interfaces;
|
|||||||
using xAiApi.Configurations;
|
using xAiApi.Configurations;
|
||||||
using xFileService.Interfaces;
|
using xFileService.Interfaces;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
using xCommons.Providers;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
@@ -13,12 +15,16 @@ namespace xAiApi.Providers
|
|||||||
IXFileProvider fileProvider,
|
IXFileProvider fileProvider,
|
||||||
IXAiDataProvider dataProvider,
|
IXAiDataProvider dataProvider,
|
||||||
XAiApiConfiguration configuration,
|
XAiApiConfiguration configuration,
|
||||||
ILogger<XDefaultEmbeddingService> logger
|
XValidationProvider validationProvider,
|
||||||
|
ILogger<XDefaultEmbeddingService> logger,
|
||||||
|
IXFileContentExtractor fileContentExtractor
|
||||||
) : base(
|
) : base(
|
||||||
fileProvider,
|
fileProvider,
|
||||||
dataProvider,
|
dataProvider,
|
||||||
configuration,
|
configuration,
|
||||||
|
validationProvider,
|
||||||
logger,
|
logger,
|
||||||
|
fileContentExtractor,
|
||||||
XAiApiConstants.XAiEmbeddingModelName
|
XAiApiConstants.XAiEmbeddingModelName
|
||||||
)
|
)
|
||||||
{ }
|
{ }
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ using xAiApi.Configurations;
|
|||||||
using Microsoft.Extensions.AI;
|
using Microsoft.Extensions.AI;
|
||||||
using xFileService.Interfaces;
|
using xFileService.Interfaces;
|
||||||
using Microsoft.Extensions.Logging;
|
using Microsoft.Extensions.Logging;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
|
|
||||||
namespace xAiApi.Providers
|
namespace xAiApi.Providers
|
||||||
{
|
{
|
||||||
@@ -17,6 +18,7 @@ namespace xAiApi.Providers
|
|||||||
ILogger<XAIServiceBase> logger,
|
ILogger<XAIServiceBase> logger,
|
||||||
XAiApiConfiguration configuration,
|
XAiApiConfiguration configuration,
|
||||||
XValidationProvider validationProvider,
|
XValidationProvider validationProvider,
|
||||||
|
IXFileContentExtractor fileContentExtractor,
|
||||||
ReasoningEffort effort = ReasoningEffort.Medium,
|
ReasoningEffort effort = ReasoningEffort.Medium,
|
||||||
ReasoningOutput output = ReasoningOutput.Full
|
ReasoningOutput output = ReasoningOutput.Full
|
||||||
) : base(
|
) : base(
|
||||||
@@ -25,6 +27,7 @@ namespace xAiApi.Providers
|
|||||||
logger,
|
logger,
|
||||||
configuration,
|
configuration,
|
||||||
validationProvider,
|
validationProvider,
|
||||||
|
fileContentExtractor,
|
||||||
XAiApiConstants.XAiDefaultThinkingModelName
|
XAiApiConstants.XAiDefaultThinkingModelName
|
||||||
)
|
)
|
||||||
{
|
{
|
||||||
|
|||||||
+4
-3
@@ -26,6 +26,7 @@ using System.Collections.Generic;
|
|||||||
using Microsoft.AspNetCore.Builder;
|
using Microsoft.AspNetCore.Builder;
|
||||||
using Microsoft.AspNetCore.Hosting;
|
using Microsoft.AspNetCore.Hosting;
|
||||||
using Microsoft.Extensions.Hosting;
|
using Microsoft.Extensions.Hosting;
|
||||||
|
using xAiApi.Interfaces.Extractors;
|
||||||
using Newtonsoft.Json.Serialization;
|
using Newtonsoft.Json.Serialization;
|
||||||
using Microsoft.Extensions.Configuration;
|
using Microsoft.Extensions.Configuration;
|
||||||
using Microsoft.Extensions.DependencyInjection;
|
using Microsoft.Extensions.DependencyInjection;
|
||||||
@@ -187,9 +188,9 @@ namespace xAiApi
|
|||||||
|
|
||||||
//
|
//
|
||||||
// Register File Content Extractors ...
|
// Register File Content Extractors ...
|
||||||
services.AddSingleton<IXFileContentExtractor, XPdfContentExtractor>();
|
// services.AddSingleton<IXFileContentExtractor, XPdfContentExtractor>();
|
||||||
services.AddSingleton<IXFileContentExtractor, XPlainTextContentExtractor>();
|
// services.AddSingleton<IXFileContentExtractor, XPlainTextContentExtractor>();
|
||||||
services.AddSingleton<IXFileContentExtractor, XCompositeFileContentExtractor>();
|
services.AddSingleton<IXFileContentExtractor, XFileContentExtractor>();
|
||||||
|
|
||||||
//
|
//
|
||||||
// Register Default Ai Service ...
|
// Register Default Ai Service ...
|
||||||
|
|||||||
Reference in New Issue
Block a user