MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-05 14:49:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

ensured data sources can be renamed and made it secure

Browse Source
This commit is contained in:
Paul Koudelka committed 2026-08-11 15:22:07 +02:00
1 parent 202e37410b
commit 92e986d9b9
13 files changed
+137 -66
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+3
View File
@@ -10030,6 +10030,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T4001510395"
-- Please select a compliance level.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T4066952091"] = "Please select a compliance level."
-- The name must not contain control characters.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T4234589878"] = "The name must not contain control characters."
-- The selected embedding provider has confidence '{0}', but this data source requires provider confidence '{1}'. Select an embedding provider with equal or higher confidence or lower the compliance level.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T476537662"] = "The selected embedding provider has confidence '{0}', but this data source requires provider confidence '{1}'. Select an embedding provider with equal or higher confidence or lower the compliance level."
app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs
+3 -3
View File
@@ -168,9 +168,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return false;
}
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name))
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl))
{
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name. (Plugin ID: {configPluginId})");
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})");
return false;
}
@@ -406,4 +406,4 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
var cleanedHostname = hostname.Trim();
return cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname;
}
}
}
app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs
+1 -1
View File
@@ -22,7 +22,7 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
await Task.CompletedTask;
}
public override Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
public override Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs
+3 -3
View File
@@ -82,8 +82,8 @@ public sealed class QdrantEdgeClientImplementation(
yield return (TB("Number of vector stores"), displayStoresCount.ToString());
}
public override Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, vectorSize), token);
public override Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token);
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
@@ -121,7 +121,7 @@ public sealed class QdrantEdgeClientImplementation(
}
// ReSharper disable NotAccessedPositionalProperty.Local
private sealed record EnsureVectorStoreRequest(string StoreName, int VectorSize);
private sealed record EnsureVectorStoreRequest(string StoreName, string DataSourceName, int VectorSize);
private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList<VectorStoragePoint> Points);
app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs
+1 -1
View File
@@ -2,7 +2,7 @@
public abstract class VectorStoreClient(string name, string path): DatabaseClient(name, path)
{
public abstract Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token);
public abstract Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingNames.cs
+4 -13
View File
@@ -2,20 +2,11 @@ namespace AIStudio.Tools.Services;
internal static class DataSourceEmbeddingNames
{
public static string GetCollectionName(string dataSourceName, string dataSourceId)
public static string GetCollectionName(string dataSourceId)
{
var safeId = dataSourceId
.ToLowerInvariant()
.Replace("-", string.Empty, StringComparison.Ordinal);
if (!Guid.TryParse(dataSourceId, out var parsedDataSourceId))
throw new ArgumentException("Data source ID must be a valid GUID.", nameof(dataSourceId));
var safeName = new string(dataSourceName
.ToLowerInvariant()
.Where(c => c is >= 'a' and <= 'z' or >= '0' and <= '9')
.Take(32)
.ToArray());
safeName = string.IsNullOrWhiteSpace(safeName) ? "datasource" : safeName;
return $"rag_{safeName}_{safeId}";
return $"rag_{parsedDataSourceId:N}";
}
}
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
+2 -2
View File
@@ -1158,8 +1158,8 @@ public sealed partial class DataSourceEmbeddingService
: null;
}
private string GetCollectionName(string dataSourceName, string dataSourceId) =>
DataSourceEmbeddingNames.GetCollectionName(dataSourceName, dataSourceId);
private string GetCollectionName(string dataSourceId) =>
DataSourceEmbeddingNames.GetCollectionName(dataSourceId);
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
+1 -2
View File
@@ -6,13 +6,12 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService
{
private async Task ResetPersistedStateAsync(
string dataSourceName,
string dataSourceId,
VectorStoreClient? vectorStore,
EmbeddingStateClient? embeddingState,
CancellationToken token)
{
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceName, dataSourceId), vectorStore, token);
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
+20 -7
View File
@@ -285,7 +285,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
this.statuses.TryRemove(dataSource.Id, out _);
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, null, null, CancellationToken.None);
await this.ResetPersistedStateAsync(dataSource.Id, null, null, CancellationToken.None);
this.statuses.TryRemove(dataSource.Id, out _);
this.PublishStatusChanged();
}
@@ -460,13 +460,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id);
var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
var collectionName = this.GetCollectionName(dataSource.Id);
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token);
token.ThrowIfCancellationRequested();
if (manifest.VectorSize > 0)
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, manifest.VectorSize, token);
var inputFiles = this.GetInputFiles(dataSource);
var indexedFiles = inputFiles.Files;
var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
foreach (var failure in inputFiles.Failures)
{
logger.LogWarning(
"Cannot index data source input '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}). Reason='{Reason}'.",
failure.FilePath,
dataSource.Name,
dataSource.Id,
failure.Reason);
}
logger.LogInformation(
"Prepared data source '{DataSourceName}' ({DataSourceId}) for embedding. AccessibleFiles={AccessibleFiles}, FailedFiles={FailedFiles}, Collection='{CollectionName}'.",
dataSource.Name,
@@ -659,7 +672,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
VectorStoreOptimizationTracker optimizationTracker,
CancellationToken token)
{
var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
var collectionName = this.GetCollectionName(dataSource.Id);
logger.LogDebug(
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
file.FullName,
@@ -757,7 +770,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (manifest.VectorSize == 0)
{
token.ThrowIfCancellationRequested();
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, vectorSize, token);
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
manifest.VectorSize = vectorSize;
logger.LogInformation(
@@ -806,9 +819,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
batch.Clear();
}
private async Task EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, int vectorSize, CancellationToken token)
private async Task EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
{
await vectorStore.EnsureVectorStoreExists(collectionName, vectorSize, token);
await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
}
private async Task UpsertPointsAsync(
@@ -1061,7 +1074,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Id,
manifest.EmbeddingSignature,
embeddingSignature);
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, vectorStore, embeddingState, token);
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token);
manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
}
app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs
+1 -1
View File
@@ -59,7 +59,7 @@ public sealed class DataSourceLocalRetrievalService(
if (maxMatches == 0)
return [];
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Name, dataSource.Id);
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
var vectorTask = this.SearchVectorAsync(dataSource, query, maxMatches, collectionName, token);
var bm25Task = this.SearchBm25Async(dataSource, query, maxMatches, token);
app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs
+3
View File
@@ -121,6 +121,9 @@ public sealed class DataSourceValidation
if (dataSourceName.Length > 40)
return TB("The name must not exceed 40 characters.");
if (dataSourceName.Any(char.IsControl))
return TB("The name must not contain control characters.");
var lowerName = dataSourceName.ToLowerInvariant();
if(lowerName != this.GetPreviousDataSourceName() && this.GetUsedDataSourceNames().Contains(lowerName))