MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-09-27 00:53:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Removed the confidence level from the RAG index database

Browse Source
This commit is contained in:
Thorsten Sommer 2026-09-15 22:28:08 +02:00
parent 2172a39206
commit ff3927dcde
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
10 changed files with 57 additions and 53 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs
View File

@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile(
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);
int ChunkCount);

4
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs
View File

@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];

3
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs
View File

@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired();
entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank");
entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type");
entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence");
entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique();
entity

1
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs
View File

@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator
{
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{
await context.Database.MigrateAsync(token);

4
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs
View File

@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult(
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);
int ChunkCount);

4
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs
View File

@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity
public DateTimeOffset EmbeddedAtUtc { get; set; }
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
}

51
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs Normal file
View File

@ -0,0 +1,51 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
/// <summary>
/// Drops the copy of the data source confidence level which every indexed file carried.
/// </summary>
/// <remarks>
/// The confidence level is what a data source asks of a provider. It is a property of the data
/// source, it is enforced live before anything is indexed or answered, and it changes no vector.
/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed.
/// </remarks>
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20260915000000_DropFileConfidenceLevel")]
public partial class DropFileConfidenceLevel : Migration
{
/// <remarks>
/// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers
/// DropColumn by rebuilding the table, and a rebuild drops the table the trigger
/// embedded_files_file_name_au hangs on, which would silently stop the full-text index from
/// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone.
/// It does refuse a column an index names, so the index has to go first.
/// </remarks>
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "idx_embedded_files_confidence",
table: "embedded_files");
migrationBuilder.Sql("""
ALTER TABLE embedded_files DROP COLUMN confidence_level;
ALTER TABLE embedded_files DROP COLUMN confidence_level_rank;
""");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("""
ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT '';
ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0;
""");
migrationBuilder.CreateIndex(
name: "idx_embedded_files_confidence",
table: "embedded_files",
column: "confidence_level_rank");
}
}

19
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs
View File

@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("INTEGER")
.HasColumnName("chunk_count");
entity.Property<string>("ConfidenceLevel")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("confidence_level");
entity.Property<int>("ConfidenceLevelRank")
.HasColumnType("INTEGER")
.HasColumnName("confidence_level_rank");
entity.Property<DateTimeOffset>("CreationUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.HasIndex("AbsolutePath")
.HasDatabaseName("idx_embedded_files_absolute_path");
entity.HasIndex("ConfidenceLevelRank")
.HasDatabaseName("idx_embedded_files_confidence");
entity.HasIndex("DataSourceId")
.HasDatabaseName("idx_embedded_files_data_source");
@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.IsRequired()
.HasColumnType("TEXT");
entity.Property<string>("ConfidenceLevel")
.IsRequired()
.HasColumnType("TEXT");
entity.Property<int>("ConfidenceLevelRank")
.HasColumnType("INTEGER");
entity.Property<DateTimeOffset>("CreationUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT");

10
app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs
View File

@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
f.creation_utc AS CreationUtc,
f.last_write_utc AS LastWriteUtc,
c.embedded_at_utc AS EmbeddedAtUtc,
f.chunk_count AS ChunkCount,
f.confidence_level AS ConfidenceLevel,
f.confidence_level_rank AS ConfidenceLevelRank
f.chunk_count AS ChunkCount
FROM embedding_chunks_fts
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
fileEntity.LastWriteUtc = file.LastWriteUtc;
fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
fileEntity.ChunkCount = file.ChunkCount;
fileEntity.ConfidenceLevel = file.ConfidenceLevel;
fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank;
}
private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
result.CreationUtc,
result.LastWriteUtc,
result.EmbeddedAtUtc,
result.ChunkCount,
result.ConfidenceLevel,
result.ConfidenceLevelRank);
result.ChunkCount);
private static string BuildFtsQuery(string query)
{

10
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -1116,7 +1116,6 @@ public sealed partial class DataSourceEmbeddingService
{
file.Refresh();
var absolutePath = Path.GetFullPath(file.FullName);
var confidenceLevel = GetDataSourceConfidenceLevel(dataSource);
return new(
this.CreateParentFileId(dataSource.Id, absolutePath),
absolutePath,
@ -1128,9 +1127,7 @@ public sealed partial class DataSourceEmbeddingService
file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch,
file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch,
embeddedAtUtc,
chunkCount,
confidenceLevel.ToString(),
(int)confidenceLevel);
chunkCount);
}
private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTimeOffset embeddedAtUtc)
@ -1146,11 +1143,6 @@ public sealed partial class DataSourceEmbeddingService
.ToList();
}
private static ConfidenceLevel GetDataSourceConfidenceLevel(IDataSource dataSource) =>
dataSource is not IInternalDataSource internalDataSource || internalDataSource.ConfidenceLevel is ConfidenceLevel.NONE
? ConfidenceLevel.UNKNOWN
: internalDataSource.ConfidenceLevel;
private static string GetFileType(FileInfo file)
{
var extension = file.Extension.TrimStart('.').ToLowerInvariant();