added metadata for bm25 and also confidence levels

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -2,8 +2,10 @@ using System.Security.Cryptography;
 using System.Text;
 using System.Text.RegularExpressions;
 
+using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Databases.EmbeddingState;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Rust;
 
@@ -27,6 +29,8 @@ public sealed partial class DataSourceEmbeddingService
 
     private sealed record ExtractedFileContent(string Text, IReadOnlyList<string> SourceSegments);
 
+    private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
+
     private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
 
     private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
@@ -748,6 +752,7 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Hostname,
             embeddingProvider.TokenizerPath,
             embeddingProvider.EffectiveTokenLimit,
+            GetDataSourceComplianceLevel(dataSource).ToString(),
             dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
             dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0,
             chunkingOptions.MaxChunkTokenLength,
@@ -847,6 +852,59 @@ public sealed partial class DataSourceEmbeddingService
         return Convert.ToHexString(bytes);
     }
 
+    private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTime embeddedAtUtc)
+    {
+        file.Refresh();
+        var absolutePath = Path.GetFullPath(file.FullName);
+        var complianceLevel = GetDataSourceComplianceLevel(dataSource);
+        return new(
+            this.CreateParentFileId(dataSource.Id, absolutePath),
+            absolutePath,
+            file.Name,
+            this.TryGetRelativePath(dataSource, file),
+            GetFileType(file),
+            fingerprint,
+            file.Exists ? file.Length : 0,
+            file.Exists ? file.CreationTimeUtc : DateTime.UnixEpoch,
+            file.Exists ? file.LastWriteTimeUtc : DateTime.UnixEpoch,
+            embeddedAtUtc,
+            chunkCount,
+            complianceLevel.ToString(),
+            (int)complianceLevel);
+    }
+
+    private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTime embeddedAtUtc)
+    {
+        return batch
+            .Select(chunk => new EmbeddingStateChunk(
+                chunk.ChunkId,
+                parentFile.ParentFileId,
+                chunk.PageNumber,
+                chunk.ChunkIndex,
+                chunk.Text,
+                embeddedAtUtc))
+            .ToList();
+    }
+
+    private static ConfidenceLevel GetDataSourceComplianceLevel(IDataSource dataSource) =>
+        dataSource.ComplianceLevel is ConfidenceLevel.NONE
+            ? ConfidenceLevel.UNKNOWN
+            : dataSource.ComplianceLevel;
+
+    private static string GetFileType(FileInfo file)
+    {
+        var extension = file.Extension.TrimStart('.').ToLowerInvariant();
+        return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
+    }
+
+    private static int? TryExtractPageNumber(string chunk)
+    {
+        var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
+        return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0
+            ? pageNumber
+            : null;
+    }
+
     private string GetCollectionName(string dataSourceName, string dataSourceId)
     {
         var safeId = dataSourceId
@@ -864,9 +922,14 @@ public sealed partial class DataSourceEmbeddingService
         return $"rag_{safeName}_{safeId}";
     }
 
-    private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex)
+    private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
+        CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
+
+    private string CreateParentFileId(string dataSourceId, string absolutePath) =>
+        CreateStableGuid($"{dataSourceId}:parent-file:{absolutePath}");
+
+    private static string CreateStableGuid(string source)
     {
-        var source = $"{dataSourceId}:{fingerprint}:{chunkIndex}";
         var hash = SHA256.HashData(Encoding.UTF8.GetBytes(source));
         var guidBytes = hash[..16].ToArray();
 

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -17,11 +17,11 @@ public sealed partial class DataSourceEmbeddingService
         embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
         if (!embeddingState.IsAvailable)
         {
-            logger.LogWarning("Could not delete SQLite embedding state for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
+            logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
             return;
         }
 
         await embeddingState.DeleteDataSourceAsync(dataSourceId, token);
-        logger.LogInformation("Reset persisted embedding state for data source '{DataSourceId}'.", dataSourceId);
+        logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId);
     }
 }

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -162,7 +162,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
         if (!embeddingState.IsAvailable)
         {
-            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
+            logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
             return true;
         }
 
@@ -395,7 +395,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                 dataSource.Id,
                 embeddingState.Name);
             token.ThrowIfCancellationRequested();
-            this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available."));
+            this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available."));
             return;
         }
 
@@ -522,15 +522,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     chunkCount);
                 await embeddingState.UpsertFileAsync(
                     dataSource.Id,
-                    new EmbeddingStateFile(
-                        file.FullName,
-                        file.Name,
-                        this.TryGetRelativePath(dataSource, file),
-                        fingerprint,
-                        file.Length,
-                        file.LastWriteTimeUtc,
-                        embeddedAtUtc,
-                        chunkCount),
+                    this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc),
                     token);
                 manifest.Files[file.FullName] = record;
                 completedFiles++;
@@ -603,22 +595,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             file.FullName,
             collectionName);
         await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
+        await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
+
+        var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTime.UtcNow);
+        await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token);
 
         var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize;
-        var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize);
+        var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
         var totalChunkCount = 0;
 
         await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
         {
-            batch.Add((chunk, totalChunkCount));
+            batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk)));
             totalChunkCount++;
 
             if (batch.Count >= embeddingBatchSize)
-                await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
+                await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token);
         }
 
         if (batch.Count > 0)
-            await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
+            await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token);
 
         if (totalChunkCount == 0)
             throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks.");
@@ -639,11 +635,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IDataSource dataSource,
         FileInfo file,
         string fingerprint,
+        EmbeddingStateFile parentFile,
         EmbeddingProvider embeddingProvider,
         IProvider provider,
         DataSourceEmbeddingManifest manifest,
         string collectionName,
-        List<(string Text, int ChunkIndex)> batch,
+        List<EmbeddingChunkDraft> batch,
         CancellationToken token)
     {
         logger.LogDebug(
@@ -694,15 +691,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         }
 
         token.ThrowIfCancellationRequested();
+        var embeddedAtUtc = DateTime.UtcNow;
         await this.UpsertPointsAsync(
             vectorStore,
             collectionName,
             dataSource,
             file,
             fingerprint,
+            parentFile,
             batch,
             vectors,
-            this.TryGetRelativePath(dataSource, file),
+            embeddedAtUtc,
+            token);
+        token.ThrowIfCancellationRequested();
+        await embeddingState.UpsertChunksAsync(
+            dataSource.Id,
+            this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc),
             token);
 
         logger.LogDebug(
@@ -725,26 +729,34 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IDataSource dataSource,
         FileInfo file,
         string fingerprint,
-        IReadOnlyList<(string Text, int ChunkIndex)> batch,
+        EmbeddingStateFile parentFile,
+        IReadOnlyList<EmbeddingChunkDraft> batch,
         IReadOnlyList<IReadOnlyList<float>> vectors,
-        string relativePath,
+        DateTime embeddedAtUtc,
         CancellationToken token)
     {
-        var embeddedAtUtc = DateTime.UtcNow;
         var points = batch.Select((item, index) => new VectorStoragePoint(
-            this.CreatePointId(dataSource.Id, fingerprint, item.ChunkIndex),
+            item.ChunkId,
             vectors[index],
             dataSource.Id,
             dataSource.Name,
             dataSource.Type.ToString(),
+            item.ChunkId,
+            parentFile.ParentFileId,
             file.FullName,
-            file.Name,
-            relativePath,
+            parentFile.AbsolutePath,
+            parentFile.FileName,
+            parentFile.RelativePath,
+            parentFile.FileType,
+            item.PageNumber,
             item.ChunkIndex,
             item.Text,
             fingerprint,
-            file.LastWriteTimeUtc,
-            embeddedAtUtc)).ToList();
+            parentFile.CreationUtc,
+            parentFile.LastWriteUtc,
+            embeddedAtUtc,
+            parentFile.ComplianceLevel,
+            parentFile.ComplianceLevelRank)).ToList();
 
         await vectorStore.InsertEmbedding(collectionName, points, token);
     }
@@ -805,7 +817,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             .ToList();
 
         logger.LogInformation(
-            "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.",
+            "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.",
             supportedDataSources.Count);
 
         foreach (var dataSource in supportedDataSources)
@@ -875,7 +887,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
 
         logger.LogInformation(
-            "Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
+            "Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
             dataSource.Name,
             dataSource.Id,
             manifest.Files.Count,