using AIStudio.Settings.DataModel; using AIStudio.Tools.PluginSystem; namespace AIStudio.Tools.Services; /// /// Counts documents, whatever a document is for the kind of data source: a file for a folder, a /// mail for a mailbox. Which word the user reads is the business of the page showing it. /// /// CurrentDocumentBlock and CurrentDocumentPage are null rather than zero while nothing is known /// about them: a document which is only about to start has no first block, and not every kind of /// document has pages to count. Block numbers start at one, the way the page states them. /// /// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that /// difference to offer the repair for this one case: the vector store cannot be read, or it is /// missing although the index database knows what it held. It is carried as its own flag so nothing /// has to read it back out of the message in LastError. Attention does the same for what the user /// has to decide, and PendingRemovalCount is the number a held back removal asks about: agreeing to /// it hands that very number back, cf. ApprovePendingMailRemovalAsync. /// /// LastSyncUtc is when the data source was last worked through as a whole, for those which are /// synced rather than watched, such as mailboxes. It stays null for the others. /// public sealed record DataSourceEmbeddingStatus( string DataSourceId, string DataSourceName, DataSourceType DataSourceType, DataSourceEmbeddingState State, int TotalDocuments, int IndexedDocuments, int FailedDocuments, string CurrentDocument, string LastError, IReadOnlyList Failures, int PermanentlySkippedDocuments = 0, int? CurrentDocumentBlock = null, int? CurrentDocumentPage = null, bool VectorStoreUnreadable = false, DataSourceAttention Attention = DataSourceAttention.NONE, int? PendingRemovalCount = null, DateTimeOffset? LastSyncUtc = null) { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus)); /// /// Files which were found in the scanned part of a local data source but intentionally not /// indexed. /// public IReadOnlyList IgnoredFiles { get; init; } = []; /// /// Files outside the configured file patterns or named by the exclusion list, summarized by file /// type. They are intentional exclusions rather than skipped documents and therefore do not /// affect progress or status. /// public IReadOnlyList FilePatternExclusions { get; init; } = []; /// /// Folders which the exclusion list left out, summarized by name. Like the files outside the /// patterns, they are intentional and do not affect progress or status. /// public IReadOnlyList ExcludedFolders { get; init; } = []; /// /// Every selected document the run skipped, whether it may become readable after changing or /// was excluded before extraction. Files outside the configured patterns are not selected and /// therefore do not count here. /// public int SkippedDocuments => this.PermanentlySkippedDocuments + this.IgnoredFiles.Count; /// /// Whether selected audio or video files wait for a transcription provider which this data /// source allows, so they cannot be indexed until the settings change. /// public bool NeedsTranscriptionProvider => this.IgnoredFiles.Any(file => file.Reason is DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING or DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED); /// /// The settled health of this data source. Activity is represented by State /// instead, so queued and running sources can keep their activity indicator. /// /// /// A few permanently unreadable files do not make an otherwise usable data source unhealthy. /// They become a warning only when none of the selected documents could be indexed. Files /// outside the configured patterns never take part in this decision. /// public DataSourceEmbeddingHealth Health { get { if (this.State is DataSourceEmbeddingState.FAILED || this.FailedDocuments > 0) return DataSourceEmbeddingHealth.ERROR; if (this.NeedsTranscriptionProvider || this.State is DataSourceEmbeddingState.COMPLETED && this.IndexedDocuments == 0 && this.SkippedDocuments > 0) return DataSourceEmbeddingHealth.WARNING; return DataSourceEmbeddingHealth.HEALTHY; } } /// /// Skipped documents are done for this run, even though nothing was indexed of them. Leaving /// them out would keep the bar short of the end for a data source which has nothing left to do. /// public int ProgressPercent => this.TotalDocuments <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedDocuments + this.SkippedDocuments) * 100d / this.TotalDocuments), 0, 100); public string StateLabel => this.State switch { DataSourceEmbeddingState.QUEUED => TB("Queued"), DataSourceEmbeddingState.RUNNING => TB("Running"), DataSourceEmbeddingState.COMPLETED => TB("Completed"), DataSourceEmbeddingState.FAILED => TB("Needs attention"), _ => TB("Idle") }; public int SortOrder => this.State switch { DataSourceEmbeddingState.RUNNING => 0, DataSourceEmbeddingState.QUEUED => 1, DataSourceEmbeddingState.FAILED => 2, DataSourceEmbeddingState.COMPLETED => 3, _ => 4, }; }