mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-18 05:37:59 +00:00
Indexed large data sources in steps so the others get their turn (#1052)
This commit is contained in:
1 parent
fe4a64f382
commit
a3c6b29313
26 files changed
+1932
-334
No files matched your search
@@ -24,13 +24,27 @@ internal interface IIndexedSourceIndexer : IDisposable
|
||||
/// <returns>True when this indexer is the one for it.</returns>
|
||||
public bool Supports(IDataSourceBase dataSource);
|
||||
|
||||
/// <summary>
|
||||
/// How much work one run does at most, before it leaves the rest to another one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// All data sources share one queue, and only one run works at a time. A run which goes on for
|
||||
/// hours holds up every other data source for as long, a mailbox whose sync is due among them.
|
||||
/// So a run asks DocumentRunProgress.HasDoneItsShare before every document it is about to read,
|
||||
/// and pauses through DocumentRunProgress.PauseRunAsync once the answer is yes.
|
||||
///
|
||||
/// Nothing fails when a run forgets to ask. The embedding service therefore writes a warning into
|
||||
/// the log when a run took more than twice its share without asking once.
|
||||
/// </remarks>
|
||||
public RunShare Share { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Works through the documents of the data source of a run, from finding them to completing the run.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A run may stop after its share of the work and leave the rest to another one, so a data source
|
||||
/// which takes hours does not hold up all others for that long. Whether and when that run comes
|
||||
/// is the embedding service's to decide.
|
||||
/// A run stops after its share of the work, cf. Share, and leaves the rest to another one, so a
|
||||
/// data source which takes hours does not hold up all others for that long. Whether and when that
|
||||
/// run comes is the embedding service's to decide.
|
||||
/// </remarks>
|
||||
/// <param name="context">The prepared run, whose data source this indexer supports.</param>
|
||||
/// <param name="refreshMode">Why the run was started.</param>
|
||||
|
||||
Reference in new issue
Block a user