namespace AIStudio.Tools.Services.Indexing; /// /// Decides which data sources are due for a run: all of them once an interval came round, and one /// whose server was out of reach a little earlier than that. /// /// /// Everything here goes by the clock of the system. A timer counts only the time the computer is /// awake, so after a night asleep, the rest of an interval would still have to pass before the next /// round. By the clock, that round is due as soon as the computer wakes up. /// /// A server out of reach is tried again a few times with growing pauses, instead of only with the /// next round. After waking up or logging in, a VPN tunnel is often up a minute or two later than /// the network, and its mailbox would otherwise wait for a whole interval. The retries start anew /// once the server could be reached, when the tracking starts, and after the computer was asleep, /// so a server which was out of reach all evening is tried again soon the next morning. Once they /// are used up, only the rounds try again: a computer which is offline for hours does not keep /// trying every few minutes. /// /// The owner asks what is due once per check period and passes the current time to every call, /// which keeps this free of timers. It is not thread-safe; the owner locks. /// /// The time between two rounds. /// The time from the start of the tracking to the first round. /// How often the owner asks what is due. A gap of more than twice as long means the computer was asleep. internal sealed class IntervalRunSchedule(TimeSpan interval, TimeSpan firstRoundDelay, TimeSpan checkPeriod) { /// /// The pauses before the retries after a server was out of reach, in this order. /// internal static readonly TimeSpan[] RETRY_DELAYS = [TimeSpan.FromMinutes(1), TimeSpan.FromMinutes(2), TimeSpan.FromMinutes(4), TimeSpan.FromMinutes(8)]; /// /// How much earlier than planned a run counts as due. /// /// /// The timer of the owner and the clock of the system are two different clocks. A check planned /// for the very moment a run is due may come a little early by the clock, and would otherwise put /// the run off by a whole check period. /// private static readonly TimeSpan EARLINESS_TOLERANCE = TimeSpan.FromSeconds(1); private readonly HashSet dataSourceIds = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary retries = new(StringComparer.OrdinalIgnoreCase); private DateTimeOffset nextRoundUtc; private DateTimeOffset lastCheckUtc; /// /// Whether any data source is tracked. /// public bool IsTrackingAny => this.dataSourceIds.Count > 0; /// /// The ids of the tracked data sources. /// public IReadOnlyCollection DataSourceIds => this.dataSourceIds; /// /// Tracks exactly the given data sources from now on. /// /// /// Which data sources are tracked never moves the rounds, so editing them cannot keep putting the /// rounds off. Only when nothing was tracked before does the first round come after the first /// round delay. /// /// The ids of the data sources. /// The current time. public void Track(IEnumerable ids, DateTimeOffset now) { var wasTrackingAny = this.IsTrackingAny; this.dataSourceIds.Clear(); this.dataSourceIds.UnionWith(ids); if (!wasTrackingAny) { this.nextRoundUtc = now + firstRoundDelay; this.lastCheckUtc = now; this.retries.Clear(); return; } foreach (var id in this.retries.Keys.Where(id => !this.dataSourceIds.Contains(id)).ToList()) this.retries.Remove(id); } /// /// Stops tracking one data source. /// /// The id of the data source. public void Stop(string id) { this.dataSourceIds.Remove(id); this.retries.Remove(id); } /// /// Stops tracking any data source. /// public void StopAll() { this.dataSourceIds.Clear(); this.retries.Clear(); } /// /// Takes what is due now: a round of every tracked data source, or the retries whose pause is over. /// /// The current time. /// Whether a round is due, and the ids of the data sources to run. public (bool IsRound, IReadOnlyList DataSourceIds) TakeDue(DateTimeOffset now) { // // The owner asks once per check period. A much longer gap means the computer was asleep, // and right after waking up, a server may be out of reach for a little while. // if (now - this.lastCheckUtc > checkPeriod * 2) this.ChangeEveryRetry(retry => retry with { Attempts = 0 }); this.lastCheckUtc = now; // // A clock which was set back must not put the next round off by more than one interval. // if (this.nextRoundUtc - now > interval) this.nextRoundUtc = now + interval; if (IsDue(this.nextRoundUtc, now)) { this.nextRoundUtc = now + interval; // // The round runs every data source anyway, a pending retry included. // this.ChangeEveryRetry(retry => retry with { DueUtc = null }); return (true, [..this.dataSourceIds]); } var dueRetries = this.retries.Where(entry => entry.Value.DueUtc is { } dueUtc && IsDue(dueUtc, now)).Select(entry => entry.Key).ToList(); foreach (var id in dueRetries) this.retries[id] = this.retries[id] with { DueUtc = null }; return (false, dueRetries); } /// /// Notes that the server of a data source was out of reach, and plans the next retry. /// /// The id of the data source. /// The current time. /// The pause before the retry, or null when the retries are used up or the data source is not tracked. public TimeSpan? RecordServerOutOfReach(string id, DateTimeOffset now) { if (!this.dataSourceIds.Contains(id)) return null; // // A data source without an entry has no retries behind it, which is just what the default // value says. // var attempts = this.retries.GetValueOrDefault(id).Attempts; if (attempts >= RETRY_DELAYS.Length) return null; var delay = RETRY_DELAYS[attempts]; this.retries[id] = new RetryState(attempts + 1, now + delay); return delay; } /// /// Notes that the server of a data source could be reached, so its next outage starts the retries anew. /// /// The id of the data source. public void RecordServerReached(string id) => this.retries.Remove(id); private static bool IsDue(DateTimeOffset dueUtc, DateTimeOffset now) => now >= dueUtc - EARLINESS_TOLERANCE; /// /// Changes the retries of every data source. /// /// /// Goes over a copy of the entries, since writing a changed value back would otherwise break the /// enumeration. There are as many entries as data sources whose server was out of reach. /// /// How a retry changes. private void ChangeEveryRetry(Func change) { foreach (var (id, retry) in this.retries.ToList()) this.retries[id] = change(retry); } /// /// The retries of one data source whose server was out of reach. The default value stands for none. /// /// How many retries were planned since the series started. /// When the next retry is due, or null when none is planned. private readonly record struct RetryState(int Attempts, DateTimeOffset? DueUtc); }