blob: a6d90482662be8dd81f324be9502d40bec10616c [file]
/**
* Unified Automation — Codex-style automation system.
*
* Two kinds:
* - "heartbeat": session-scoped polling (resume into same session)
* - "cron": standalone scheduled runs (create fresh session each time)
*/
import type {
AutomationDefinition,
AutomationExecutionTemplate,
AutomationKind,
AutomationSchedule,
} from '@maka/core/automation';
import { AUTOMATION_LAST_ERROR_LIMIT, truncateAutomationText } from '@maka/core/automation';
import { compileCronExpression } from '@maka/core/cron-expression';
export { matchesCronField } from '@maka/core/cron-expression';
export type {
AutomationDefinition,
AutomationExecutionTemplate,
AutomationKind,
AutomationSchedule,
AutomationStatus,
} from '@maka/core/automation';
export interface AutomationManagerDeps {
generateId: () => string;
now: () => number;
/** Randomness source for schedule jitter. Injectable for deterministic tests. */
random?: () => number;
}
const MAX_AUTOMATIONS_PER_SESSION = 20;
const MAX_CONSECUTIVE_FAILURES = 5;
const DEFAULT_EXPIRY_DAYS = 7;
const DEFAULT_AUTOMATION_FAILURE_MESSAGE = 'Automation run failed';
/** Maximum jitter cap for recurring re-schedules: 15 minutes. */
const MAX_JITTER_MS = 15 * 60 * 1000;
/** Maximum early jitter for one-shot fires landing on round minutes: 90 seconds. */
const ONE_SHOT_JITTER_MS = 90 * 1000;
/**
* Thundering-herd jitter, ported verbatim from the old wakeup-scheduler.
*
* - Recurring (interval/cron): up to 10% of the delay, capped at 15 minutes.
* - One-shot firing on :00 or :30: up to 90s early jitter (returned as
* negative). Otherwise 0 for one-shot. The round-mark property belongs to
* the ACTUAL fire timestamp, not the delay (a 30-minute delay from 10:07
* fires at 10:37 — no round mark).
*/
export function computeJitter(
delayMs: number,
recurring: boolean,
random: () => number = Math.random,
firesAtMs?: number,
): number {
if (recurring) {
const maxJitter = Math.min(delayMs * 0.1, MAX_JITTER_MS);
return Math.floor(random() * maxJitter);
}
if (firesAtMs !== undefined && new Date(firesAtMs).getMinutes() % 30 === 0) {
return -Math.floor(random() * ONE_SHOT_JITTER_MS);
}
return 0;
}
export class AutomationManager {
private automations = new Map<string, AutomationDefinition>();
constructor(private readonly deps: AutomationManagerDeps) {}
create(input: {
kind: AutomationKind;
name: string;
prompt: string;
sessionId: string;
schedule: AutomationSchedule;
maxFires?: number;
expiresAt?: number;
durable?: boolean;
execution?: AutomationExecutionTemplate;
}): AutomationDefinition | { error: string } {
// Only count active/paused automations toward the limit (not completed/expired).
const activeCount = this.listForSession(input.sessionId).filter(
(a) => a.status === 'active' || a.status === 'paused',
).length;
if (activeCount >= MAX_AUTOMATIONS_PER_SESSION) {
return {
error: `Maximum ${MAX_AUTOMATIONS_PER_SESSION} active automations per session reached.`,
};
}
if (input.kind === 'heartbeat') {
const existing = this.listForSession(input.sessionId).filter(
(a) => a.kind === 'heartbeat' && a.status === 'active',
);
if (existing.length >= 5) {
return { error: 'Maximum 5 active heartbeat automations per session.' };
}
}
const now = this.deps.now();
const id = this.deps.generateId();
const nextFireAt = this.computeNextFire(input.schedule, now);
if (nextFireAt === null && input.schedule.type === 'cron') {
return {
error: `Invalid cron expression: "${input.schedule.expression}". Could not compute next fire time.`,
};
}
const defaultExpiry = now + DEFAULT_EXPIRY_DAYS * 24 * 60 * 60 * 1000;
// Cron is a standalone scheduled task (fresh session each run) — it is
// meaningless if it dies on restart, so it defaults to durable. Heartbeat
// injects into its own session and has no coherent post-restart target, so
// it is ALWAYS session-bound (never durable) — a durable heartbeat would be
// a zombie after restart. `durable` is therefore a cron-only concept; an
// explicit value only refines cron.
const durable = input.kind === 'cron' ? (input.durable ?? true) : false;
const automation: AutomationDefinition = {
id,
kind: input.kind,
name: input.name,
status: 'active',
prompt: input.prompt,
sessionId: input.sessionId,
schedule: input.schedule,
createdAt: now,
updatedAt: now,
nextFireAt,
lastFireAt: null,
lastRunId: null,
fireCount: 0,
maxFires: input.maxFires ?? null,
expiresAt: input.expiresAt ?? defaultExpiry,
lastError: null,
consecutiveFailures: 0,
...(durable ? { durable: true } : {}),
...(input.kind === 'cron' && input.execution ? { execution: input.execution } : {}),
};
this.automations.set(id, automation);
this.pruneTerminal(input.sessionId);
return automation;
}
get(id: string): AutomationDefinition | undefined {
return this.automations.get(id);
}
delete(id: string, sessionId?: string): boolean {
const automation = this.automations.get(id);
if (!automation) return false;
if (sessionId && !this.manageableBy(automation, sessionId)) return false;
this.automations.delete(id);
return true;
}
pause(id: string, sessionId: string): AutomationDefinition | undefined {
const automation = this.automations.get(id);
if (!automation || !this.manageableBy(automation, sessionId)) return undefined;
if (automation.status !== 'active') return undefined;
automation.status = 'paused';
automation.updatedAt = this.deps.now();
return automation;
}
resume(id: string, sessionId: string): AutomationDefinition | undefined {
const automation = this.automations.get(id);
if (!automation || !this.manageableBy(automation, sessionId)) return undefined;
if (automation.status !== 'paused') return undefined;
// Refuse to resume an automation whose fire budget is already spent. A
// maxFires-exhausted (or a one-shot that already fired) automation only
// reaches 'paused' via the attemptFailed path, which leaves nextFireAt=null.
// Re-arming it here would grant a fire beyond the declared hard cap — the
// next tick would bump fireCount past maxFires (or re-fire a 'once'),
// spawning a real extra run. maxFires is a cap on ATTEMPTS, so a spent
// budget cannot be revived by resume.
if (automation.maxFires && automation.fireCount >= automation.maxFires) return undefined;
if (automation.schedule.type === 'once' && automation.fireCount > 0) return undefined;
automation.status = 'active';
automation.updatedAt = this.deps.now();
// Resume starts a clean streak — a fire that paused this automation must not
// count toward re-pausing it after a single fresh failure.
automation.consecutiveFailures = 0;
automation.lastError = null;
automation.nextFireAt = this.computeNextFire(automation.schedule, this.deps.now());
return automation;
}
listForSession(sessionId: string): AutomationDefinition[] {
return [...this.automations.values()].filter((a) => a.sessionId === sessionId);
}
/**
* Automations a session can see and manage: its own (any kind) plus every
* durable one. Durable automations (cron by default) are app-global — they
* outlive their creator session and reload from disk on restart with their
* original sessionId, so a fresh session must still be able to list and
* manage them. Non-durable heartbeats stay private to their session.
*/
listVisibleForSession(sessionId: string): AutomationDefinition[] {
return [...this.automations.values()].filter(
(a) => a.sessionId === sessionId || a.durable === true,
);
}
/** A session may manage its own automations plus any durable (app-global) one. */
private manageableBy(automation: AutomationDefinition, sessionId: string): boolean {
return automation.sessionId === sessionId || automation.durable === true;
}
listActive(): AutomationDefinition[] {
return [...this.automations.values()].filter((a) => a.status === 'active');
}
/**
* Mark an expired automation terminal. Returns true if it was expired.
* Used by the scheduler's eager expiry sweep.
*/
sweepExpired(id: string): boolean {
const automation = this.automations.get(id);
if (!automation || automation.status !== 'active') return false;
const now = this.deps.now();
if (automation.expiresAt && now >= automation.expiresAt) {
automation.status = 'expired';
automation.nextFireAt = null;
automation.updatedAt = now;
return true;
}
return false;
}
/**
* Begin a fire attempt: advance the schedule and counters, but do NOT commit
* terminal completion — that happens only on a real success (attemptSucceeded).
* Checks expiry first. Returns the automation if it should fire, else undefined.
*/
attemptStarted(id: string): AutomationDefinition | undefined {
const automation = this.automations.get(id);
if (!automation || automation.status !== 'active') return undefined;
const now = this.deps.now();
// Check expiry BEFORE firing — don't execute expired automations.
if (automation.expiresAt && now >= automation.expiresAt) {
automation.status = 'expired';
automation.nextFireAt = null;
automation.updatedAt = now;
return undefined;
}
automation.lastFireAt = now;
automation.fireCount++;
automation.updatedAt = now;
// A one-shot does not auto-retry: null its nextFireAt now. A recurring job
// advances to its next slot. Completion (once / maxFires) is committed only
// after a successful outcome in attemptSucceeded.
automation.nextFireAt =
automation.schedule.type === 'once' ? null : this.computeNextFire(automation.schedule, now);
// maxFires is a hard cap on the number of fire ATTEMPTS: once this attempt
// reaches the cap, no further fire is scheduled — regardless of whether this
// one ultimately succeeds or fails. (Terminal status is still committed by
// attemptSucceeded/attemptFailed based on this attempt's outcome.) Without
// this, a failing recurring automation would keep firing past maxFires until
// the consecutive-failure cap, and fireCount could exceed maxFires.
if (automation.maxFires && automation.fireCount >= automation.maxFires) {
automation.nextFireAt = null;
}
return automation;
}
/**
* Record a fire attempt deferred by the idle-gate (target busy). Pure
* observability — surfaced in the model-facing list output.
*/
recordDeferredFire(id: string): void {
const automation = this.automations.get(id);
if (!automation || automation.status !== 'active') return;
automation.deferredFireCount = (automation.deferredFireCount ?? 0) + 1;
}
/**
* Skip a fire without executing — advance to next schedule time.
* Used only when the scheduler's defer/retry window (~45min, mirroring the
* old wakeup-scheduler's exponential-backoff budget) is exhausted — never on
* a transient busy blip.
*/
skipFire(id: string): void {
const automation = this.automations.get(id);
if (!automation || automation.status !== 'active') return;
const now = this.deps.now();
automation.updatedAt = now;
// A one-shot has no "next slot": re-arming it via computeNextFire re-adds the
// full delay, so repeated skips (e.g. a long incognito window or a busy
// session) would drift it forward indefinitely and then silently drop it at
// expiry. Its fire window has passed — settle it terminally instead.
if (automation.schedule.type === 'once') {
automation.nextFireAt = null;
automation.status = 'expired';
automation.lastError = 'Fire window skipped (session busy or privacy mode)';
return;
}
automation.nextFireAt = this.computeNextFire(automation.schedule, now);
}
/**
* Commit a successful fire outcome: reset failure state, record the run id,
* and NOW apply completion (once / maxFires reached).
*/
attemptSucceeded(id: string, runId?: string): void {
const automation = this.automations.get(id);
if (!automation) return;
settleAutomationAttempt(automation, { status: 'completed', runId }, this.deps.now());
}
/**
* Record a failed fire outcome. Accumulates toward the consecutive-failure
* cap (→ paused). A one-shot that fails has no next fire, so it is paused so
* it is visible rather than a silent idle zombie.
*/
attemptFailed(id: string, error: string): void {
const automation = this.automations.get(id);
if (!automation) return;
settleAutomationAttempt(automation, { status: 'failed', error }, this.deps.now());
}
removeAllForSession(sessionId: string): number {
let count = 0;
for (const [id, auto] of this.automations) {
if (auto.sessionId === sessionId && auto.kind === 'heartbeat') {
this.automations.delete(id);
count++;
}
}
return count;
}
dispose(): void {
this.automations.clear();
}
/** Bulk-register pre-existing automations (e.g. loaded from durable store on startup). */
registerAll(automations: AutomationDefinition[]): void {
const now = this.deps.now();
for (const automation of automations) {
// Reconcile an interrupted fire: a fire that started (fireCount bumped,
// nextFireAt nulled) but whose run never settled — the app quit mid-run —
// persists as active with nextFireAt=null. Left alone it is a silent
// zombie (never fires again until the 7-day expiry sweep).
if (automation.status === 'active' && automation.nextFireAt === null) {
const budgetSpent =
(automation.maxFires != null && automation.fireCount >= automation.maxFires) ||
(automation.schedule.type === 'once' && automation.fireCount > 0);
if (budgetSpent) {
// The one/last fire was already attempted (fireCount reflects it), so
// settle it terminally rather than re-run it (at-most-once semantics).
// Its outcome was never committed, so record the uncertainty instead of
// asserting a clean success — a genuine success leaves lastError null.
automation.status = 'completed';
automation.lastError =
'Interrupted on restart before the fire outcome was recorded; not re-run.';
} else {
// A recurring automation should always carry a future fire time; a null
// here is a corrupt/interrupted state — re-arm it.
automation.nextFireAt = this.computeNextFire(automation.schedule, now);
}
}
this.automations.set(automation.id, automation);
}
}
/** Replace the projection without applying legacy restart reconciliation. */
hydrate(automations: readonly AutomationDefinition[]): void {
this.automations.clear();
for (const automation of automations) {
this.automations.set(automation.id, structuredClone(automation));
}
}
/** Return all automations (all statuses, all sessions). */
listAll(): AutomationDefinition[] {
return [...this.automations.values()];
}
/** Remove completed/expired automations beyond a grace buffer (matches the
* old wakeup-scheduler's MAX_RECORDS_PER_SESSION=50 observable history). */
private pruneTerminal(sessionId: string): void {
const terminal = this.listForSession(sessionId).filter(
(a) => a.status === 'completed' || a.status === 'expired',
);
const MAX_TERMINAL_KEPT = 50;
if (terminal.length <= MAX_TERMINAL_KEPT) return;
terminal.sort((a, b) => a.updatedAt - b.updatedAt);
for (let i = 0; i < terminal.length - MAX_TERMINAL_KEPT; i++) {
this.automations.delete(terminal[i].id);
}
}
/**
* Next fire time with thundering-herd jitter (see computeJitter):
* - once: base delay; if the fire lands on a :00/:30 wall-clock minute, pull
* it up to 90s EARLY (never before fromTime).
* - interval/cron (recurring): push up to 10% of the delay late, capped at
* 15min. Jitter is strictly non-negative for recurring schedules so a cron
* can never fire BEFORE its mark (an early cron fire would recompute the
* same mark next time and double-fire).
*/
private computeNextFire(schedule: AutomationSchedule, fromTime: number): number | null {
const random = this.deps.random ?? Math.random;
switch (schedule.type) {
case 'once': {
const delayMs = schedule.delaySeconds * 1000;
const base = fromTime + delayMs;
return Math.max(fromTime, base + computeJitter(delayMs, false, random, base));
}
case 'interval': {
const delayMs = schedule.seconds * 1000;
return fromTime + delayMs + computeJitter(delayMs, true, random);
}
case 'cron': {
const base = computeNextCronFire(schedule.expression, fromTime);
if (base === null) return null;
return base + computeJitter(base - fromTime, true, random);
}
}
}
}
export type AutomationAttemptOutcome =
| { readonly status: 'completed'; readonly runId?: string }
| {
readonly status: 'failed' | 'cancelled';
readonly error: string;
readonly runId?: string;
};
/** Settle an attempt that was accepted while the definition was active. */
export function settleAutomationAttempt(
automation: AutomationDefinition,
outcome: AutomationAttemptOutcome,
now: number,
): void {
if (automation.status === 'completed' || automation.status === 'expired') return;
if (outcome.runId) automation.lastRunId = outcome.runId;
automation.updatedAt = now;
if (outcome.status === 'completed') {
automation.consecutiveFailures = 0;
automation.lastError = null;
if (
automation.status === 'active' &&
(automation.schedule.type === 'once' ||
(automation.maxFires !== null && automation.fireCount >= automation.maxFires))
) {
automation.status = 'completed';
automation.nextFireAt = null;
}
return;
}
automation.consecutiveFailures += 1;
const diagnostic = truncateAutomationText(outcome.error, AUTOMATION_LAST_ERROR_LIMIT);
automation.lastError = diagnostic.length > 0 ? diagnostic : DEFAULT_AUTOMATION_FAILURE_MESSAGE;
if (
automation.status === 'active' &&
(automation.consecutiveFailures >= MAX_CONSECUTIVE_FAILURES || automation.nextFireAt === null)
) {
automation.status = 'paused';
}
}
/**
* Compute the next Unix-ms timestamp at which a 5-field cron expression fires,
* strictly after `fromTime`. Returns null for a malformed expression or one
* that cannot occur within the bounded search window.
*
* TIMEZONE CONTRACT: evaluation happens in the HOST's local timezone. Candidate
* instants are decomposed with `Date` local getters (`getMinutes`, `getHours`,
* `getDate`, `getMonth`, `getDay`), so `0 9 * * *` means "09:00 local wall-clock
* time" on the machine running this process. Across DST transitions the wall
* clock is respected (a skipped/repeated local hour shifts the fire instant
* accordingly). There is no per-automation IANA timezone; if the process moves
* timezones, schedules re-anchor to the new local time. Threading an explicit
* IANA zone would ripple through the schedule type and every caller, so it is
* intentionally out of scope for this parser.
*/
export function computeNextCronFire(expression: string, fromTime: number): number | null {
const compiled = compileCronExpression(expression, { profile: 'automation-v1' });
return compiled.ok ? compiled.value.nextAfter(fromTime) : null;
}
export { MAX_AUTOMATIONS_PER_SESSION, MAX_CONSECUTIVE_FAILURES, DEFAULT_EXPIRY_DAYS };