fix: B1 — lockJob 原子递增 attemptCount + Engine 重试解锁逻辑
- lockJob updateMany 新增 attemptCount: { increment: 1 }
- Engine retryable 错误: unlockForRetry() 重置为 queued 后再抛给 BullMQ
- Engine permanent 错误: 直接 markFailed + return(不抛错,BullMQ 不重试)
- unlockForRetry: 重置 lifecycleStatus→queued + 清除 lockedBy/lockedAt/lockUntil
- 更新测试: lockJob 验证 attemptCount 递增; schema validation 验证不抛错
Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
d82ae8ba71
commit
cdcf056ba5
@ -166,14 +166,15 @@ describe('AiJobExecutionEngineImpl', () => {
|
|||||||
expect(lifecycleRepo.markFailed).not.toHaveBeenCalled();
|
expect(lifecycleRepo.markFailed).not.toHaveBeenCalled();
|
||||||
});
|
});
|
||||||
|
|
||||||
it('schema validation → permanent → markFailed', async () => {
|
it('schema validation → permanent → markFailed(不抛错,BullMQ 不再重试)', async () => {
|
||||||
prisma.aiJob.findUnique.mockResolvedValue(makeJob());
|
prisma.aiJob.findUnique.mockResolvedValue(makeJob());
|
||||||
lifecycleRepo.lockJob.mockResolvedValue(makeJob({ lifecycleStatus: 'running' }));
|
lifecycleRepo.lockJob.mockResolvedValue(makeJob({ lifecycleStatus: 'running', attemptCount: 1 }));
|
||||||
prisma.aiJobSnapshot.findUnique.mockResolvedValue({ schemaVersion: '1.0', content: {} });
|
prisma.aiJobSnapshot.findUnique.mockResolvedValue({ schemaVersion: '1.0', content: {} });
|
||||||
|
|
||||||
aiGateway.generate.mockRejectedValue(new Error('schema validation failed'));
|
aiGateway.generate.mockRejectedValue(new Error('schema validation failed'));
|
||||||
|
|
||||||
await expect(engine.execute('job-001', ctx)).rejects.toThrow();
|
// 永久错误:不应抛出(避免 BullMQ retry)
|
||||||
|
await engine.execute('job-001', ctx);
|
||||||
expect(lifecycleRepo.markFailed).toHaveBeenCalledWith(
|
expect(lifecycleRepo.markFailed).toHaveBeenCalledWith(
|
||||||
'job-001',
|
'job-001',
|
||||||
expect.objectContaining({ errorCode: 'schema_validation_failed' }),
|
expect.objectContaining({ errorCode: 'schema_validation_failed' }),
|
||||||
|
|||||||
@ -230,22 +230,19 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
|||||||
);
|
);
|
||||||
|
|
||||||
if (classified.retryable) {
|
if (classified.retryable) {
|
||||||
// 重试:抛给 BullMQ(不调用 markFailed — BullMQ 会重新投递)
|
// 重试:先解锁回 queued(BullMQ retry → lockJob 可再次抢锁),然后抛给 BullMQ
|
||||||
|
await this.unlockForRetry(aiJobId);
|
||||||
throw execErr;
|
throw execErr;
|
||||||
}
|
}
|
||||||
|
|
||||||
// 永久错误
|
// 永久错误:无论 attemptCount 多少,直接 markFailed(不重试)
|
||||||
const currentAttempt = (lockedJob.attemptCount || 0) + 1;
|
|
||||||
if (currentAttempt >= (def.execution.maxRetries + 1)) {
|
|
||||||
await this.lifecycleRepo.markFailed(aiJobId, {
|
await this.lifecycleRepo.markFailed(aiJobId, {
|
||||||
errorCode: classified.errorCode,
|
errorCode: classified.errorCode,
|
||||||
publicErrorMessage: classified.publicMessage,
|
publicErrorMessage: classified.publicMessage,
|
||||||
internalErrorMessage: execErr.message?.slice(0, 500),
|
internalErrorMessage: execErr.message?.slice(0, 500),
|
||||||
});
|
});
|
||||||
} else {
|
// markFailed 后不抛错 → BullMQ 认为 job 完成(不再重试)
|
||||||
// 还有重试配额 → 抛给 BullMQ
|
return;
|
||||||
throw execErr;
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
} catch (outerErr: any) {
|
} catch (outerErr: any) {
|
||||||
// 捕获 RESOLVE 阶段的错误
|
// 捕获 RESOLVE 阶段的错误
|
||||||
@ -266,6 +263,24 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
|||||||
|
|
||||||
// ── helpers ──
|
// ── helpers ──
|
||||||
|
|
||||||
|
/**
|
||||||
|
* 解锁 Job 回 queued 状态,供 BullMQ 重试。
|
||||||
|
* 保持 attemptCount(已在 lockJob 中原子递增)。
|
||||||
|
*/
|
||||||
|
private async unlockForRetry(jobId: string): Promise<void> {
|
||||||
|
// 仅持有锁的 Worker 调用 — 无并发竞争,update 即可。
|
||||||
|
await this.prisma.aiJob.update({
|
||||||
|
where: { id: jobId },
|
||||||
|
data: {
|
||||||
|
lifecycleStatus: 'queued',
|
||||||
|
status: 'pending',
|
||||||
|
lockedBy: null,
|
||||||
|
lockedAt: null,
|
||||||
|
lockUntil: null,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
private computeHash(content: string): string {
|
private computeHash(content: string): string {
|
||||||
const crypto = require('crypto');
|
const crypto = require('crypto');
|
||||||
return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16);
|
return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16);
|
||||||
|
|||||||
@ -178,6 +178,7 @@ describe('AiJobLifecycleRepository', () => {
|
|||||||
expect(callArgs.data.lockedBy).toBe('worker-1');
|
expect(callArgs.data.lockedBy).toBe('worker-1');
|
||||||
expect(callArgs.data.startedAt).toBeTruthy();
|
expect(callArgs.data.startedAt).toBeTruthy();
|
||||||
expect(callArgs.data.lockUntil).toBeTruthy();
|
expect(callArgs.data.lockUntil).toBeTruthy();
|
||||||
|
expect(callArgs.data.attemptCount).toEqual({ increment: 1 }); // 每次 lockJob 原子递增
|
||||||
expect(result.lifecycleStatus).toBe('running');
|
expect(result.lifecycleStatus).toBe('running');
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
@ -144,6 +144,7 @@ export class AiJobLifecycleRepository {
|
|||||||
lockedAt: now,
|
lockedAt: now,
|
||||||
lockUntil,
|
lockUntil,
|
||||||
startedAt: now,
|
startedAt: now,
|
||||||
|
attemptCount: { increment: 1 }, // 原子递增,每次 lockJob 真实执行次数 +1
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user