fix: B1 — lockJob 原子递增 attemptCount + Engine 重试解锁逻辑
Some checks failed
Deploy API Server / build-and-unit (push) Failing after 28s
Deploy API Server / current-integration (push) Has been skipped
Deploy API Server / backward-compat (push) Has been skipped
Deploy API Server / deploy (push) Has been skipped

- lockJob updateMany 新增 attemptCount: { increment: 1 }
- Engine retryable 错误: unlockForRetry() 重置为 queued 后再抛给 BullMQ
- Engine permanent 错误: 直接 markFailed + return(不抛错,BullMQ 不重试)
- unlockForRetry: 重置 lifecycleStatus→queued + 清除 lockedBy/lockedAt/lockUntil
- 更新测试: lockJob 验证 attemptCount 递增; schema validation 验证不抛错

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wangdl 2026-06-20 18:42:19 +08:00
parent d82ae8ba71
commit cdcf056ba5
4 changed files with 34 additions and 16 deletions

View File

@ -166,14 +166,15 @@ describe('AiJobExecutionEngineImpl', () => {
expect(lifecycleRepo.markFailed).not.toHaveBeenCalled(); expect(lifecycleRepo.markFailed).not.toHaveBeenCalled();
}); });
it('schema validation → permanent → markFailed', async () => { it('schema validation → permanent → markFailed不抛错BullMQ 不再重试)', async () => {
prisma.aiJob.findUnique.mockResolvedValue(makeJob()); prisma.aiJob.findUnique.mockResolvedValue(makeJob());
lifecycleRepo.lockJob.mockResolvedValue(makeJob({ lifecycleStatus: 'running' })); lifecycleRepo.lockJob.mockResolvedValue(makeJob({ lifecycleStatus: 'running', attemptCount: 1 }));
prisma.aiJobSnapshot.findUnique.mockResolvedValue({ schemaVersion: '1.0', content: {} }); prisma.aiJobSnapshot.findUnique.mockResolvedValue({ schemaVersion: '1.0', content: {} });
aiGateway.generate.mockRejectedValue(new Error('schema validation failed')); aiGateway.generate.mockRejectedValue(new Error('schema validation failed'));
await expect(engine.execute('job-001', ctx)).rejects.toThrow(); // 永久错误:不应抛出(避免 BullMQ retry
await engine.execute('job-001', ctx);
expect(lifecycleRepo.markFailed).toHaveBeenCalledWith( expect(lifecycleRepo.markFailed).toHaveBeenCalledWith(
'job-001', 'job-001',
expect.objectContaining({ errorCode: 'schema_validation_failed' }), expect.objectContaining({ errorCode: 'schema_validation_failed' }),

View File

@ -230,22 +230,19 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
); );
if (classified.retryable) { if (classified.retryable) {
// 重试:抛给 BullMQ不调用 markFailed — BullMQ 会重新投递) // 重试:先解锁回 queuedBullMQ retry → lockJob 可再次抢锁),然后抛给 BullMQ
await this.unlockForRetry(aiJobId);
throw execErr; throw execErr;
} }
// 永久错误 // 永久错误:无论 attemptCount 多少,直接 markFailed不重试
const currentAttempt = (lockedJob.attemptCount || 0) + 1;
if (currentAttempt >= (def.execution.maxRetries + 1)) {
await this.lifecycleRepo.markFailed(aiJobId, { await this.lifecycleRepo.markFailed(aiJobId, {
errorCode: classified.errorCode, errorCode: classified.errorCode,
publicErrorMessage: classified.publicMessage, publicErrorMessage: classified.publicMessage,
internalErrorMessage: execErr.message?.slice(0, 500), internalErrorMessage: execErr.message?.slice(0, 500),
}); });
} else { // markFailed 后不抛错 → BullMQ 认为 job 完成(不再重试)
// 还有重试配额 → 抛给 BullMQ return;
throw execErr;
}
} }
} catch (outerErr: any) { } catch (outerErr: any) {
// 捕获 RESOLVE 阶段的错误 // 捕获 RESOLVE 阶段的错误
@ -266,6 +263,24 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
// ── helpers ── // ── helpers ──
/**
* Job queued BullMQ
* attemptCount lockJob
*/
private async unlockForRetry(jobId: string): Promise<void> {
// 仅持有锁的 Worker 调用 — 无并发竞争update 即可。
await this.prisma.aiJob.update({
where: { id: jobId },
data: {
lifecycleStatus: 'queued',
status: 'pending',
lockedBy: null,
lockedAt: null,
lockUntil: null,
},
});
}
private computeHash(content: string): string { private computeHash(content: string): string {
const crypto = require('crypto'); const crypto = require('crypto');
return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16); return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16);

View File

@ -178,6 +178,7 @@ describe('AiJobLifecycleRepository', () => {
expect(callArgs.data.lockedBy).toBe('worker-1'); expect(callArgs.data.lockedBy).toBe('worker-1');
expect(callArgs.data.startedAt).toBeTruthy(); expect(callArgs.data.startedAt).toBeTruthy();
expect(callArgs.data.lockUntil).toBeTruthy(); expect(callArgs.data.lockUntil).toBeTruthy();
expect(callArgs.data.attemptCount).toEqual({ increment: 1 }); // 每次 lockJob 原子递增
expect(result.lifecycleStatus).toBe('running'); expect(result.lifecycleStatus).toBe('running');
}); });

View File

@ -144,6 +144,7 @@ export class AiJobLifecycleRepository {
lockedAt: now, lockedAt: now,
lockUntil, lockUntil,
startedAt: now, startedAt: now,
attemptCount: { increment: 1 }, // 原子递增,每次 lockJob 真实执行次数 +1
}, },
}); });