cdcf056ba5
- lockJob updateMany 新增 attemptCount: { increment: 1 }
- Engine retryable 错误: unlockForRetry() 重置为 queued 后再抛给 BullMQ
- Engine permanent 错误: 直接 markFailed + return（不抛错，BullMQ 不重试）
- unlockForRetry: 重置 lifecycleStatus→queued + 清除 lockedBy/lockedAt/lockUntil
- 更新测试: lockJob 验证 attemptCount 递增; schema validation 验证不抛错
Co-Authored-By: Claude <noreply@anthropic.com>
316 lines
12 KiB
TypeScript
316 lines
12 KiB
TypeScript
import { Injectable, Logger } from '@nestjs/common';
|
||
import { AiGatewayService } from '../ai/gateway/ai-gateway.service';
|
||
import { AiJobLifecycleRepository } from './ai-job-lifecycle.repository';
|
||
import { JobDefinitionRegistry } from './job-definition-registry';
|
||
import { AiJobStateMachine } from './ai-job-state-machine';
|
||
import { PrismaService } from '../../infrastructure/database/prisma.service';
|
||
import {
|
||
AiJobExecutionEngine,
|
||
EngineJobContext,
|
||
} from './ai-job-execution-engine.interface';
|
||
import {
|
||
JobLockConflictError,
|
||
JobAlreadyTerminalError,
|
||
JobNotCancellableError,
|
||
} from './ai-job.errors';
|
||
|
||
// ═══════════════════════════════════════════════════════════
|
||
// 错误分类（ADR-003 §5.2）
|
||
// ═══════════════════════════════════════════════════════════
|
||
|
||
interface ClassifiedError {
|
||
errorCode: string;
|
||
publicMessage: string;
|
||
retryable: boolean;
|
||
}
|
||
|
||
function classifyError(err: any): ClassifiedError {
|
||
const msg = err?.message || '';
|
||
|
||
if (err?.code === 'JOB_CANCELLED' || msg.includes('cancelled')) {
|
||
return { errorCode: 'cancelled', publicMessage: 'Job was cancelled', retryable: false };
|
||
}
|
||
if (msg.includes('429') || msg.includes('rate') || msg.includes('Rate')) {
|
||
return { errorCode: 'provider_rate_limited', publicMessage: 'AI 服务繁忙，请稍后重试', retryable: true };
|
||
}
|
||
if (msg.includes('timeout') || msg.includes('ETIMEDOUT') || msg.includes('AbortError')) {
|
||
return { errorCode: 'provider_timeout', publicMessage: 'AI 服务响应超时，请重试', retryable: true };
|
||
}
|
||
if (msg.includes('5xx') || msg.includes('503') || msg.includes('502') || msg.includes('unavailable')) {
|
||
return { errorCode: 'provider_unavailable', publicMessage: 'AI 服务暂不可用', retryable: true };
|
||
}
|
||
if (msg.includes('schema') || msg.includes('validation') || msg.includes('invalid')) {
|
||
return { errorCode: 'schema_validation_failed', publicMessage: 'AI 输出格式异常', retryable: false };
|
||
}
|
||
if (msg.includes('business') || msg.includes('Business')) {
|
||
return { errorCode: 'business_validation_failed', publicMessage: 'AI 输出不符合业务规则', retryable: false };
|
||
}
|
||
if (msg.includes('reference') || msg.includes('Reference')) {
|
||
return { errorCode: 'reference_validation_failed', publicMessage: 'AI 输出引用无效', retryable: false };
|
||
}
|
||
if (err instanceof JobLockConflictError || err instanceof JobAlreadyTerminalError) {
|
||
return { errorCode: 'internal_error', publicMessage: err.message, retryable: false };
|
||
}
|
||
|
||
return { errorCode: 'internal_error', publicMessage: '内部错误', retryable: true };
|
||
}
|
||
|
||
/**
|
||
* M-AI-03-08: AiJobExecutionEngine
|
||
*
|
||
* 统一执行管线（ADR-003 §5.1）：
|
||
* PREPARE → RESOLVE → EXECUTE → PROJECT → COMPLETE
|
||
*
|
||
* 供 AiInteractiveJobWorker / AiBackgroundJobWorker 调用。
|
||
* 替代 #288 中的 no-op 占位。
|
||
*/
|
||
@Injectable()
|
||
export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
||
private readonly logger = new Logger(AiJobExecutionEngineImpl.name);
|
||
|
||
constructor(
|
||
private readonly prisma: PrismaService,
|
||
private readonly lifecycleRepo: AiJobLifecycleRepository,
|
||
private readonly registry: JobDefinitionRegistry,
|
||
private readonly stateMachine: AiJobStateMachine,
|
||
private readonly aiGateway: AiGatewayService,
|
||
// #292: ResultProjector 将在后续 Issue 注入
|
||
) {}
|
||
|
||
async execute(aiJobId: string, context: EngineJobContext): Promise<void> {
|
||
// ── PREPARE ──
|
||
// 1. 读取 AiJob
|
||
const job = await this.prisma.aiJob.findUnique({ where: { id: aiJobId } });
|
||
if (!job) {
|
||
this.logger.error(`Job ${aiJobId} not found`);
|
||
return;
|
||
}
|
||
|
||
// 2. 检查是否终态
|
||
const currentStatus = this.stateMachine.parse(job.lifecycleStatus);
|
||
if (this.stateMachine.isTerminal(currentStatus)) {
|
||
this.logger.warn(`Job ${aiJobId} already in terminal status "${currentStatus}" — skipping`);
|
||
return;
|
||
}
|
||
|
||
// 3. 检查取消请求（queued 状态直接取消，running 状态设置信号）
|
||
if (job.cancelRequestedAt) {
|
||
await this.lifecycleRepo.markCancelled(aiJobId);
|
||
return;
|
||
}
|
||
|
||
// 4. Registry 获取 Definition
|
||
const def = this.registry.get(job.jobType);
|
||
|
||
// 5. CAS 抢锁 (queued → running)，递增 attemptCount
|
||
// lockJob 内部：如果 AttemptCount > maxAttempts 或已经是终态，会抛错
|
||
let lockedJob: any;
|
||
try {
|
||
lockedJob = await this.lifecycleRepo.lockJob(aiJobId, context.jobId || 'engine');
|
||
} catch (err: any) {
|
||
// 已被其他 Worker 抢走 → 静默退出，不重试
|
||
if (err instanceof JobLockConflictError) {
|
||
this.logger.warn(`Job ${aiJobId} already locked by another worker — skipping`);
|
||
return;
|
||
}
|
||
if (err instanceof JobAlreadyTerminalError) {
|
||
this.logger.warn(`Job ${aiJobId} already terminal — skipping`);
|
||
return;
|
||
}
|
||
throw err;
|
||
}
|
||
|
||
await context.updateProgress(10);
|
||
|
||
// ── RESOLVE ──
|
||
let snapshot: any;
|
||
try {
|
||
// 6. 加载 Snapshot
|
||
const snap = await this.prisma.aiJobSnapshot.findUnique({
|
||
where: { jobId: aiJobId },
|
||
});
|
||
if (snap) {
|
||
// 验证 schemaVersion 兼容性
|
||
if (snap.schemaVersion !== def.input.schemaVersion) {
|
||
await this.lifecycleRepo.markFailed(aiJobId, {
|
||
errorCode: 'schema_validation_failed',
|
||
publicErrorMessage: 'Snapshot schema version mismatch',
|
||
internalErrorMessage: `Expected ${def.input.schemaVersion}, got ${snap.schemaVersion}`,
|
||
});
|
||
return;
|
||
}
|
||
snapshot = snap.content;
|
||
}
|
||
|
||
// 7. Provider timeout via AbortController
|
||
const controller = new AbortController();
|
||
const timeoutMs = def.execution.timeoutMs || 30000;
|
||
const timeoutId = setTimeout(() => controller.abort(), timeoutMs);
|
||
|
||
try {
|
||
// 8. 取消检查（Provider 调用前）
|
||
const currentJob = await this.prisma.aiJob.findUnique({
|
||
where: { id: aiJobId },
|
||
select: { cancelRequestedAt: true },
|
||
});
|
||
if (currentJob?.cancelRequestedAt) {
|
||
await this.lifecycleRepo.markCancelled(aiJobId);
|
||
return;
|
||
}
|
||
|
||
await context.updateProgress(30);
|
||
|
||
// ── EXECUTE ──
|
||
// 9. 调用 AiGatewayService（不直接导入 Provider SDK）
|
||
const response = await this.aiGateway.generate(
|
||
{
|
||
userId: job.userId,
|
||
feature: job.jobType,
|
||
tier: def.model.modelTier as any,
|
||
promptKey: def.prompt.promptKey,
|
||
promptVersion: def.prompt.promptVersion,
|
||
messages: [], // Snapshot content 应通过 promptTemplate 渲染，此处由 AiGateway 处理
|
||
maxTokens: def.model.maxTokens,
|
||
outputSchema: undefined, // 由 AiGateway 的 parseJson 处理
|
||
},
|
||
timeoutMs,
|
||
);
|
||
|
||
clearTimeout(timeoutId);
|
||
|
||
await context.updateProgress(70);
|
||
|
||
// 10. 取消检查（Projector 前）
|
||
const jobAfterExec = await this.prisma.aiJob.findUnique({
|
||
where: { id: aiJobId },
|
||
select: { cancelRequestedAt: true },
|
||
});
|
||
if (jobAfterExec?.cancelRequestedAt) {
|
||
await this.lifecycleRepo.markCancelled(aiJobId);
|
||
return;
|
||
}
|
||
|
||
// ── PROJECT ──
|
||
// #292: ResultProjector 在此调用。当前阶段仅写 validatedOutput + outputHash。
|
||
await this.prisma.aiJob.update({
|
||
where: { id: aiJobId },
|
||
data: {
|
||
validatedOutput: response.parsed as any,
|
||
outputHash: this.computeHash(JSON.stringify(response.parsed)),
|
||
},
|
||
});
|
||
|
||
await context.updateProgress(90);
|
||
|
||
// ── COMPLETE ──
|
||
// 11. Usage logging
|
||
await this.writeUsageLog(job.userId, aiJobId, def, response, lockedJob.attemptCount || 0).catch((err) => {
|
||
this.logger.error(`Usage log failed for job=${aiJobId}: ${err.message}`);
|
||
});
|
||
|
||
// 12. 标记成功
|
||
await this.lifecycleRepo.markSucceeded(aiJobId);
|
||
|
||
await context.updateProgress(100);
|
||
this.logger.log(`Job ${aiJobId} (${job.jobType}) completed successfully`);
|
||
} catch (execErr: any) {
|
||
clearTimeout(timeoutId);
|
||
|
||
// 取消检查
|
||
if (execErr?.message?.includes('cancelled')) {
|
||
await this.lifecycleRepo.markCancelled(aiJobId);
|
||
return;
|
||
}
|
||
|
||
// 错误分类与处理
|
||
const classified = classifyError(execErr);
|
||
this.logger.error(
|
||
`Job ${aiJobId} execution error: ${classified.errorCode} ` +
|
||
`retryable=${classified.retryable} msg=${execErr.message}`,
|
||
);
|
||
|
||
if (classified.retryable) {
|
||
// 重试：先解锁回 queued（BullMQ retry → lockJob 可再次抢锁），然后抛给 BullMQ
|
||
await this.unlockForRetry(aiJobId);
|
||
throw execErr;
|
||
}
|
||
|
||
// 永久错误：无论 attemptCount 多少，直接 markFailed（不重试）
|
||
await this.lifecycleRepo.markFailed(aiJobId, {
|
||
errorCode: classified.errorCode,
|
||
publicErrorMessage: classified.publicMessage,
|
||
internalErrorMessage: execErr.message?.slice(0, 500),
|
||
});
|
||
// markFailed 后不抛错 → BullMQ 认为 job 完成（不再重试）
|
||
return;
|
||
}
|
||
} catch (outerErr: any) {
|
||
// 捕获 RESOLVE 阶段的错误
|
||
const classified = classifyError(outerErr);
|
||
this.logger.error(
|
||
`Job ${aiJobId} resolve error: ${classified.errorCode} — ${outerErr.message}`,
|
||
);
|
||
if (!classified.retryable) {
|
||
await this.lifecycleRepo.markFailed(aiJobId, {
|
||
errorCode: classified.errorCode,
|
||
publicErrorMessage: classified.publicMessage,
|
||
internalErrorMessage: outerErr.message?.slice(0, 500),
|
||
}).catch(() => {});
|
||
}
|
||
throw outerErr;
|
||
}
|
||
}
|
||
|
||
// ── helpers ──
|
||
|
||
/**
|
||
* 解锁 Job 回 queued 状态，供 BullMQ 重试。
|
||
* 保持 attemptCount（已在 lockJob 中原子递增）。
|
||
*/
|
||
private async unlockForRetry(jobId: string): Promise<void> {
|
||
// 仅持有锁的 Worker 调用 — 无并发竞争，update 即可。
|
||
await this.prisma.aiJob.update({
|
||
where: { id: jobId },
|
||
data: {
|
||
lifecycleStatus: 'queued',
|
||
status: 'pending',
|
||
lockedBy: null,
|
||
lockedAt: null,
|
||
lockUntil: null,
|
||
},
|
||
});
|
||
}
|
||
|
||
private computeHash(content: string): string {
|
||
const crypto = require('crypto');
|
||
return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16);
|
||
}
|
||
|
||
private async writeUsageLog(
|
||
userId: string,
|
||
jobId: string,
|
||
_def: any,
|
||
response: any,
|
||
attemptNo: number,
|
||
): Promise<void> {
|
||
await this.prisma.aiUsageLog.create({
|
||
data: {
|
||
userId,
|
||
jobId,
|
||
provider: response?.usage?.provider || 'deepseek',
|
||
model: response?.usage?.model || 'deepseek-chat',
|
||
tier: 'primary',
|
||
promptKey: _def?.prompt?.promptKey || 'unknown',
|
||
promptVersion: _def?.prompt?.promptVersion || 'unknown',
|
||
inputTokens: response?.usage?.inputTokens || 0,
|
||
outputTokens: response?.usage?.outputTokens || 0,
|
||
estimatedCost: response?.usage?.estimatedCost || 0,
|
||
latencyMs: response?.usage?.latencyMs || 0,
|
||
success: true,
|
||
attemptNo,
|
||
credentialMode: 'platform_key',
|
||
} as any,
|
||
});
|
||
}
|
||
}
|