import { Injectable, Logger } from '@nestjs/common'; import { AiGatewayService } from '../ai/gateway/ai-gateway.service'; import { AiJobLifecycleRepository } from './ai-job-lifecycle.repository'; import { JobDefinitionRegistry } from './job-definition-registry'; import { AiJobStateMachine } from './ai-job-state-machine'; import { PrismaService } from '../../infrastructure/database/prisma.service'; import { AiJobExecutionEngine, EngineJobContext, } from './ai-job-execution-engine.interface'; import { JobLockConflictError, JobAlreadyTerminalError, JobNotCancellableError, } from './ai-job.errors'; // ═══════════════════════════════════════════════════════════ // 错误分类(ADR-003 §5.2) // ═══════════════════════════════════════════════════════════ interface ClassifiedError { errorCode: string; publicMessage: string; retryable: boolean; } function classifyError(err: any): ClassifiedError { const msg = err?.message || ''; if (err?.code === 'JOB_CANCELLED' || msg.includes('cancelled')) { return { errorCode: 'cancelled', publicMessage: 'Job was cancelled', retryable: false }; } if (msg.includes('429') || msg.includes('rate') || msg.includes('Rate')) { return { errorCode: 'provider_rate_limited', publicMessage: 'AI 服务繁忙,请稍后重试', retryable: true }; } if (msg.includes('timeout') || msg.includes('ETIMEDOUT') || msg.includes('AbortError')) { return { errorCode: 'provider_timeout', publicMessage: 'AI 服务响应超时,请重试', retryable: true }; } if (msg.includes('5xx') || msg.includes('503') || msg.includes('502') || msg.includes('unavailable')) { return { errorCode: 'provider_unavailable', publicMessage: 'AI 服务暂不可用', retryable: true }; } if (msg.includes('schema') || msg.includes('validation') || msg.includes('invalid')) { return { errorCode: 'schema_validation_failed', publicMessage: 'AI 输出格式异常', retryable: false }; } if (msg.includes('business') || msg.includes('Business')) { return { errorCode: 'business_validation_failed', publicMessage: 'AI 输出不符合业务规则', retryable: false }; } if (msg.includes('reference') || msg.includes('Reference')) { return { errorCode: 'reference_validation_failed', publicMessage: 'AI 输出引用无效', retryable: false }; } if (err instanceof JobLockConflictError || err instanceof JobAlreadyTerminalError) { return { errorCode: 'internal_error', publicMessage: err.message, retryable: false }; } return { errorCode: 'internal_error', publicMessage: '内部错误', retryable: true }; } /** * M-AI-03-08: AiJobExecutionEngine * * 统一执行管线(ADR-003 §5.1): * PREPARE → RESOLVE → EXECUTE → PROJECT → COMPLETE * * 供 AiInteractiveJobWorker / AiBackgroundJobWorker 调用。 * 替代 #288 中的 no-op 占位。 */ @Injectable() export class AiJobExecutionEngineImpl implements AiJobExecutionEngine { private readonly logger = new Logger(AiJobExecutionEngineImpl.name); constructor( private readonly prisma: PrismaService, private readonly lifecycleRepo: AiJobLifecycleRepository, private readonly registry: JobDefinitionRegistry, private readonly stateMachine: AiJobStateMachine, private readonly aiGateway: AiGatewayService, // #292: ResultProjector 将在后续 Issue 注入 ) {} async execute(aiJobId: string, context: EngineJobContext): Promise { // ── PREPARE ── // 1. 读取 AiJob const job = await this.prisma.aiJob.findUnique({ where: { id: aiJobId } }); if (!job) { this.logger.error(`Job ${aiJobId} not found`); return; } // 2. 检查是否终态 const currentStatus = this.stateMachine.parse(job.lifecycleStatus); if (this.stateMachine.isTerminal(currentStatus)) { this.logger.warn(`Job ${aiJobId} already in terminal status "${currentStatus}" — skipping`); return; } // 3. 检查取消请求(queued 状态直接取消,running 状态设置信号) if (job.cancelRequestedAt) { await this.lifecycleRepo.markCancelled(aiJobId); return; } // 4. Registry 获取 Definition const def = this.registry.get(job.jobType); // 5. CAS 抢锁 (queued → running),递增 attemptCount // lockJob 内部:如果 AttemptCount > maxAttempts 或已经是终态,会抛错 let lockedJob: any; try { lockedJob = await this.lifecycleRepo.lockJob(aiJobId, context.jobId || 'engine'); } catch (err: any) { // 已被其他 Worker 抢走 → 静默退出,不重试 if (err instanceof JobLockConflictError) { this.logger.warn(`Job ${aiJobId} already locked by another worker — skipping`); return; } if (err instanceof JobAlreadyTerminalError) { this.logger.warn(`Job ${aiJobId} already terminal — skipping`); return; } throw err; } await context.updateProgress(10); // ── RESOLVE ── let snapshot: any; try { // 6. 加载 Snapshot const snap = await this.prisma.aiJobSnapshot.findUnique({ where: { jobId: aiJobId }, }); if (snap) { // 验证 schemaVersion 兼容性 if (snap.schemaVersion !== def.input.schemaVersion) { await this.lifecycleRepo.markFailed(aiJobId, { errorCode: 'schema_validation_failed', publicErrorMessage: 'Snapshot schema version mismatch', internalErrorMessage: `Expected ${def.input.schemaVersion}, got ${snap.schemaVersion}`, }); return; } snapshot = snap.content; } // 7. Provider timeout via AbortController const controller = new AbortController(); const timeoutMs = def.execution.timeoutMs || 30000; const timeoutId = setTimeout(() => controller.abort(), timeoutMs); try { // 8. 取消检查(Provider 调用前) const currentJob = await this.prisma.aiJob.findUnique({ where: { id: aiJobId }, select: { cancelRequestedAt: true }, }); if (currentJob?.cancelRequestedAt) { await this.lifecycleRepo.markCancelled(aiJobId); return; } await context.updateProgress(30); // ── EXECUTE ── // 9. 调用 AiGatewayService(不直接导入 Provider SDK) const response = await this.aiGateway.generate( { userId: job.userId, feature: job.jobType, tier: def.model.modelTier as any, promptKey: def.prompt.promptKey, promptVersion: def.prompt.promptVersion, messages: [], // Snapshot content 应通过 promptTemplate 渲染,此处由 AiGateway 处理 maxTokens: def.model.maxTokens, outputSchema: undefined, // 由 AiGateway 的 parseJson 处理 }, timeoutMs, ); clearTimeout(timeoutId); await context.updateProgress(70); // 10. 取消检查(Projector 前) const jobAfterExec = await this.prisma.aiJob.findUnique({ where: { id: aiJobId }, select: { cancelRequestedAt: true }, }); if (jobAfterExec?.cancelRequestedAt) { await this.lifecycleRepo.markCancelled(aiJobId); return; } // ── PROJECT ── // #292: ResultProjector 在此调用。当前阶段仅写 validatedOutput + outputHash。 await this.prisma.aiJob.update({ where: { id: aiJobId }, data: { validatedOutput: response.parsed as any, outputHash: this.computeHash(JSON.stringify(response.parsed)), }, }); await context.updateProgress(90); // ── COMPLETE ── // 11. Usage logging await this.writeUsageLog(job.userId, aiJobId, def, response, lockedJob.attemptCount || 0).catch((err) => { this.logger.error(`Usage log failed for job=${aiJobId}: ${err.message}`); }); // 12. 标记成功 await this.lifecycleRepo.markSucceeded(aiJobId); await context.updateProgress(100); this.logger.log(`Job ${aiJobId} (${job.jobType}) completed successfully`); } catch (execErr: any) { clearTimeout(timeoutId); // 取消检查 if (execErr?.message?.includes('cancelled')) { await this.lifecycleRepo.markCancelled(aiJobId); return; } // 错误分类与处理 const classified = classifyError(execErr); this.logger.error( `Job ${aiJobId} execution error: ${classified.errorCode} ` + `retryable=${classified.retryable} msg=${execErr.message}`, ); if (classified.retryable) { // 重试:先解锁回 queued(BullMQ retry → lockJob 可再次抢锁),然后抛给 BullMQ await this.unlockForRetry(aiJobId); throw execErr; } // 永久错误:无论 attemptCount 多少,直接 markFailed(不重试) await this.lifecycleRepo.markFailed(aiJobId, { errorCode: classified.errorCode, publicErrorMessage: classified.publicMessage, internalErrorMessage: execErr.message?.slice(0, 500), }); // markFailed 后不抛错 → BullMQ 认为 job 完成(不再重试) return; } } catch (outerErr: any) { // 捕获 RESOLVE 阶段的错误 const classified = classifyError(outerErr); this.logger.error( `Job ${aiJobId} resolve error: ${classified.errorCode} — ${outerErr.message}`, ); if (!classified.retryable) { await this.lifecycleRepo.markFailed(aiJobId, { errorCode: classified.errorCode, publicErrorMessage: classified.publicMessage, internalErrorMessage: outerErr.message?.slice(0, 500), }).catch(() => {}); } throw outerErr; } } // ── helpers ── /** * 解锁 Job 回 queued 状态,供 BullMQ 重试。 * 保持 attemptCount(已在 lockJob 中原子递增)。 */ private async unlockForRetry(jobId: string): Promise { // 仅持有锁的 Worker 调用 — 无并发竞争,update 即可。 await this.prisma.aiJob.update({ where: { id: jobId }, data: { lifecycleStatus: 'queued', status: 'pending', lockedBy: null, lockedAt: null, lockUntil: null, }, }); } private computeHash(content: string): string { const crypto = require('crypto'); return crypto.createHash('sha256').update(content).digest('hex').substring(0, 16); } private async writeUsageLog( userId: string, jobId: string, _def: any, response: any, attemptNo: number, ): Promise { await this.prisma.aiUsageLog.create({ data: { userId, jobId, provider: response?.usage?.provider || 'deepseek', model: response?.usage?.model || 'deepseek-chat', tier: 'primary', promptKey: _def?.prompt?.promptKey || 'unknown', promptVersion: _def?.prompt?.promptVersion || 'unknown', inputTokens: response?.usage?.inputTokens || 0, outputTokens: response?.usage?.outputTokens || 0, estimatedCost: response?.usage?.estimatedCost || 0, latencyMs: response?.usage?.latencyMs || 0, success: true, attemptNo, credentialMode: 'platform_key', } as any, }); } }