From 06a0e21bbfe153dd054df53aa897be93e3b8f0bc Mon Sep 17 00:00:00 2001 From: wangdl Date: Mon, 22 Jun 2026 21:43:15 +0800 Subject: [PATCH] feat(M-AI-08-04): learning analysis validator + engine integration - Schema validation: strengths/weaknesses/trends/risks/recommendations limits - Evidence validation: sourceType enum, metricKey required - Business validation: insufficientData confidence cap, evidence-required conclusions - Content safety: medical diagnosis, personality judgment, markdown code blocks - Engine: validate after executor in learning_analysis dispatch Co-Authored-By: Claude --- .../ai-job/ai-job-execution-engine.spec.ts | 2 + src/modules/ai-job/ai-job-execution-engine.ts | 10 +- src/modules/ai-job/ai-job.module.ts | 2 + .../ai-job/learning-analysis-validator.ts | 118 ++++++++++++++++++ 4 files changed, 131 insertions(+), 1 deletion(-) create mode 100644 src/modules/ai-job/learning-analysis-validator.ts diff --git a/src/modules/ai-job/ai-job-execution-engine.spec.ts b/src/modules/ai-job/ai-job-execution-engine.spec.ts index 589558a..4bb4c5f 100644 --- a/src/modules/ai-job/ai-job-execution-engine.spec.ts +++ b/src/modules/ai-job/ai-job-execution-engine.spec.ts @@ -15,6 +15,7 @@ import { ReviewCardGenerationValidator } from './review-card-generation-validato import { QuizGenerationExecutor } from './quiz-generation-executor'; import { QuizGenerationValidator } from './quiz-generation-validator'; import { LearningAnalysisExecutor } from './learning-analysis-executor'; +import { LearningAnalysisValidator } from './learning-analysis-validator'; import { PrismaService } from '../../infrastructure/database/prisma.service'; import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors'; @@ -98,6 +99,7 @@ describe('AiJobExecutionEngineImpl', () => { { provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } }, { provide: QuizGenerationValidator, useValue: { validate: jest.fn() } }, { provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } }, + { provide: LearningAnalysisValidator, useValue: { validate: jest.fn() } }, { provide: ActiveRecallObservabilityService, useValue: { incrementUnifiedExecuteSuccess: jest.fn(), incrementUnifiedExecuteFailed: jest.fn(), diff --git a/src/modules/ai-job/ai-job-execution-engine.ts b/src/modules/ai-job/ai-job-execution-engine.ts index b3aee1e..5c43065 100644 --- a/src/modules/ai-job/ai-job-execution-engine.ts +++ b/src/modules/ai-job/ai-job-execution-engine.ts @@ -24,7 +24,7 @@ import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-anal import { QuizGenerationExecutor } from './quiz-generation-executor'; import { QuizGenerationValidator } from './quiz-generation-validator'; import { LearningAnalysisExecutor } from './learning-analysis-executor'; -import { LearningAnalysisExecutor } from './learning-analysis-executor'; +import { LearningAnalysisValidator } from './learning-analysis-validator'; import { AiJobExecutionEngine, EngineJobContext, @@ -104,6 +104,7 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine { private readonly quizExecutor: QuizGenerationExecutor, private readonly quizValidator: QuizGenerationValidator, private readonly learningAnalysisExecutor: LearningAnalysisExecutor, + private readonly learningAnalysisValidator: LearningAnalysisValidator, private readonly observability: ActiveRecallObservabilityService, private readonly feynmanObs: FeynmanObservabilityService, ) {} @@ -278,6 +279,13 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine { `Learning Analysis Executor completed: job=${aiJobId} ` + `confidence=${(parsedOutput as any)?.confidence}`, ); + // M-AI-08-04: Output validation + try { + this.learningAnalysisValidator.validate(parsedOutput as LearningAnalysisResult, snapshot as any); + } catch (validationErr: any) { + this.logger.warn(`Learning analysis validation failed for job=${aiJobId}: ${validationErr.message}`); + throw validationErr; + } } else { // 默认路径：直接调用 AiGateway（synthetic_job 等） response = await this.aiGateway.generate( diff --git a/src/modules/ai-job/ai-job.module.ts b/src/modules/ai-job/ai-job.module.ts index b99c7f8..8299354 100644 --- a/src/modules/ai-job/ai-job.module.ts +++ b/src/modules/ai-job/ai-job.module.ts @@ -42,6 +42,7 @@ import { QuizGenerationProjector } from './quiz-generation-projector'; import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service'; import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder'; import { LearningAnalysisExecutor } from './learning-analysis-executor'; +import { LearningAnalysisValidator } from './learning-analysis-validator'; import { FeynmanBusinessValidator, FeynmanReferenceValidator, @@ -92,6 +93,7 @@ import { AppConfigModule } from '../config/config.module'; LearningAnalysisRegistrationService, LearningAnalysisSnapshotBuilder, LearningAnalysisExecutor, + LearningAnalysisValidator, { provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any, { provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl }, ], diff --git a/src/modules/ai-job/learning-analysis-validator.ts b/src/modules/ai-job/learning-analysis-validator.ts new file mode 100644 index 0000000..f334bd6 --- /dev/null +++ b/src/modules/ai-job/learning-analysis-validator.ts @@ -0,0 +1,118 @@ +import { Injectable, Logger } from '@nestjs/common'; +import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-analysis.schema'; +import { BusinessValidationError, ReferenceValidationError } from './active-recall-validator'; +import type { LearningAnalysisSnapshot } from './learning-analysis-snapshot-builder'; + +export { BusinessValidationError, ReferenceValidationError }; + +const VALID_SOURCE_TYPES = ['study_metric', 'review_metric', 'quiz_metric', 'active_recall', 'feynman', 'knowledge_progress']; +const VALID_DIRECTIONS = ['improving', 'declining', 'stable']; +const VALID_SEVERITIES = ['low', 'medium', 'high']; + +const CODE_BLOCK_PATTERN = /```[\s\S]*?```/; +const MEDICAL_PATTERNS = [/ADHD|多动症|抑郁症|depression|焦虑症|anxiety|自闭|autism|强迫症|OCD|创伤后|PTSD|双相|bipolar/i]; +const PERSONALITY_PATTERNS = [/你是一个?.*的人/, /你的性格/, /你天生/, /你太懒/, /你太笨/, /你不适合/]; + +@Injectable() +export class LearningAnalysisValidator { + private readonly logger = new Logger(LearningAnalysisValidator.name); + + validate(output: LearningAnalysisResult, snapshot?: LearningAnalysisSnapshot): void { + const violations: string[] = []; + + // ── Schema validation ── + if (!output.summary || output.summary.trim().length === 0) violations.push('summary required'); + if (output.summary && output.summary.length > 500) violations.push('summary exceeds 500 chars'); + + if (!Array.isArray(output.strengths)) violations.push('strengths must be array'); + else if (output.strengths.length > 3) violations.push(`strengths max 3, got ${output.strengths.length}`); + + if (!Array.isArray(output.weaknesses)) violations.push('weaknesses must be array'); + else if (output.weaknesses.length > 5) violations.push(`weaknesses max 5, got ${output.weaknesses.length}`); + + if (!Array.isArray(output.trends)) violations.push('trends must be array'); + else if (output.trends.length > 3) violations.push(`trends max 3, got ${output.trends.length}`); + + if (!Array.isArray(output.risks)) violations.push('risks must be array'); + else if (output.risks.length > 3) violations.push(`risks max 3, got ${output.risks.length}`); + + if (!Array.isArray(output.recommendations)) violations.push('recommendations must be array'); + else if (output.recommendations.length === 0) violations.push('recommendations empty'); + else if (output.recommendations.length > 5) violations.push(`recommendations max 5, got ${output.recommendations.length}`); + + if (typeof output.confidence !== 'number' || output.confidence < 0 || output.confidence > 1) { + violations.push('confidence must be 0-1'); + } + if (typeof output.insufficientData !== 'boolean') violations.push('insufficientData must be boolean'); + + // ── Evidence validation ── + const allEvidenceItems = [ + ...(output.strengths || []).flatMap(s => s.evidenceRefs || []), + ...(output.weaknesses || []).flatMap(w => w.evidenceRefs || []), + ...(output.trends || []).flatMap(t => t.evidenceRefs || []), + ...(output.risks || []).flatMap(r => r.evidenceRefs || []), + ...(output.recommendations || []).flatMap(r => r.evidenceRefs || []), + ]; + + for (const ref of allEvidenceItems) { + if (!VALID_SOURCE_TYPES.includes(ref.sourceType)) { + violations.push(`invalid evidence sourceType: "${ref.sourceType}"`); + } + if (!ref.metricKey || typeof ref.metricKey !== 'string') { + violations.push('evidence missing metricKey'); + } + // Validate entityId belongs to snapshot if provided + if (ref.entityId && snapshot) { + const snap = snapshot.snapshot; + const validIds = new Set([ + ...(snap.knowledgeProgress.weakItems?.map(w => w.id) || []), + ]); + // Not in weak items - could be in other lists; skip strict validation for now + } + } + + // ── Business validation ── + if (output.insufficientData) { + if (output.confidence > 0.5) { + violations.push('insufficientData=true but confidence > 0.5'); + } + } + + if (output.strengths?.length > 0 || output.weaknesses?.length > 0) { + for (const s of output.strengths || []) { + if (!s.evidenceRefs || s.evidenceRefs.length === 0) { + violations.push(`strength "${s.title}" has no evidence`); + } + } + for (const w of output.weaknesses || []) { + if (!w.evidenceRefs || w.evidenceRefs.length === 0) { + violations.push(`weakness "${w.title}" has no evidence`); + } + } + } + + // ── Content safety ── + const allText = [ + output.summary || '', + ...(output.strengths || []).flatMap(s => [s.title, s.description]), + ...(output.weaknesses || []).flatMap(w => [w.title, w.description]), + ...(output.recommendations || []).flatMap(r => [r.title, r.reason]), + ]; + + for (const text of allText) { + if (CODE_BLOCK_PATTERN.test(text)) violations.push('output contains markdown code block'); + for (const pat of MEDICAL_PATTERNS) { + if (pat.test(text)) { violations.push('output contains medical diagnosis terms'); break; } + } + for (const pat of PERSONALITY_PATTERNS) { + if (pat.test(text)) { violations.push('output contains personality judgment'); break; } + } + } + + if (violations.length > 0) { + this.logger.warn(`Learning analysis validation failed: ${violations.join('; ')}`); + throw new BusinessValidationError(`Validation failed: ${violations.length} violation(s)`, violations); + } + this.logger.log('Learning analysis validation passed'); + } +}