feat(M-AI-08-04): learning analysis validator + engine integration
Some checks failed
Deploy API Server / build-and-unit (push) Failing after 28s
Deploy API Server / current-integration (push) Has been skipped
Deploy API Server / backward-compat (push) Has been skipped
Deploy API Server / deploy (push) Has been skipped

- Schema validation: strengths/weaknesses/trends/risks/recommendations limits
- Evidence validation: sourceType enum, metricKey required
- Business validation: insufficientData confidence cap, evidence-required conclusions
- Content safety: medical diagnosis, personality judgment, markdown code blocks
- Engine: validate after executor in learning_analysis dispatch

Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
wangdl 2026-06-22 21:43:15 +08:00
parent 1bb131a1bc
commit 06a0e21bbf
4 changed files with 131 additions and 1 deletions

View File

@ -15,6 +15,7 @@ import { ReviewCardGenerationValidator } from './review-card-generation-validato
import { QuizGenerationExecutor } from './quiz-generation-executor'; import { QuizGenerationExecutor } from './quiz-generation-executor';
import { QuizGenerationValidator } from './quiz-generation-validator'; import { QuizGenerationValidator } from './quiz-generation-validator';
import { LearningAnalysisExecutor } from './learning-analysis-executor'; import { LearningAnalysisExecutor } from './learning-analysis-executor';
import { LearningAnalysisValidator } from './learning-analysis-validator';
import { PrismaService } from '../../infrastructure/database/prisma.service'; import { PrismaService } from '../../infrastructure/database/prisma.service';
import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors'; import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors';
@ -98,6 +99,7 @@ describe('AiJobExecutionEngineImpl', () => {
{ provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } }, { provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } },
{ provide: QuizGenerationValidator, useValue: { validate: jest.fn() } }, { provide: QuizGenerationValidator, useValue: { validate: jest.fn() } },
{ provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } }, { provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } },
{ provide: LearningAnalysisValidator, useValue: { validate: jest.fn() } },
{ provide: ActiveRecallObservabilityService, useValue: { { provide: ActiveRecallObservabilityService, useValue: {
incrementUnifiedExecuteSuccess: jest.fn(), incrementUnifiedExecuteSuccess: jest.fn(),
incrementUnifiedExecuteFailed: jest.fn(), incrementUnifiedExecuteFailed: jest.fn(),

View File

@ -24,7 +24,7 @@ import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-anal
import { QuizGenerationExecutor } from './quiz-generation-executor'; import { QuizGenerationExecutor } from './quiz-generation-executor';
import { QuizGenerationValidator } from './quiz-generation-validator'; import { QuizGenerationValidator } from './quiz-generation-validator';
import { LearningAnalysisExecutor } from './learning-analysis-executor'; import { LearningAnalysisExecutor } from './learning-analysis-executor';
import { LearningAnalysisExecutor } from './learning-analysis-executor'; import { LearningAnalysisValidator } from './learning-analysis-validator';
import { import {
AiJobExecutionEngine, AiJobExecutionEngine,
EngineJobContext, EngineJobContext,
@ -104,6 +104,7 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
private readonly quizExecutor: QuizGenerationExecutor, private readonly quizExecutor: QuizGenerationExecutor,
private readonly quizValidator: QuizGenerationValidator, private readonly quizValidator: QuizGenerationValidator,
private readonly learningAnalysisExecutor: LearningAnalysisExecutor, private readonly learningAnalysisExecutor: LearningAnalysisExecutor,
private readonly learningAnalysisValidator: LearningAnalysisValidator,
private readonly observability: ActiveRecallObservabilityService, private readonly observability: ActiveRecallObservabilityService,
private readonly feynmanObs: FeynmanObservabilityService, private readonly feynmanObs: FeynmanObservabilityService,
) {} ) {}
@ -278,6 +279,13 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
`Learning Analysis Executor completed: job=${aiJobId} ` + `Learning Analysis Executor completed: job=${aiJobId} ` +
`confidence=${(parsedOutput as any)?.confidence}`, `confidence=${(parsedOutput as any)?.confidence}`,
); );
// M-AI-08-04: Output validation
try {
this.learningAnalysisValidator.validate(parsedOutput as LearningAnalysisResult, snapshot as any);
} catch (validationErr: any) {
this.logger.warn(`Learning analysis validation failed for job=${aiJobId}: ${validationErr.message}`);
throw validationErr;
}
} else { } else {
// 默认路径:直接调用 AiGatewaysynthetic_job 等) // 默认路径:直接调用 AiGatewaysynthetic_job 等)
response = await this.aiGateway.generate( response = await this.aiGateway.generate(

View File

@ -42,6 +42,7 @@ import { QuizGenerationProjector } from './quiz-generation-projector';
import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service'; import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service';
import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder'; import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder';
import { LearningAnalysisExecutor } from './learning-analysis-executor'; import { LearningAnalysisExecutor } from './learning-analysis-executor';
import { LearningAnalysisValidator } from './learning-analysis-validator';
import { import {
FeynmanBusinessValidator, FeynmanBusinessValidator,
FeynmanReferenceValidator, FeynmanReferenceValidator,
@ -92,6 +93,7 @@ import { AppConfigModule } from '../config/config.module';
LearningAnalysisRegistrationService, LearningAnalysisRegistrationService,
LearningAnalysisSnapshotBuilder, LearningAnalysisSnapshotBuilder,
LearningAnalysisExecutor, LearningAnalysisExecutor,
LearningAnalysisValidator,
{ provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any, { provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any,
{ provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl }, { provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl },
], ],

View File

@ -0,0 +1,118 @@
import { Injectable, Logger } from '@nestjs/common';
import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-analysis.schema';
import { BusinessValidationError, ReferenceValidationError } from './active-recall-validator';
import type { LearningAnalysisSnapshot } from './learning-analysis-snapshot-builder';
export { BusinessValidationError, ReferenceValidationError };
const VALID_SOURCE_TYPES = ['study_metric', 'review_metric', 'quiz_metric', 'active_recall', 'feynman', 'knowledge_progress'];
const VALID_DIRECTIONS = ['improving', 'declining', 'stable'];
const VALID_SEVERITIES = ['low', 'medium', 'high'];
const CODE_BLOCK_PATTERN = /```[\s\S]*?```/;
const MEDICAL_PATTERNS = [/ADHD|多动症|抑郁症|depression|焦虑症|anxiety|自闭|autism|强迫症|OCD|创伤后|PTSD|双相|bipolar/i];
const PERSONALITY_PATTERNS = [/你是一个?.*的人/, /你的性格/, /你天生/, /你太懒/, /你太笨/, /你不适合/];
@Injectable()
export class LearningAnalysisValidator {
private readonly logger = new Logger(LearningAnalysisValidator.name);
validate(output: LearningAnalysisResult, snapshot?: LearningAnalysisSnapshot): void {
const violations: string[] = [];
// ── Schema validation ──
if (!output.summary || output.summary.trim().length === 0) violations.push('summary required');
if (output.summary && output.summary.length > 500) violations.push('summary exceeds 500 chars');
if (!Array.isArray(output.strengths)) violations.push('strengths must be array');
else if (output.strengths.length > 3) violations.push(`strengths max 3, got ${output.strengths.length}`);
if (!Array.isArray(output.weaknesses)) violations.push('weaknesses must be array');
else if (output.weaknesses.length > 5) violations.push(`weaknesses max 5, got ${output.weaknesses.length}`);
if (!Array.isArray(output.trends)) violations.push('trends must be array');
else if (output.trends.length > 3) violations.push(`trends max 3, got ${output.trends.length}`);
if (!Array.isArray(output.risks)) violations.push('risks must be array');
else if (output.risks.length > 3) violations.push(`risks max 3, got ${output.risks.length}`);
if (!Array.isArray(output.recommendations)) violations.push('recommendations must be array');
else if (output.recommendations.length === 0) violations.push('recommendations empty');
else if (output.recommendations.length > 5) violations.push(`recommendations max 5, got ${output.recommendations.length}`);
if (typeof output.confidence !== 'number' || output.confidence < 0 || output.confidence > 1) {
violations.push('confidence must be 0-1');
}
if (typeof output.insufficientData !== 'boolean') violations.push('insufficientData must be boolean');
// ── Evidence validation ──
const allEvidenceItems = [
...(output.strengths || []).flatMap(s => s.evidenceRefs || []),
...(output.weaknesses || []).flatMap(w => w.evidenceRefs || []),
...(output.trends || []).flatMap(t => t.evidenceRefs || []),
...(output.risks || []).flatMap(r => r.evidenceRefs || []),
...(output.recommendations || []).flatMap(r => r.evidenceRefs || []),
];
for (const ref of allEvidenceItems) {
if (!VALID_SOURCE_TYPES.includes(ref.sourceType)) {
violations.push(`invalid evidence sourceType: "${ref.sourceType}"`);
}
if (!ref.metricKey || typeof ref.metricKey !== 'string') {
violations.push('evidence missing metricKey');
}
// Validate entityId belongs to snapshot if provided
if (ref.entityId && snapshot) {
const snap = snapshot.snapshot;
const validIds = new Set([
...(snap.knowledgeProgress.weakItems?.map(w => w.id) || []),
]);
// Not in weak items - could be in other lists; skip strict validation for now
}
}
// ── Business validation ──
if (output.insufficientData) {
if (output.confidence > 0.5) {
violations.push('insufficientData=true but confidence > 0.5');
}
}
if (output.strengths?.length > 0 || output.weaknesses?.length > 0) {
for (const s of output.strengths || []) {
if (!s.evidenceRefs || s.evidenceRefs.length === 0) {
violations.push(`strength "${s.title}" has no evidence`);
}
}
for (const w of output.weaknesses || []) {
if (!w.evidenceRefs || w.evidenceRefs.length === 0) {
violations.push(`weakness "${w.title}" has no evidence`);
}
}
}
// ── Content safety ──
const allText = [
output.summary || '',
...(output.strengths || []).flatMap(s => [s.title, s.description]),
...(output.weaknesses || []).flatMap(w => [w.title, w.description]),
...(output.recommendations || []).flatMap(r => [r.title, r.reason]),
];
for (const text of allText) {
if (CODE_BLOCK_PATTERN.test(text)) violations.push('output contains markdown code block');
for (const pat of MEDICAL_PATTERNS) {
if (pat.test(text)) { violations.push('output contains medical diagnosis terms'); break; }
}
for (const pat of PERSONALITY_PATTERNS) {
if (pat.test(text)) { violations.push('output contains personality judgment'); break; }
}
}
if (violations.length > 0) {
this.logger.warn(`Learning analysis validation failed: ${violations.join('; ')}`);
throw new BusinessValidationError(`Validation failed: ${violations.length} violation(s)`, violations);
}
this.logger.log('Learning analysis validation passed');
}
}