feat(M-AI-08-04): learning analysis validator + engine integration
- Schema validation: strengths/weaknesses/trends/risks/recommendations limits - Evidence validation: sourceType enum, metricKey required - Business validation: insufficientData confidence cap, evidence-required conclusions - Content safety: medical diagnosis, personality judgment, markdown code blocks - Engine: validate after executor in learning_analysis dispatch Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
1bb131a1bc
commit
06a0e21bbf
@ -15,6 +15,7 @@ import { ReviewCardGenerationValidator } from './review-card-generation-validato
|
||||
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
||||
import { QuizGenerationValidator } from './quiz-generation-validator';
|
||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||
import { PrismaService } from '../../infrastructure/database/prisma.service';
|
||||
import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors';
|
||||
|
||||
@ -98,6 +99,7 @@ describe('AiJobExecutionEngineImpl', () => {
|
||||
{ provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } },
|
||||
{ provide: QuizGenerationValidator, useValue: { validate: jest.fn() } },
|
||||
{ provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } },
|
||||
{ provide: LearningAnalysisValidator, useValue: { validate: jest.fn() } },
|
||||
{ provide: ActiveRecallObservabilityService, useValue: {
|
||||
incrementUnifiedExecuteSuccess: jest.fn(),
|
||||
incrementUnifiedExecuteFailed: jest.fn(),
|
||||
|
||||
@ -24,7 +24,7 @@ import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-anal
|
||||
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
||||
import { QuizGenerationValidator } from './quiz-generation-validator';
|
||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||
import {
|
||||
AiJobExecutionEngine,
|
||||
EngineJobContext,
|
||||
@ -104,6 +104,7 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
||||
private readonly quizExecutor: QuizGenerationExecutor,
|
||||
private readonly quizValidator: QuizGenerationValidator,
|
||||
private readonly learningAnalysisExecutor: LearningAnalysisExecutor,
|
||||
private readonly learningAnalysisValidator: LearningAnalysisValidator,
|
||||
private readonly observability: ActiveRecallObservabilityService,
|
||||
private readonly feynmanObs: FeynmanObservabilityService,
|
||||
) {}
|
||||
@ -278,6 +279,13 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
||||
`Learning Analysis Executor completed: job=${aiJobId} ` +
|
||||
`confidence=${(parsedOutput as any)?.confidence}`,
|
||||
);
|
||||
// M-AI-08-04: Output validation
|
||||
try {
|
||||
this.learningAnalysisValidator.validate(parsedOutput as LearningAnalysisResult, snapshot as any);
|
||||
} catch (validationErr: any) {
|
||||
this.logger.warn(`Learning analysis validation failed for job=${aiJobId}: ${validationErr.message}`);
|
||||
throw validationErr;
|
||||
}
|
||||
} else {
|
||||
// 默认路径:直接调用 AiGateway(synthetic_job 等)
|
||||
response = await this.aiGateway.generate(
|
||||
|
||||
@ -42,6 +42,7 @@ import { QuizGenerationProjector } from './quiz-generation-projector';
|
||||
import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service';
|
||||
import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder';
|
||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||
import {
|
||||
FeynmanBusinessValidator,
|
||||
FeynmanReferenceValidator,
|
||||
@ -92,6 +93,7 @@ import { AppConfigModule } from '../config/config.module';
|
||||
LearningAnalysisRegistrationService,
|
||||
LearningAnalysisSnapshotBuilder,
|
||||
LearningAnalysisExecutor,
|
||||
LearningAnalysisValidator,
|
||||
{ provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any,
|
||||
{ provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl },
|
||||
],
|
||||
|
||||
118
src/modules/ai-job/learning-analysis-validator.ts
Normal file
118
src/modules/ai-job/learning-analysis-validator.ts
Normal file
@ -0,0 +1,118 @@
|
||||
import { Injectable, Logger } from '@nestjs/common';
|
||||
import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-analysis.schema';
|
||||
import { BusinessValidationError, ReferenceValidationError } from './active-recall-validator';
|
||||
import type { LearningAnalysisSnapshot } from './learning-analysis-snapshot-builder';
|
||||
|
||||
export { BusinessValidationError, ReferenceValidationError };
|
||||
|
||||
const VALID_SOURCE_TYPES = ['study_metric', 'review_metric', 'quiz_metric', 'active_recall', 'feynman', 'knowledge_progress'];
|
||||
const VALID_DIRECTIONS = ['improving', 'declining', 'stable'];
|
||||
const VALID_SEVERITIES = ['low', 'medium', 'high'];
|
||||
|
||||
const CODE_BLOCK_PATTERN = /```[\s\S]*?```/;
|
||||
const MEDICAL_PATTERNS = [/ADHD|多动症|抑郁症|depression|焦虑症|anxiety|自闭|autism|强迫症|OCD|创伤后|PTSD|双相|bipolar/i];
|
||||
const PERSONALITY_PATTERNS = [/你是一个?.*的人/, /你的性格/, /你天生/, /你太懒/, /你太笨/, /你不适合/];
|
||||
|
||||
@Injectable()
|
||||
export class LearningAnalysisValidator {
|
||||
private readonly logger = new Logger(LearningAnalysisValidator.name);
|
||||
|
||||
validate(output: LearningAnalysisResult, snapshot?: LearningAnalysisSnapshot): void {
|
||||
const violations: string[] = [];
|
||||
|
||||
// ── Schema validation ──
|
||||
if (!output.summary || output.summary.trim().length === 0) violations.push('summary required');
|
||||
if (output.summary && output.summary.length > 500) violations.push('summary exceeds 500 chars');
|
||||
|
||||
if (!Array.isArray(output.strengths)) violations.push('strengths must be array');
|
||||
else if (output.strengths.length > 3) violations.push(`strengths max 3, got ${output.strengths.length}`);
|
||||
|
||||
if (!Array.isArray(output.weaknesses)) violations.push('weaknesses must be array');
|
||||
else if (output.weaknesses.length > 5) violations.push(`weaknesses max 5, got ${output.weaknesses.length}`);
|
||||
|
||||
if (!Array.isArray(output.trends)) violations.push('trends must be array');
|
||||
else if (output.trends.length > 3) violations.push(`trends max 3, got ${output.trends.length}`);
|
||||
|
||||
if (!Array.isArray(output.risks)) violations.push('risks must be array');
|
||||
else if (output.risks.length > 3) violations.push(`risks max 3, got ${output.risks.length}`);
|
||||
|
||||
if (!Array.isArray(output.recommendations)) violations.push('recommendations must be array');
|
||||
else if (output.recommendations.length === 0) violations.push('recommendations empty');
|
||||
else if (output.recommendations.length > 5) violations.push(`recommendations max 5, got ${output.recommendations.length}`);
|
||||
|
||||
if (typeof output.confidence !== 'number' || output.confidence < 0 || output.confidence > 1) {
|
||||
violations.push('confidence must be 0-1');
|
||||
}
|
||||
if (typeof output.insufficientData !== 'boolean') violations.push('insufficientData must be boolean');
|
||||
|
||||
// ── Evidence validation ──
|
||||
const allEvidenceItems = [
|
||||
...(output.strengths || []).flatMap(s => s.evidenceRefs || []),
|
||||
...(output.weaknesses || []).flatMap(w => w.evidenceRefs || []),
|
||||
...(output.trends || []).flatMap(t => t.evidenceRefs || []),
|
||||
...(output.risks || []).flatMap(r => r.evidenceRefs || []),
|
||||
...(output.recommendations || []).flatMap(r => r.evidenceRefs || []),
|
||||
];
|
||||
|
||||
for (const ref of allEvidenceItems) {
|
||||
if (!VALID_SOURCE_TYPES.includes(ref.sourceType)) {
|
||||
violations.push(`invalid evidence sourceType: "${ref.sourceType}"`);
|
||||
}
|
||||
if (!ref.metricKey || typeof ref.metricKey !== 'string') {
|
||||
violations.push('evidence missing metricKey');
|
||||
}
|
||||
// Validate entityId belongs to snapshot if provided
|
||||
if (ref.entityId && snapshot) {
|
||||
const snap = snapshot.snapshot;
|
||||
const validIds = new Set([
|
||||
...(snap.knowledgeProgress.weakItems?.map(w => w.id) || []),
|
||||
]);
|
||||
// Not in weak items - could be in other lists; skip strict validation for now
|
||||
}
|
||||
}
|
||||
|
||||
// ── Business validation ──
|
||||
if (output.insufficientData) {
|
||||
if (output.confidence > 0.5) {
|
||||
violations.push('insufficientData=true but confidence > 0.5');
|
||||
}
|
||||
}
|
||||
|
||||
if (output.strengths?.length > 0 || output.weaknesses?.length > 0) {
|
||||
for (const s of output.strengths || []) {
|
||||
if (!s.evidenceRefs || s.evidenceRefs.length === 0) {
|
||||
violations.push(`strength "${s.title}" has no evidence`);
|
||||
}
|
||||
}
|
||||
for (const w of output.weaknesses || []) {
|
||||
if (!w.evidenceRefs || w.evidenceRefs.length === 0) {
|
||||
violations.push(`weakness "${w.title}" has no evidence`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ── Content safety ──
|
||||
const allText = [
|
||||
output.summary || '',
|
||||
...(output.strengths || []).flatMap(s => [s.title, s.description]),
|
||||
...(output.weaknesses || []).flatMap(w => [w.title, w.description]),
|
||||
...(output.recommendations || []).flatMap(r => [r.title, r.reason]),
|
||||
];
|
||||
|
||||
for (const text of allText) {
|
||||
if (CODE_BLOCK_PATTERN.test(text)) violations.push('output contains markdown code block');
|
||||
for (const pat of MEDICAL_PATTERNS) {
|
||||
if (pat.test(text)) { violations.push('output contains medical diagnosis terms'); break; }
|
||||
}
|
||||
for (const pat of PERSONALITY_PATTERNS) {
|
||||
if (pat.test(text)) { violations.push('output contains personality judgment'); break; }
|
||||
}
|
||||
}
|
||||
|
||||
if (violations.length > 0) {
|
||||
this.logger.warn(`Learning analysis validation failed: ${violations.join('; ')}`);
|
||||
throw new BusinessValidationError(`Validation failed: ${violations.length} violation(s)`, violations);
|
||||
}
|
||||
this.logger.log('Learning analysis validation passed');
|
||||
}
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user