feat(M-AI-08-04): learning analysis validator + engine integration
- Schema validation: strengths/weaknesses/trends/risks/recommendations limits - Evidence validation: sourceType enum, metricKey required - Business validation: insufficientData confidence cap, evidence-required conclusions - Content safety: medical diagnosis, personality judgment, markdown code blocks - Engine: validate after executor in learning_analysis dispatch Co-Authored-By: Claude <noreply@anthropic.com>
This commit is contained in:
parent
1bb131a1bc
commit
06a0e21bbf
@ -15,6 +15,7 @@ import { ReviewCardGenerationValidator } from './review-card-generation-validato
|
|||||||
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
||||||
import { QuizGenerationValidator } from './quiz-generation-validator';
|
import { QuizGenerationValidator } from './quiz-generation-validator';
|
||||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||||
|
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||||
import { PrismaService } from '../../infrastructure/database/prisma.service';
|
import { PrismaService } from '../../infrastructure/database/prisma.service';
|
||||||
import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors';
|
import { JobLockConflictError, JobAlreadyTerminalError } from './ai-job.errors';
|
||||||
|
|
||||||
@ -98,6 +99,7 @@ describe('AiJobExecutionEngineImpl', () => {
|
|||||||
{ provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } },
|
{ provide: QuizGenerationExecutor, useValue: { execute: jest.fn() } },
|
||||||
{ provide: QuizGenerationValidator, useValue: { validate: jest.fn() } },
|
{ provide: QuizGenerationValidator, useValue: { validate: jest.fn() } },
|
||||||
{ provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } },
|
{ provide: LearningAnalysisExecutor, useValue: { execute: jest.fn() } },
|
||||||
|
{ provide: LearningAnalysisValidator, useValue: { validate: jest.fn() } },
|
||||||
{ provide: ActiveRecallObservabilityService, useValue: {
|
{ provide: ActiveRecallObservabilityService, useValue: {
|
||||||
incrementUnifiedExecuteSuccess: jest.fn(),
|
incrementUnifiedExecuteSuccess: jest.fn(),
|
||||||
incrementUnifiedExecuteFailed: jest.fn(),
|
incrementUnifiedExecuteFailed: jest.fn(),
|
||||||
|
|||||||
@ -24,7 +24,7 @@ import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-anal
|
|||||||
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
import { QuizGenerationExecutor } from './quiz-generation-executor';
|
||||||
import { QuizGenerationValidator } from './quiz-generation-validator';
|
import { QuizGenerationValidator } from './quiz-generation-validator';
|
||||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||||
import {
|
import {
|
||||||
AiJobExecutionEngine,
|
AiJobExecutionEngine,
|
||||||
EngineJobContext,
|
EngineJobContext,
|
||||||
@ -104,6 +104,7 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
|||||||
private readonly quizExecutor: QuizGenerationExecutor,
|
private readonly quizExecutor: QuizGenerationExecutor,
|
||||||
private readonly quizValidator: QuizGenerationValidator,
|
private readonly quizValidator: QuizGenerationValidator,
|
||||||
private readonly learningAnalysisExecutor: LearningAnalysisExecutor,
|
private readonly learningAnalysisExecutor: LearningAnalysisExecutor,
|
||||||
|
private readonly learningAnalysisValidator: LearningAnalysisValidator,
|
||||||
private readonly observability: ActiveRecallObservabilityService,
|
private readonly observability: ActiveRecallObservabilityService,
|
||||||
private readonly feynmanObs: FeynmanObservabilityService,
|
private readonly feynmanObs: FeynmanObservabilityService,
|
||||||
) {}
|
) {}
|
||||||
@ -278,6 +279,13 @@ export class AiJobExecutionEngineImpl implements AiJobExecutionEngine {
|
|||||||
`Learning Analysis Executor completed: job=${aiJobId} ` +
|
`Learning Analysis Executor completed: job=${aiJobId} ` +
|
||||||
`confidence=${(parsedOutput as any)?.confidence}`,
|
`confidence=${(parsedOutput as any)?.confidence}`,
|
||||||
);
|
);
|
||||||
|
// M-AI-08-04: Output validation
|
||||||
|
try {
|
||||||
|
this.learningAnalysisValidator.validate(parsedOutput as LearningAnalysisResult, snapshot as any);
|
||||||
|
} catch (validationErr: any) {
|
||||||
|
this.logger.warn(`Learning analysis validation failed for job=${aiJobId}: ${validationErr.message}`);
|
||||||
|
throw validationErr;
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
// 默认路径:直接调用 AiGateway(synthetic_job 等)
|
// 默认路径:直接调用 AiGateway(synthetic_job 等)
|
||||||
response = await this.aiGateway.generate(
|
response = await this.aiGateway.generate(
|
||||||
|
|||||||
@ -42,6 +42,7 @@ import { QuizGenerationProjector } from './quiz-generation-projector';
|
|||||||
import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service';
|
import { LearningAnalysisRegistrationService } from './learning-analysis-registration.service';
|
||||||
import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder';
|
import { LearningAnalysisSnapshotBuilder } from './learning-analysis-snapshot-builder';
|
||||||
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
import { LearningAnalysisExecutor } from './learning-analysis-executor';
|
||||||
|
import { LearningAnalysisValidator } from './learning-analysis-validator';
|
||||||
import {
|
import {
|
||||||
FeynmanBusinessValidator,
|
FeynmanBusinessValidator,
|
||||||
FeynmanReferenceValidator,
|
FeynmanReferenceValidator,
|
||||||
@ -92,6 +93,7 @@ import { AppConfigModule } from '../config/config.module';
|
|||||||
LearningAnalysisRegistrationService,
|
LearningAnalysisRegistrationService,
|
||||||
LearningAnalysisSnapshotBuilder,
|
LearningAnalysisSnapshotBuilder,
|
||||||
LearningAnalysisExecutor,
|
LearningAnalysisExecutor,
|
||||||
|
LearningAnalysisValidator,
|
||||||
{ provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any,
|
{ provide: RESULT_PROJECTORS, useFactory: (synthetic: SyntheticResultProjector, activeRecall: ActiveRecallProjector, feynman: FeynmanProjector, reviewCard: ReviewCardGenerationProjector, quiz: QuizGenerationProjector) => [synthetic, activeRecall, feynman, reviewCard, quiz], inject: [SyntheticResultProjector, ActiveRecallProjector, FeynmanProjector, ReviewCardGenerationProjector, QuizGenerationProjector] } as any,
|
||||||
{ provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl },
|
{ provide: AI_JOB_EXECUTION_ENGINE, useExisting: AiJobExecutionEngineImpl },
|
||||||
],
|
],
|
||||||
|
|||||||
118
src/modules/ai-job/learning-analysis-validator.ts
Normal file
118
src/modules/ai-job/learning-analysis-validator.ts
Normal file
@ -0,0 +1,118 @@
|
|||||||
|
import { Injectable, Logger } from '@nestjs/common';
|
||||||
|
import type { LearningAnalysisResult } from '../ai/prompts/schemas/learning-analysis.schema';
|
||||||
|
import { BusinessValidationError, ReferenceValidationError } from './active-recall-validator';
|
||||||
|
import type { LearningAnalysisSnapshot } from './learning-analysis-snapshot-builder';
|
||||||
|
|
||||||
|
export { BusinessValidationError, ReferenceValidationError };
|
||||||
|
|
||||||
|
const VALID_SOURCE_TYPES = ['study_metric', 'review_metric', 'quiz_metric', 'active_recall', 'feynman', 'knowledge_progress'];
|
||||||
|
const VALID_DIRECTIONS = ['improving', 'declining', 'stable'];
|
||||||
|
const VALID_SEVERITIES = ['low', 'medium', 'high'];
|
||||||
|
|
||||||
|
const CODE_BLOCK_PATTERN = /```[\s\S]*?```/;
|
||||||
|
const MEDICAL_PATTERNS = [/ADHD|多动症|抑郁症|depression|焦虑症|anxiety|自闭|autism|强迫症|OCD|创伤后|PTSD|双相|bipolar/i];
|
||||||
|
const PERSONALITY_PATTERNS = [/你是一个?.*的人/, /你的性格/, /你天生/, /你太懒/, /你太笨/, /你不适合/];
|
||||||
|
|
||||||
|
@Injectable()
|
||||||
|
export class LearningAnalysisValidator {
|
||||||
|
private readonly logger = new Logger(LearningAnalysisValidator.name);
|
||||||
|
|
||||||
|
validate(output: LearningAnalysisResult, snapshot?: LearningAnalysisSnapshot): void {
|
||||||
|
const violations: string[] = [];
|
||||||
|
|
||||||
|
// ── Schema validation ──
|
||||||
|
if (!output.summary || output.summary.trim().length === 0) violations.push('summary required');
|
||||||
|
if (output.summary && output.summary.length > 500) violations.push('summary exceeds 500 chars');
|
||||||
|
|
||||||
|
if (!Array.isArray(output.strengths)) violations.push('strengths must be array');
|
||||||
|
else if (output.strengths.length > 3) violations.push(`strengths max 3, got ${output.strengths.length}`);
|
||||||
|
|
||||||
|
if (!Array.isArray(output.weaknesses)) violations.push('weaknesses must be array');
|
||||||
|
else if (output.weaknesses.length > 5) violations.push(`weaknesses max 5, got ${output.weaknesses.length}`);
|
||||||
|
|
||||||
|
if (!Array.isArray(output.trends)) violations.push('trends must be array');
|
||||||
|
else if (output.trends.length > 3) violations.push(`trends max 3, got ${output.trends.length}`);
|
||||||
|
|
||||||
|
if (!Array.isArray(output.risks)) violations.push('risks must be array');
|
||||||
|
else if (output.risks.length > 3) violations.push(`risks max 3, got ${output.risks.length}`);
|
||||||
|
|
||||||
|
if (!Array.isArray(output.recommendations)) violations.push('recommendations must be array');
|
||||||
|
else if (output.recommendations.length === 0) violations.push('recommendations empty');
|
||||||
|
else if (output.recommendations.length > 5) violations.push(`recommendations max 5, got ${output.recommendations.length}`);
|
||||||
|
|
||||||
|
if (typeof output.confidence !== 'number' || output.confidence < 0 || output.confidence > 1) {
|
||||||
|
violations.push('confidence must be 0-1');
|
||||||
|
}
|
||||||
|
if (typeof output.insufficientData !== 'boolean') violations.push('insufficientData must be boolean');
|
||||||
|
|
||||||
|
// ── Evidence validation ──
|
||||||
|
const allEvidenceItems = [
|
||||||
|
...(output.strengths || []).flatMap(s => s.evidenceRefs || []),
|
||||||
|
...(output.weaknesses || []).flatMap(w => w.evidenceRefs || []),
|
||||||
|
...(output.trends || []).flatMap(t => t.evidenceRefs || []),
|
||||||
|
...(output.risks || []).flatMap(r => r.evidenceRefs || []),
|
||||||
|
...(output.recommendations || []).flatMap(r => r.evidenceRefs || []),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const ref of allEvidenceItems) {
|
||||||
|
if (!VALID_SOURCE_TYPES.includes(ref.sourceType)) {
|
||||||
|
violations.push(`invalid evidence sourceType: "${ref.sourceType}"`);
|
||||||
|
}
|
||||||
|
if (!ref.metricKey || typeof ref.metricKey !== 'string') {
|
||||||
|
violations.push('evidence missing metricKey');
|
||||||
|
}
|
||||||
|
// Validate entityId belongs to snapshot if provided
|
||||||
|
if (ref.entityId && snapshot) {
|
||||||
|
const snap = snapshot.snapshot;
|
||||||
|
const validIds = new Set([
|
||||||
|
...(snap.knowledgeProgress.weakItems?.map(w => w.id) || []),
|
||||||
|
]);
|
||||||
|
// Not in weak items - could be in other lists; skip strict validation for now
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Business validation ──
|
||||||
|
if (output.insufficientData) {
|
||||||
|
if (output.confidence > 0.5) {
|
||||||
|
violations.push('insufficientData=true but confidence > 0.5');
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (output.strengths?.length > 0 || output.weaknesses?.length > 0) {
|
||||||
|
for (const s of output.strengths || []) {
|
||||||
|
if (!s.evidenceRefs || s.evidenceRefs.length === 0) {
|
||||||
|
violations.push(`strength "${s.title}" has no evidence`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const w of output.weaknesses || []) {
|
||||||
|
if (!w.evidenceRefs || w.evidenceRefs.length === 0) {
|
||||||
|
violations.push(`weakness "${w.title}" has no evidence`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ── Content safety ──
|
||||||
|
const allText = [
|
||||||
|
output.summary || '',
|
||||||
|
...(output.strengths || []).flatMap(s => [s.title, s.description]),
|
||||||
|
...(output.weaknesses || []).flatMap(w => [w.title, w.description]),
|
||||||
|
...(output.recommendations || []).flatMap(r => [r.title, r.reason]),
|
||||||
|
];
|
||||||
|
|
||||||
|
for (const text of allText) {
|
||||||
|
if (CODE_BLOCK_PATTERN.test(text)) violations.push('output contains markdown code block');
|
||||||
|
for (const pat of MEDICAL_PATTERNS) {
|
||||||
|
if (pat.test(text)) { violations.push('output contains medical diagnosis terms'); break; }
|
||||||
|
}
|
||||||
|
for (const pat of PERSONALITY_PATTERNS) {
|
||||||
|
if (pat.test(text)) { violations.push('output contains personality judgment'); break; }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (violations.length > 0) {
|
||||||
|
this.logger.warn(`Learning analysis validation failed: ${violations.join('; ')}`);
|
||||||
|
throw new BusinessValidationError(`Validation failed: ${violations.length} violation(s)`, violations);
|
||||||
|
}
|
||||||
|
this.logger.log('Learning analysis validation passed');
|
||||||
|
}
|
||||||
|
}
|
||||||
Loading…
x
Reference in New Issue
Block a user