WF harness

export interface WFHarnessPayloadSpec {
  payloadKey: string;
  requiredKeys: readonly string[];
}

export interface SchedulerStateExpectation {
  conceptId: string;
  path: string;
  expected: unknown;
}

export type SchedulerTransitionStep<TSubskill extends string = string> =
  | {
      kind: 'outcome';
      conceptId: string;
      currentTurn: number;
      outcome: PracticeOutcome;
      subskillUpdates?: readonly SubskillUpdate<TSubskill>[];
    }
  | {
      kind: 'supplemental';
      conceptId: string;
      currentTurn: number;
      wasClean?: boolean;
      subskillUpdates?: readonly SubskillUpdate<TSubskill>[];
    };

export interface SchedulerTransitionScenario<TSubskill extends string = string> {
  name: string;
  initialStored?: unknown;
  steps: readonly SchedulerTransitionStep<TSubskill>[];
  expectations: readonly SchedulerStateExpectation[];
}

export interface SchedulerSelectionScenario<TSubskill extends string = string> {
  name: string;
  initialStored?: unknown;
  steps?: readonly SchedulerTransitionStep<TSubskill>[];
  nextTurn: number;
  expectedConceptId: string;
  eligibleConceptIds?: readonly string[];
}

export interface WFHarnessSchedulerConfig<TSubskill extends string = string> {
  policy?: SchedulerPolicy<TSubskill> | SchedulerPolicyConfig<TSubskill>;
  transitionScenarios?: readonly SchedulerTransitionScenario<TSubskill>[];
  selectionScenarios?: readonly SchedulerSelectionScenario<TSubskill>[];
}

export type QuestionQualityIssueClass =
  | 'context_leakage'
  | 'signal_failure'
  | 'structure_helper_leakage'
  | 'subskill_goal_conflation'
  | 'instruction_validator_divergence'
  | 'distractor_collapse';

export type QuestionQualityVisibleText = Record<string, string | readonly string[] | null | undefined>;

export interface QuestionQualityItem {
  id: string;
  conceptId: string;
  stage?: string;
  targetLayer?: string;
  supportMode?: string;
  visibleText: QuestionQualityVisibleText;
  metadata?: Record<string, unknown>;
}

export type QuestionQualityPredicateResult =
  | string
  | readonly string[]
  | false
  | null
  | undefined;

export type QuestionQualityPatternRule = {
  id: string;
  issueClass: QuestionQualityIssueClass;
  message?: string;
  surfaces?: readonly string[];
  pattern: RegExp;
};

export type QuestionQualityPredicateRule = {
  id: string;
  issueClass: QuestionQualityIssueClass;
  message?: string;
  evaluate: (item: QuestionQualityItem) => QuestionQualityPredicateResult;
};

export type QuestionQualityRule =
  | QuestionQualityPatternRule
  | QuestionQualityPredicateRule;

export type QuestionQualityAppliesTo = (item: QuestionQualityItem) => boolean;

export interface BaseQuestionQualityRuleBuilderOptions {
  id: string;
  appliesTo?: QuestionQualityAppliesTo;
  message?: string;
}

export interface ContextLeakageRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  hasLeakage: (item: QuestionQualityItem) => boolean;
}

export interface SignalFailureRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  hasRequiredSignal: (item: QuestionQualityItem) => boolean;
}

export interface StructureHelperLeakageRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  hasHelperLeakage: (item: QuestionQualityItem) => boolean;
}

export interface SubskillGoalConflationRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  isLocalSubskillCheck: (item: QuestionQualityItem) => boolean;
  reconnectsToGoal: (item: QuestionQualityItem) => boolean;
}

export interface InstructionValidatorDivergenceRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  getInstructionExpectation: (item: QuestionQualityItem) => unknown;
  getValidatorExpectation: (item: QuestionQualityItem) => unknown;
  expectationsMatch?: (
    instructionExpectation: unknown,
    validatorExpectation: unknown,
    item: QuestionQualityItem
  ) => boolean;
}

export interface DistractorCollapseRuleOptions extends BaseQuestionQualityRuleBuilderOptions {
  getChoices: (item: QuestionQualityItem) => string | readonly string[] | null | undefined;
  normalizeChoice?: (choice: string) => string;
  minimumDistinctChoices?: number;
}

export interface WFHarnessQuestionQualityConfig {
  items: readonly QuestionQualityItem[];
  rules: readonly QuestionQualityRule[];
}

export interface WFHarnessConfig<TType extends string = string, TSubskill extends string = string> {
  registeredTypes: readonly TType[];
  renderInteractiveCases: readonly TType[];
  interactivePayloadMap: Partial<Record<TType, WFHarnessPayloadSpec>>;
  questionPool: readonly Question<TType>[];
  conceptTree: readonly ConceptNode[];
  generators: readonly Generator<Question<TType>>[];
  quizClientPath?: string;
  quizClientSource?: string;
  renderPatternFor?: (type: string) => RegExp;
  scheduler?: WFHarnessSchedulerConfig<TSubskill>;
  questionQuality?: WFHarnessQuestionQualityConfig;
}

export interface ValidationResult {
  group: number;
  name: string;
  passed: boolean;
  failures: string[];
}

export interface ValidationGroup {
  group: number;
  name: string;
  results: ValidationResult[];
}

export const WF_GROUP_NAMES = {
  1: 'Question type coverage',
  2: 'Render dispatch coverage',
  3: 'Interactive payload shape',
  4: 'Boundary check',
  5: 'Concept consistency',
  6: 'Generator determinism',
  7: 'Scheduler coverage',
  8: 'Question quality',
} as const satisfies Record<number, string>;

export const WF_SAMPLE_SEEDS = [1, 42, 100, 2024, 99999] as const;

function escapeRegExp(text: string): string {
  return text.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
}

function defaultRenderPatternFor(type: string): RegExp {
  return new RegExp(`currentQuestion\\.type\\s*===\\s*['"]${escapeRegExp(type)}['"]`);
}

function formatDiagnostic(value: unknown): string {
  if (typeof value === 'string') return value;

  try {
    return JSON.stringify(value);
  } catch {
    return String(value);
  }
}

function isDeepStrictEqual(actual: unknown, expected: unknown): boolean {
  try {
    return JSON.stringify(actual) === JSON.stringify(expected);
  } catch {
    return Object.is(actual, expected);
  }
}

function createResult(
  group: number,
  name: string,
  failures: string[],
  options?: { soft?: boolean }
): ValidationResult {
  return {
    group,
    name,
    passed: options?.soft ? true : failures.length === 0,
    failures,
  };
}

function getByPath(obj: unknown, dotted: string): unknown {
  return dotted.split('.').reduce<unknown>((acc, key) => {
    if (acc && typeof acc === 'object' && key in (acc as Record<string, unknown>)) {
      return (acc as Record<string, unknown>)[key];
    }
    return undefined;
  }, obj);
}

function isNonEmpty(value: unknown): boolean {
  if (value == null) return false;
  if (typeof value === 'string') return value.length > 0;
  if (Array.isArray(value)) return value.length > 0;
  if (typeof value === 'object') return Object.keys(value as object).length > 0;
  if (typeof value === 'number') return true;
  return Boolean(value);
}

function normalizeVisibleText(value: string | readonly string[] | null | undefined): string {
  if (Array.isArray(value)) return value.join('\n');
  return typeof value === 'string' ? value : '';
}

function collectQuestionQualityText(
  item: QuestionQualityItem,
  surfaces?: readonly string[]
): Array<{ surface: string; text: string }> {
  const entries = Object.entries(item.visibleText)
    .filter(([surface]) => !surfaces || surfaces.includes(surface));

  return entries
    .map(([surface, value]) => ({ surface, text: normalizeVisibleText(value) }))
    .filter(({ text }) => text.trim().length > 0);
}

function normalizePredicateResult(result: QuestionQualityPredicateResult): string[] {
  if (!result) return [];
  return typeof result === 'string' ? [result] : [...result];
}

function shouldEvaluateQuestionQualityRule(
  item: QuestionQualityItem,
  appliesTo?: QuestionQualityAppliesTo
): boolean {
  return appliesTo ? appliesTo(item) : true;
}

function withQuestionQualityMessage(
  message: string | undefined,
  detail?: string
): string {
  if (message && detail) return `${message}: ${detail}`;
  return detail ?? message ?? 'rule failed';
}

function formatExpectationForDiagnostic(value: unknown): string {
  if (typeof value === 'string') return value;
  return formatDiagnostic(value);
}

function normalizeQuestionQualityChoices(
  value: string | readonly string[] | null | undefined
): string[] {
  if (Array.isArray(value)) return [...value];
  if (typeof value === 'string') {
    return value
      .split(/\r?\n/)
      .map((choice) => choice.trim())
      .filter((choice) => choice.length > 0);
  }
  return [];
}

export function createContextLeakageRule(
  options: ContextLeakageRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'context_leakage',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;
      return options.hasLeakage(item) ? withQuestionQualityMessage(options.message) : false;
    },
  };
}

export function createSignalFailureRule(
  options: SignalFailureRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'signal_failure',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;
      return options.hasRequiredSignal(item) ? false : withQuestionQualityMessage(options.message);
    },
  };
}

export function createStructureHelperLeakageRule(
  options: StructureHelperLeakageRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'structure_helper_leakage',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;
      return options.hasHelperLeakage(item) ? withQuestionQualityMessage(options.message) : false;
    },
  };
}

export function createSubskillGoalConflationRule(
  options: SubskillGoalConflationRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'subskill_goal_conflation',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;
      if (!options.isLocalSubskillCheck(item)) return false;
      return options.reconnectsToGoal(item) ? false : withQuestionQualityMessage(options.message);
    },
  };
}

export function createInstructionValidatorDivergenceRule(
  options: InstructionValidatorDivergenceRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'instruction_validator_divergence',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;
      const instructionExpectation = options.getInstructionExpectation(item);
      const validatorExpectation = options.getValidatorExpectation(item);
      const matches = options.expectationsMatch
        ? options.expectationsMatch(instructionExpectation, validatorExpectation, item)
        : isDeepStrictEqual(instructionExpectation, validatorExpectation);
      if (matches) return false;

      return withQuestionQualityMessage(
        options.message,
        `instruction=${formatExpectationForDiagnostic(instructionExpectation)} validator=${formatExpectationForDiagnostic(validatorExpectation)}`
      );
    },
  };
}

export function createDistractorCollapseRule(
  options: DistractorCollapseRuleOptions
): QuestionQualityPredicateRule {
  return {
    id: options.id,
    issueClass: 'distractor_collapse',
    evaluate(item) {
      if (!shouldEvaluateQuestionQualityRule(item, options.appliesTo)) return false;

      const choices = normalizeQuestionQualityChoices(options.getChoices(item));
      if (choices.length === 0) return false;

      const normalizeChoice = options.normalizeChoice ?? ((choice: string) => choice.trim().toLowerCase());
      const uniqueChoices = new Set(
        choices
          .map(normalizeChoice)
          .filter((choice) => choice.length > 0)
      );
      const minimumDistinctChoices = options.minimumDistinctChoices ?? choices.length;

      return uniqueChoices.size >= minimumDistinctChoices
        ? false
        : withQuestionQualityMessage(options.message);
    },
  };
}

function getNodeBuiltinModule(specifier: string): unknown {
  const runtimeProcess = (globalThis as {
    process?: {
      getBuiltinModule?: (id: string) => unknown;
    };
  }).process;
  return runtimeProcess?.getBuiltinModule?.(specifier);
}

function readQuizClientSource(quizClientPath: string): string {
  const fsModule = getNodeBuiltinModule('node:fs') as {
    readFileSync?: (filePath: string, encoding: BufferEncoding) => string;
  } | undefined;
  const pathModule = getNodeBuiltinModule('node:path') as {
    resolve?: (...segments: string[]) => string;
  } | undefined;
  const runtimeProcess = (globalThis as {
    process?: {
      cwd?: () => string;
    };
  }).process;

  if (!fsModule?.readFileSync || !pathModule?.resolve || !runtimeProcess?.cwd) {
    throw new Error('quizClientPath source reading requires a Node.js runtime; pass quizClientSource in browser bundles');
  }

  return fsModule.readFileSync(pathModule.resolve(runtimeProcess.cwd(), quizClientPath), 'utf8');
}

export function validateTypeCoverage<TType extends string, TSubskill extends string = string>(
  config: WFHarnessConfig<TType, TSubskill>
): ValidationResult[] {
  const registered = new Set<string>(config.registeredTypes);
  const unknownTypes = Array.from(
    new Set(
      config.questionPool
        .map(question => question.type)
        .filter(type => !registered.has(type))
    )
  ).sort();

  const usedTypes = new Set(config.questionPool.map(question => question.type));
  const unusedTypes = config.registeredTypes.filter(type => !usedTypes.has(type));

  return [
    createResult(
      1,
      'every question type used in the pool is registered',
      unknownTypes.map(type => `questionPool uses unregistered type '${type}'`)
    ),
    createResult(
      1,
      'registered types with zero questions are reported as warnings',
      unusedTypes.map(type => `registeredTypes includes '${type}' but the question pool has no questions of that type`),
      { soft: true }
    ),
  ];
}

export function validateRenderDispatch<TType extends string, TSubskill extends string = string>(
  config: WFHarnessConfig<TType, TSubskill>
): ValidationResult[] {
  if (!config.quizClientPath && config.quizClientSource == null) {
    return [
      createResult(
        2,
        'render dispatch validation is skipped when quizClientPath and quizClientSource are omitted',
        [],
      ),
    ];
  }

  let source: string;
  try {
    source = config.quizClientSource ?? readQuizClientSource(config.quizClientPath ?? '');
  } catch (error) {
    const message = error instanceof Error ? error.message : String(error);
    return [
      createResult(
        2,
        'quiz client source can be read from quizClientPath',
        [`failed to read '${config.quizClientPath}': ${message}`]
      ),
    ];
  }

  const patternFor = config.renderPatternFor ?? defaultRenderPatternFor;
  const missingBranches = config.renderInteractiveCases.filter(
    type => !patternFor(type).test(source)
  );

  const branchRegex = /currentQuestion\.type\s*===\s*['"]([^'"]+)['"]/g;
  const referencedTypes = new Set<string>();
  let match: RegExpExecArray | null = null;
  while ((match = branchRegex.exec(source)) !== null) {
    const capturedType = match[1];
    if (capturedType) referencedTypes.add(capturedType);
  }

  const registered = new Set<string>(config.registeredTypes);
  const strayBranches = Array.from(referencedTypes).filter(type => !registered.has(type)).sort();

  return [
    createResult(
      2,
      'quiz client has a render branch for every configured interactive type',
      missingBranches.map(type => `quiz client is missing a render branch for '${type}'`)
    ),
    createResult(
      2,
      'every currentQuestion.type branch in the quiz client refers to a registered type',
      strayBranches.map(type => `quiz client dispatches unregistered type '${type}'`)
    ),
  ];
}

export function validateInteractivePayloadShape<TType extends string, TSubskill extends string = string>(
  config: WFHarnessConfig<TType, TSubskill>
): ValidationResult[] {
  const results: ValidationResult[] = [];

  for (const [type, spec] of Object.entries(config.interactivePayloadMap) as Array<
    [TType, WFHarnessPayloadSpec | undefined]
  >) {
    if (!spec) continue;

    const questionsOfType = config.questionPool.filter(question => question.type === type);
    if (questionsOfType.length === 0) continue;

    const missingPayload = questionsOfType
      .filter(question => question.interactive?.[spec.payloadKey] == null)
      .map(question => `${question.id}: missing interactive.${spec.payloadKey}`);

    const missingKeys: string[] = [];
    for (const question of questionsOfType) {
      const payload = question.interactive?.[spec.payloadKey];
      if (payload == null) continue;

      const brokenKeys = spec.requiredKeys.filter(key => !isNonEmpty(getByPath(payload, key)));
      if (brokenKeys.length > 0) {
        missingKeys.push(
          `${question.id}: interactive.${spec.payloadKey} is missing ${brokenKeys.join(', ')}`
        );
      }
    }

    results.push(
      createResult(
        3,
        `every ${type} question has interactive.${spec.payloadKey} populated`,
        missingPayload
      ),
      createResult(
        3,
        `every ${type} payload includes all required keys`,
        missingKeys
      )
    );
  }

  if (results.length > 0) {
    return results;
  }

  return [
    createResult(
      3,
      'interactive payload checks are skipped when no interactive payload specs apply to the current pool',
      []
    ),
  ];
}

export function validateBoundaryCheck<TType extends string, TSubskill extends string = string>(
  config: WFHarnessConfig<TType, TSubskill>
): ValidationResult[] {
  const payloadKeys = Array.from(
    new Set(
      Object.values(config.interactivePayloadMap)
        .filter((spec): spec is WFHarnessPayloadSpec => spec != null)
        .map(spec => spec.payloadKey)
    )
  );

  const dispatchTypes = new Set<string>(config.renderInteractiveCases);
  const interactiveTypeFailures: string[] = [];

  for (const question of config.questionPool) {
    if (!dispatchTypes.has(question.type)) continue;

    const spec = config.interactivePayloadMap[question.type];
    if (!spec) {
      interactiveTypeFailures.push(
        `${question.id}: renderInteractiveCases includes '${question.type}' but interactivePayloadMap has no matching payload spec`
      );
      continue;
    }

    const presentPayloads = payloadKeys.filter(payloadKey => question.interactive?.[payloadKey] != null);
    if (presentPayloads.length !== 1 || presentPayloads[0] !== spec.payloadKey) {
      interactiveTypeFailures.push(
        `${question.id}: expected only ${spec.payloadKey}, found [${presentPayloads.join(', ')}]`
      );
    }
  }

  const nonDispatchFailures: string[] = [];
  for (const question of config.questionPool) {
    if (dispatchTypes.has(question.type)) continue;

    const strayPayloads = payloadKeys.filter(payloadKey => question.interactive?.[payloadKey] != null);
    if (strayPayloads.length > 0) {
      nonDispatchFailures.push(
        `${question.id}: non-dispatch type '${question.type}' carries [${strayPayloads.join(', ')}]`
      );
    }
  }

  return [
    createResult(
      4,
      'interactive-dispatch questions carry exactly one matching payload',
      interactiveTypeFailures
    ),
    createResult(
      4,
      'non-dispatch questions carry none of the interactive payload keys',
      nonDispatchFailures
    ),
  ];
}

export function validateConceptConsistency<TType extends string, TSubskill extends string = string>(
  config: WFHarnessConfig<TType, TSubskill>
): ValidationResult[] {
  const idCounts = new Map<string, number>();
  for (const question of config.questionPool) {
    idCounts.set(question.id, (idCounts.get(question.id) ?? 0) + 1);
  }

  const duplicateIds = Array.from(idCounts.entries())
    .filter(([, count]) => count > 1)
    .map(([id]) => `duplicate question id '${id}'`)
    .sort();

  const missingFieldFailures: string[] = [];
  for (const question of config.questionPool) {
    const missingFields: string[] = [];
    if (!isNonEmpty(question.id)) missingFields.push('id');
    if (!isNonEmpty(question.type)) missingFields.push('type');
    if (!isNonEmpty(question.concept)) missingFields.push('concept');
    if (!isNonEmpty(question.question)) missingFields.push('question');
    if (!isNonEmpty(question.correctAnswer)) missingFields.push('correctAnswer');

    if (missingFields.length > 0) {
      missingFieldFailures.push(
        `${question.id || '(unknown)'}: missing ${missingFields.join(', ')}`
      );
    }
  }

  const conceptIds = new Set(config.conceptTree.map(concept => concept.id));
  const orphanConcepts = config.questionPool
    .filter(question => !conceptIds.has(question.concept))
    .map(question => `${question.id}: concept '${question.concept}' does not exist in conceptTree`);

  return [
    createResult(5, 'all question IDs are unique', duplicateIds),
    createResult(
      5,
      'every question has the required fields: id, type, concept, question, correctAnswer',
      missingFieldFailures
    ),
    createResult(
      5,
      'every question.concept resolves to a ConceptNode.id in conceptTree',
      orphanConcepts
    ),
  ];
}