Press n or j to go to the next uncovered block, b, p or k for the previous block.
| 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737 738 739 740 741 742 743 744 745 746 747 748 749 750 751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788 789 790 791 792 793 794 795 796 797 798 799 800 801 802 803 804 805 806 807 808 809 810 811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879 880 881 882 883 884 885 886 887 888 889 890 891 892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919 920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963 964 965 966 967 968 969 970 971 972 973 974 975 976 977 | /** * Flowchart Doctor - Validates flowchart definitions and identifies issues * * Provides structured error reporting with provenance information * to help identify exactly where problems exist in flowchart configurations. */ import type { FlowchartDefinition, TransformExpression } from './schema' import { parseMermaidFile } from './parser' import { evaluate, type EvalContext } from './evaluator' // ============================================================================= // Types // ============================================================================= export type DiagnosticSeverity = 'error' | 'warning' | 'info' export interface DiagnosticLocation { /** Top-level section of the flowchart definition */ section: | 'problemInput' | 'variables' | 'nodes' | 'generation' | 'constraints' | 'display' | 'mermaid' | 'transform' | 'answer' | 'tests' /** Specific field path within the section (e.g., "derived.answer" or "fields[0]") */ path?: string /** Human-readable description of the location */ description: string } export interface FlowchartDiagnostic { /** Unique code for this diagnostic type */ code: string /** Severity level */ severity: DiagnosticSeverity /** Short summary of the issue */ title: string /** Detailed explanation of the problem */ message: string /** Where in the flowchart the issue was found */ location: DiagnosticLocation /** Optional suggestion for how to fix the issue */ suggestion?: string } export interface DiagnosticReport { /** Whether the flowchart passed all checks */ isHealthy: boolean /** Number of errors found */ errorCount: number /** Number of warnings found */ warningCount: number /** All diagnostics found */ diagnostics: FlowchartDiagnostic[] } // ============================================================================= // Diagnostic Codes // ============================================================================= export const DiagnosticCodes = { // Derived field issues (DRV-xxx) DERIVED_REFERENCES_VARIABLE: 'DRV-001', DERIVED_REFERENCES_UNKNOWN: 'DRV-002', DERIVED_REFERENCES_LATER: 'DRV-003', // Generation config issues (GEN-xxx) GENERATION_MISSING_PREFERRED: 'GEN-001', GENERATION_MISSING_FIELD: 'GEN-002', // Node issues (NODE-xxx) NODE_MISSING_NEXT: 'NODE-001', NODE_INVALID_REFERENCE: 'NODE-002', // Variable issues (VAR-xxx) - DEPRECATED, use transform instead VARIABLE_INVALID_EXPRESSION: 'VAR-001', // Transform issues (XFORM-xxx) TRANSFORM_CIRCULAR_REFERENCE: 'XFORM-001', TRANSFORM_UNDEFINED_REFERENCE: 'XFORM-002', TRANSFORM_SYNTAX_ERROR: 'XFORM-003', // Answer definition issues (ANS-xxx) ANSWER_MISSING_DEFINITION: 'ANS-001', ANSWER_INVALID_REFERENCE: 'ANS-002', ANSWER_TEMPLATE_ERROR: 'ANS-003', // Display issues (DISP-xxx) DISPLAY_MISSING_ANSWER: 'DISP-001', DISPLAY_DIVISION_WITHOUT_HANDLER: 'DISP-002', // Mermaid issues (MERM-xxx) MERMAID_ESCAPED_QUOTES: 'MERM-001', MERMAID_NODE_MISMATCH: 'MERM-002', // Test coverage issues (TEST-xxx) TEST_NO_EXPECTED_ANSWERS: 'TEST-001', TEST_INCOMPLETE_COVERAGE: 'TEST-002', TEST_FAILING: 'TEST-003', TEST_EXPECTED_KEYS_MISMATCH: 'TEST-004', } as const // ============================================================================= // Built-in Functions (shared with evaluator) // ============================================================================= const BUILT_IN_FUNCTIONS = new Set([ 'true', 'false', 'null', 'Math', 'abs', 'floor', 'ceil', 'round', 'min', 'max', 'gcd', 'lcm', 'sign', 'mod', 'pow', 'sqrt', ]) // ============================================================================= // Helper Functions // ============================================================================= /** * Extract variable references from an expression. * Returns identifiers that are NOT function calls, property access, built-ins, or string literals. */ function extractVariableReferences(expr: string): string[] { const refs: string[] = [] // First, remove string literals from the expression to avoid matching identifiers inside them // This handles both single and double quoted strings const exprWithoutStrings = expr .replace(/"(?:[^"\\]|\\.)*"/g, '""') // Remove double-quoted strings .replace(/'(?:[^'\\]|\\.)*'/g, "''") // Remove single-quoted strings const identifierPattern = /[a-zA-Z_][a-zA-Z0-9_]*/g let match while ((match = identifierPattern.exec(exprWithoutStrings)) !== null) { const identifier = match[0] const startIndex = match.index // Skip if preceded by a dot (property access like obj.prop) if (startIndex > 0 && exprWithoutStrings[startIndex - 1] === '.') { continue } // Skip if followed by open paren (function call like func()) const afterMatch = exprWithoutStrings.slice(startIndex + identifier.length) if (/^\s*\(/.test(afterMatch)) { continue } // Skip built-ins if (BUILT_IN_FUNCTIONS.has(identifier)) { continue } refs.push(identifier) } return refs } // ============================================================================= // Validation Checks // ============================================================================= /** * Check derived fields for invalid references */ function checkDerivedFields(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] const derived = definition.generation?.derived if (!derived || Object.keys(derived).length === 0) { return diagnostics } // Collect input field names const inputFieldNames = new Set(definition.problemInput.fields.map((f) => f.name)) // Collect variable names (these are NOT allowed in derived expressions) const variableNames = new Set(Object.keys(definition.variables || {})) // Track derived field names as we process them const processedDerivedNames = new Set<string>() for (const [fieldName, expression] of Object.entries(derived)) { const references = extractVariableReferences(expression) for (const identifier of references) { const isInputField = inputFieldNames.has(identifier) const isPreviousDerived = processedDerivedNames.has(identifier) const isVariable = variableNames.has(identifier) const isLaterDerived = !isPreviousDerived && identifier in derived if (!isInputField && !isPreviousDerived) { if (isVariable) { // References a computed variable - this is the main bug we're catching diagnostics.push({ code: DiagnosticCodes.DERIVED_REFERENCES_VARIABLE, severity: 'error', title: 'Derived field references computed variable', message: `The derived field "${fieldName}" references "${identifier}" which is a computed variable defined in the variables section. Derived fields are computed during example generation, before the flowchart executes, so they cannot access computed variables.`, location: { section: 'generation', path: `derived.${fieldName}`, description: `generation.derived["${fieldName}"]`, }, suggestion: `Either move the computation to the variables section, or rewrite the expression to only use input fields: ${[...inputFieldNames].join(', ')}`, }) } else if (isLaterDerived) { // Check if it's a self-reference (circular) const isSelfReference = identifier === fieldName diagnostics.push({ code: DiagnosticCodes.DERIVED_REFERENCES_LATER, severity: 'error', title: isSelfReference ? 'Derived field references itself (circular)' : 'Derived field references later-defined field', message: isSelfReference ? `The derived field "${fieldName}" references itself. This creates a circular reference that cannot be computed.` : `The derived field "${fieldName}" references "${identifier}" which is defined later in the derived fields list. Derived fields are processed in order, so earlier fields cannot reference later ones.`, location: { section: 'generation', path: `derived.${fieldName}`, description: `generation.derived["${fieldName}"]`, }, suggestion: isSelfReference ? `Replace the self-reference with the actual computation. For example, if "${fieldName}" should be computed from input fields, write the formula instead of just "${fieldName}".` : `Reorder the derived fields so that "${identifier}" is defined before "${fieldName}"`, }) } else { // References an unknown identifier diagnostics.push({ code: DiagnosticCodes.DERIVED_REFERENCES_UNKNOWN, severity: 'error', title: 'Derived field references unknown identifier', message: `The derived field "${fieldName}" references "${identifier}" which is not a known input field or previously-defined derived field.`, location: { section: 'generation', path: `derived.${fieldName}`, description: `generation.derived["${fieldName}"]`, }, suggestion: `Valid references are input fields (${[...inputFieldNames].join(', ')}) and previously-defined derived fields (${[...processedDerivedNames].join(', ') || 'none yet'})`, }) } } } processedDerivedNames.add(fieldName) } return diagnostics } /** * Check for missing answer definition. * * All flowcharts MUST have an `answer` definition that specifies * how to extract and display the final answer. */ function checkDisplayAnswer(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] if (!definition.answer) { diagnostics.push({ code: DiagnosticCodes.DISPLAY_MISSING_ANSWER, severity: 'error', title: 'Missing answer definition', message: 'This flowchart has no answer definition. Worksheets and PDFs will show "?" for all answers.', location: { section: 'answer', path: '', description: 'answer (missing)', }, suggestion: 'Add an "answer" definition with values and display templates. Example: "answer": { "values": { "result": "finalAnswer" }, "display": { "text": "{{result}}" } }', }) } return diagnostics } /** * Check that generation.preferred has entries for all input fields */ function checkGenerationPreferred(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] const preferred = definition.generation?.preferred const fields = definition.problemInput.fields if (!preferred) { // No generation config at all - this might be intentional for some flowcharts return diagnostics } const preferredFieldNames = new Set(Object.keys(preferred)) for (const field of fields) { if (!preferredFieldNames.has(field.name)) { diagnostics.push({ code: DiagnosticCodes.GENERATION_MISSING_FIELD, severity: 'warning', title: 'Missing preferred values for input field', message: `The input field "${field.name}" does not have preferred values defined in generation.preferred. Example generation may produce less pedagogically useful values.`, location: { section: 'generation', path: 'preferred', description: `generation.preferred (missing "${field.name}")`, }, suggestion: `Add preferred values: { "key": "${field.name}", "values": [...] }`, }) } } return diagnostics } /** * Generate a context-aware suggestion for division-by-zero handling. * * Tries to detect patterns like: * - Fraction numerator/denominator pairs (simpNum/simpDenom, answerNum/answerDenom) * - Single value divisions (x = a / b) * * Returns a suggestion using the actual variable names from the flowchart. */ function generateDivisionSuggestion( target: string, expr: string, variables: Record<string, { init: string }> ): string { // Pattern 1: Fraction - target ends in "Num" and there's a matching "Denom" // e.g., simpNum -> simpDenom, answerNum -> answerDenom if (target.endsWith('Num')) { const prefix = target.slice(0, -3) // Remove "Num" const denomVar = `${prefix}Denom` if (denomVar in variables) { return `This looks like a fraction answer. Add display.answer to show the full fraction: "display": { "answer": "${target} + '/' + ${denomVar}" }` } } // Pattern 2: Try to extract the divisor from the expression // Match patterns like "foo / bar" or "foo/bar" const divisionMatch = expr.match(/(\w+)\s*\/\s*(\w+)/) if (divisionMatch) { const divisor = divisionMatch[2] return `Add a display.answer expression that handles when ${divisor} is zero: "display": { "answer": "${divisor} != 0 ? ${target} : 'undefined'" }` } // Fallback: generic suggestion using the actual target variable return `Add a display.answer expression that handles the division-by-zero case: "display": { "answer": "${target}" } Or if you need to handle the zero case: "display": { "answer": "divisor != 0 ? ${target} : 'undefined'" }` } /** * Check for potential division by zero in generation.target without display.answer. * * When generation.target points to a variable that involves division, * the answer can become NaN if the divisor is zero. Without a display.answer * expression to handle this case, worksheets will show "NaN" for such problems. */ function checkDivisionInTarget(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] const target = definition.generation?.target if (!target) { return diagnostics } // If display.answer exists, assume it handles edge cases if (definition.display?.answer) { return diagnostics } // Get the target variable's init expression const variables = definition.variables || {} const targetVar = variables[target] if (!targetVar) { return diagnostics } // Check if the init expression contains division // Simple heuristic: look for / that's not inside a comment or string const expr = targetVar.init if (expr.includes('/')) { // More precise check: ensure it's actually a division operator // Skip if it's just part of a string like '/' or a comment const hasDivision = /[^'"]\/[^'"]/.test(expr) || /^\s*[^'"]*\//.test(expr) if (hasDivision) { // Generate a context-aware suggestion based on the actual variables // Check if this looks like a fraction (target ends in Num and there's a matching Denom) const suggestion = generateDivisionSuggestion(target, expr, variables) diagnostics.push({ code: DiagnosticCodes.DISPLAY_DIVISION_WITHOUT_HANDLER, severity: 'warning', title: 'Answer variable contains division without display.answer handler', message: `The generation.target "${target}" is computed using division (${expr}). When the divisor is zero, this produces NaN. Without a display.answer expression to handle this case, worksheets will show "NaN" for affected problems.`, location: { section: 'display', path: 'answer', description: `generation.target "${target}" → variables["${target}"].init`, }, suggestion, }) } } return diagnostics } // ============================================================================= // Transform Checks (New Unified Computation Model) // ============================================================================= /** * Check node transforms for issues like circular references, undefined variables, and syntax errors. */ function checkNodeTransforms(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] // Collect all input field names const inputFieldNames = new Set(definition.problemInput.fields.map((f) => f.name)) // Collect all variable names (from deprecated variables section) const variableNames = new Set(Object.keys(definition.variables || {})) // Track all transform keys that will be available (accumulated across all nodes) // This is an approximation since we don't know the actual walk order const allTransformKeys = new Set<string>() // First pass: collect all transform keys for (const [_nodeId, node] of Object.entries(definition.nodes)) { const transforms = node.transform || [] for (const transform of transforms) { allTransformKeys.add(transform.key) } } // Second pass: check each node's transforms for (const [nodeId, node] of Object.entries(definition.nodes)) { const transforms = node.transform || [] const localKeys = new Set<string>() for (let i = 0; i < transforms.length; i++) { const transform = transforms[i] // Check for circular reference (key references itself in expr) const refs = extractVariableReferences(transform.expr) if (refs.includes(transform.key) && !localKeys.has(transform.key)) { diagnostics.push({ code: DiagnosticCodes.TRANSFORM_CIRCULAR_REFERENCE, severity: 'error', title: 'Circular reference in transform', message: `Transform "${transform.key}" references itself in its expression "${transform.expr}". This creates a circular dependency.`, location: { section: 'transform', path: `nodes.${nodeId}.transform[${i}]`, description: `Node "${nodeId}" transform[${i}]: ${transform.key}`, }, suggestion: `Remove the self-reference or define "${transform.key}" in a previous transform before referencing it.`, }) } // Check for undefined references for (const ref of refs) { const isInput = inputFieldNames.has(ref) const isVariable = variableNames.has(ref) const isLocalPrevious = localKeys.has(ref) const isGlobalTransform = allTransformKeys.has(ref) // Allow: input fields, legacy variables, previously defined local keys, or any global transform key if (!isInput && !isVariable && !isLocalPrevious && !isGlobalTransform) { diagnostics.push({ code: DiagnosticCodes.TRANSFORM_UNDEFINED_REFERENCE, severity: 'warning', title: 'Transform may reference undefined variable', message: `Transform "${transform.key}" in node "${nodeId}" references "${ref}" which is not a known input field, variable, or transform key.`, location: { section: 'transform', path: `nodes.${nodeId}.transform[${i}]`, description: `Node "${nodeId}" transform[${i}]: ${transform.key}`, }, suggestion: `Ensure "${ref}" is defined before this node is reached during the walk. Known inputs: ${[...inputFieldNames].join(', ')}`, }) } } // Check for syntax errors by trying to parse the expression try { // Create a mock context to test expression parsing const mockContext: EvalContext = { problem: {}, computed: {}, userState: {}, } // We can't fully evaluate, but we can check for basic parsing // The evaluate function will throw on syntax errors evaluate(transform.expr, mockContext) } catch (err) { // Only report if it looks like a syntax error, not a reference error const errorMsg = err instanceof Error ? err.message : String(err) if ( errorMsg.includes('syntax') || errorMsg.includes('Unexpected') || errorMsg.includes('Invalid') ) { diagnostics.push({ code: DiagnosticCodes.TRANSFORM_SYNTAX_ERROR, severity: 'error', title: 'Transform expression syntax error', message: `Transform "${transform.key}" in node "${nodeId}" has a syntax error: ${errorMsg}`, location: { section: 'transform', path: `nodes.${nodeId}.transform[${i}]`, description: `Node "${nodeId}" transform[${i}]: ${transform.key}`, }, suggestion: `Fix the expression syntax: "${transform.expr}"`, }) } } localKeys.add(transform.key) } } return diagnostics } /** * Check the answer definition for issues. */ function checkAnswerDefinition(definition: FlowchartDefinition): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] const answerDef = definition.answer // If no answer definition, check for legacy display.answer if (!answerDef) { // Legacy mode - checkDisplayAnswer will handle this return diagnostics } // Collect all possible computed values (input fields + variables + all transform keys) const inputFieldNames = new Set(definition.problemInput.fields.map((f) => f.name)) const variableNames = new Set(Object.keys(definition.variables || {})) const allTransformKeys = new Set<string>() for (const [_nodeId, node] of Object.entries(definition.nodes)) { const transforms = node.transform || [] for (const transform of transforms) { allTransformKeys.add(transform.key) } } // Check that answer.values references valid computed values for (const [name, ref] of Object.entries(answerDef.values)) { const isInput = inputFieldNames.has(ref) const isVariable = variableNames.has(ref) const isTransform = allTransformKeys.has(ref) if (!isInput && !isVariable && !isTransform) { diagnostics.push({ code: DiagnosticCodes.ANSWER_INVALID_REFERENCE, severity: 'error', title: 'Answer references unknown value', message: `Answer value "${name}" references "${ref}" which is not a known input field, variable, or transform key.`, location: { section: 'answer', path: `values.${name}`, description: `answer.values["${name}"]`, }, suggestion: `Ensure "${ref}" is computed during the walk. Available: inputs (${[...inputFieldNames].join(', ')}), transforms (${[...allTransformKeys].slice(0, 5).join(', ')}${allTransformKeys.size > 5 ? '...' : ''})`, }) } } // Check display templates for interpolation issues const templates = [ { name: 'text', value: answerDef.display.text }, { name: 'web', value: answerDef.display.web }, { name: 'typst', value: answerDef.display.typst }, ] for (const template of templates) { if (!template.value) continue // Check for {{variable}} references const matches = template.value.matchAll(/\{\{([^}]+)\}\}/g) for (const match of matches) { const content = match[1].trim() // If it starts with =, it's an expression - skip detailed checking if (content.startsWith('=')) continue // It's a variable reference const isKnown = inputFieldNames.has(content) || variableNames.has(content) || allTransformKeys.has(content) || Object.keys(answerDef.values).includes(content) if (!isKnown) { diagnostics.push({ code: DiagnosticCodes.ANSWER_TEMPLATE_ERROR, severity: 'warning', title: 'Answer template references unknown value', message: `Answer display.${template.name} template references "{{${content}}}" which may not be available.`, location: { section: 'answer', path: `display.${template.name}`, description: `answer.display.${template.name}`, }, suggestion: `Ensure "${content}" is defined as an input, transform, or answer value.`, }) } } } return diagnostics } // ============================================================================= // Mermaid Content Checks // ============================================================================= /** * Check for escaped quotes in mermaid content. * * LLMs sometimes generate escaped quotes like `\"` which break mermaid parsing. * Mermaid expects node content in double quotes: `NODE["content"]` * If the content itself contains `\"`, mermaid can't parse it. */ function checkMermaidEscapedQuotes(mermaidContent: string): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] // Check for escaped double quotes (common LLM mistake) if (mermaidContent.includes('\\"') || mermaidContent.includes("\\'")) { // Find the line number where this occurs const lines = mermaidContent.split('\n') let lineNumber = 0 for (let i = 0; i < lines.length; i++) { if (lines[i].includes('\\"') || lines[i].includes("\\'")) { lineNumber = i + 1 break } } diagnostics.push({ code: DiagnosticCodes.MERMAID_ESCAPED_QUOTES, severity: 'error', title: 'Escaped quotes in mermaid content', message: 'The mermaid content contains escaped quotes (backslash-quote) which break parsing. Use single quotes or rephrase to avoid quotes entirely.', location: { section: 'mermaid', path: `line ${lineNumber}`, description: `Line ${lineNumber} of mermaid content`, }, suggestion: "Replace escaped quotes with single quotes (') or remove quotes. Example: change 'same \"state\"' to 'same state' or \"same 'state'\"", }) } return diagnostics } /** * Check that all nodes defined in the JSON exist in the mermaid content. * A mismatch here will cause rendering failures because the loader creates * placeholder content for missing nodes, but decision options and navigation * will be broken. */ function checkMermaidNodeConsistency( definition: FlowchartDefinition, mermaidContent: string ): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] // Parse mermaid to get node IDs const parsedMermaid = parseMermaidFile(mermaidContent) const mermaidNodeIds = new Set(Object.keys(parsedMermaid.nodes)) const jsonNodeIds = Object.keys(definition.nodes) // Find JSON nodes missing from mermaid const missingInMermaid = jsonNodeIds.filter((id) => !mermaidNodeIds.has(id)) if (missingInMermaid.length > 0) { // Get the mermaid node IDs to suggest what might be the intended mapping const mermaidIdsArray = Array.from(mermaidNodeIds) diagnostics.push({ code: DiagnosticCodes.MERMAID_NODE_MISMATCH, severity: 'error', title: 'JSON and mermaid node IDs do not match', message: `${missingInMermaid.length} node(s) defined in JSON are not found in mermaid: ${missingInMermaid.slice(0, 5).join(', ')}${missingInMermaid.length > 5 ? '...' : ''}. Mermaid has these nodes: ${mermaidIdsArray.slice(0, 5).join(', ')}${mermaidIdsArray.length > 5 ? '...' : ''}`, location: { section: 'mermaid', description: 'Node ID mapping between JSON and mermaid', }, suggestion: `The node IDs in the JSON "nodes" object must EXACTLY match the node IDs in the mermaid flowchart. For example, if your JSON has "nodes": { "START": {...} }, then the mermaid must have START["..."] or START{"..."} etc. REGENERATE the mermaid content using the SAME node IDs as defined in the JSON: ${jsonNodeIds.slice(0, 8).join(', ')}${jsonNodeIds.length > 8 ? '...' : ''}`, }) } return diagnostics } // ============================================================================= // Test Coverage Checks // ============================================================================= /** * ValidationReport type (matches test-case-validator.ts) * Defined here to avoid circular imports */ interface ValidationReportForDoctor { passed: boolean results: Array<{ example: { name: string; expectedAnswer?: string } actualAnswer: string | null expectedAnswer: string passed: boolean error?: string }> coverage: { totalPaths: number coveredPaths: number uncoveredPaths: string[] coveragePercent: number } summary: { total: number passed: number failed: number errors: number } } /** * Check test coverage and generate diagnostics for issues. * * This converts a validation report into doctor diagnostics so that * coverage issues appear alongside other flowchart problems. */ export function checkTestCoverage( definition: FlowchartDefinition, validationReport: ValidationReportForDoctor ): FlowchartDiagnostic[] { const diagnostics: FlowchartDiagnostic[] = [] const examples = definition.problemInput.examples || [] const examplesWithTests = examples.filter((ex) => ex.expectedAnswer) // Check 1: No test cases at all if (examplesWithTests.length === 0) { diagnostics.push({ code: DiagnosticCodes.TEST_NO_EXPECTED_ANSWERS, severity: 'warning', title: 'No test cases with expected answers', message: 'None of the examples in problemInput.examples have expectedAnswer defined. Without test cases, display.answer bugs cannot be automatically detected.', location: { section: 'problemInput', path: 'examples', description: 'problemInput.examples', }, suggestion: 'Add expectedAnswer to each example that shows the exact string display.answer should produce. For example: { "name": "Simple case", "values": {...}, "expectedAnswer": "5" }', }) return diagnostics // No point checking other things if there are no tests } // Check 2: Failing tests const failingTests = validationReport.results.filter((r) => !r.passed) if (failingTests.length > 0) { for (const failure of failingTests) { const errorDetail = failure.error ? `Error: ${failure.error}` : `Expected "${failure.expectedAnswer}" but got "${failure.actualAnswer}"` diagnostics.push({ code: DiagnosticCodes.TEST_FAILING, severity: 'error', title: `Test "${failure.example.name}" is failing`, message: `${errorDetail}. The display.answer expression does not produce the expected output for this test case.`, location: { section: 'display', path: 'answer', description: `display.answer (test: "${failure.example.name}")`, }, suggestion: `Fix the display.answer expression so it produces "${failure.expectedAnswer}" when evaluated with the test inputs. Check for issues like: wrong conditional logic, missing whole number handling, or incorrect string formatting.`, }) } } // Check 3: Incomplete path coverage const { totalPaths, coveredPaths, uncoveredPaths, coveragePercent } = validationReport.coverage if (totalPaths > 0 && coveredPaths < totalPaths) { const uncoveredList = uncoveredPaths.length > 0 ? uncoveredPaths.slice(0, 3).join(', ') + (uncoveredPaths.length > 3 ? ` and ${uncoveredPaths.length - 3} more` : '') : 'some paths' diagnostics.push({ code: DiagnosticCodes.TEST_INCOMPLETE_COVERAGE, severity: 'warning', title: `Test coverage is ${coveragePercent}% (${coveredPaths}/${totalPaths} paths)`, message: `Not all flowchart paths are covered by test cases. Uncovered paths: ${uncoveredList}. Bugs in display.answer for these paths may go undetected.`, location: { section: 'problemInput', path: 'examples', description: 'problemInput.examples (coverage)', }, suggestion: `Add test cases for the uncovered paths. Each example should exercise a different path through the flowchart to ensure display.answer works correctly for all cases.`, }) } return diagnostics } // ============================================================================= // Main Doctor Function // ============================================================================= /** * Run all diagnostic checks on a flowchart definition. * * @param definition - The flowchart definition to validate * @param mermaidContent - Optional mermaid content to check for syntax issues * @returns A diagnostic report with all found issues */ export function diagnoseFlowchart( definition: FlowchartDefinition, mermaidContent?: string ): DiagnosticReport { const diagnostics: FlowchartDiagnostic[] = [] // Run definition checks diagnostics.push(...checkDerivedFields(definition)) diagnostics.push(...checkDisplayAnswer(definition)) diagnostics.push(...checkDivisionInTarget(definition)) diagnostics.push(...checkGenerationPreferred(definition)) // Run new unified computation model checks diagnostics.push(...checkNodeTransforms(definition)) diagnostics.push(...checkAnswerDefinition(definition)) // Run mermaid content checks if provided if (mermaidContent) { diagnostics.push(...checkMermaidEscapedQuotes(mermaidContent)) diagnostics.push(...checkMermaidNodeConsistency(definition, mermaidContent)) } // Count by severity const errorCount = diagnostics.filter((d) => d.severity === 'error').length const warningCount = diagnostics.filter((d) => d.severity === 'warning').length return { isHealthy: errorCount === 0, errorCount, warningCount, diagnostics, } } /** * Get only errors (not warnings) from a diagnostic report */ export function getErrors(report: DiagnosticReport): FlowchartDiagnostic[] { return report.diagnostics.filter((d) => d.severity === 'error') } /** * Get only warnings from a diagnostic report */ export function getWarnings(report: DiagnosticReport): FlowchartDiagnostic[] { return report.diagnostics.filter((d) => d.severity === 'warning') } /** * Format a diagnostic for display in console or logs */ export function formatDiagnostic(diagnostic: FlowchartDiagnostic): string { const severityIcon = diagnostic.severity === 'error' ? '❌' : diagnostic.severity === 'warning' ? '⚠️' : 'ℹ️' return `${severityIcon} [${diagnostic.code}] ${diagnostic.title} Location: ${diagnostic.location.description} ${diagnostic.message}${diagnostic.suggestion ? `\n 💡 ${diagnostic.suggestion}` : ''}` } /** * Format a diagnostic as an LLM refinement prompt. * This creates clear, actionable instructions for the LLM to fix the issue. */ export function formatDiagnosticForRefinement(diagnostic: FlowchartDiagnostic): string { const parts: string[] = [] // Start with clear instruction parts.push(`Fix the following ${diagnostic.severity} in the flowchart configuration:`) parts.push('') // The issue parts.push(`**Issue:** ${diagnostic.title}`) parts.push(`**Location:** ${diagnostic.location.description}`) parts.push(`**Details:** ${diagnostic.message}`) // The fix if (diagnostic.suggestion) { parts.push('') parts.push(`**How to fix:** ${diagnostic.suggestion}`) } return parts.join('\n') } /** * Format multiple diagnostics for LLM refinement. * Groups them logically and provides context. */ export function formatDiagnosticsForRefinement(diagnostics: FlowchartDiagnostic[]): string { if (diagnostics.length === 0) return '' if (diagnostics.length === 1) { return formatDiagnosticForRefinement(diagnostics[0]) } const parts: string[] = [] parts.push(`Fix the following ${diagnostics.length} issues in the flowchart configuration:`) parts.push('') for (let i = 0; i < diagnostics.length; i++) { const d = diagnostics[i] parts.push(`### Issue ${i + 1}: ${d.title}`) parts.push(`- **Location:** ${d.location.description}`) parts.push(`- **Details:** ${d.message}`) if (d.suggestion) { parts.push(`- **How to fix:** ${d.suggestion}`) } parts.push('') } return parts.join('\n') } |