All files / web/src/app/vision-training/train/components TrainingDiagnosticsContext.tsx

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
'use client'

import { createContext, useContext, useMemo, type ReactNode } from 'react'
import {
  isColumnClassifierSamples,
  isColumnClassifierDatasetInfo,
  type SamplesData,
  type DatasetInfo,
  type EpochData,
  type TrainingConfig,
  type TrainingResult,
} from './wizard/types'

export interface DiagnosticReason {
  type: 'imbalance' | 'insufficient-data' | 'poor-convergence' | 'unknown'
  severity: 'warning' | 'error'
  title: string
  description: string
  action: string
  details?: {
    underrepresented?: number[]
    minCount?: number
    maxCount?: number
    totalImages?: number
  }
}

interface TrainingDiagnostics {
  // Raw data
  samples: SamplesData | null
  datasetInfo: DatasetInfo | null
  epochHistory: EpochData[]
  config: TrainingConfig
  result: TrainingResult | null

  // Computed diagnostics
  shouldShowRemediation: boolean
  reasons: DiagnosticReason[]
}

const TrainingDiagnosticsContext = createContext<TrainingDiagnostics | null>(null)

export function useTrainingDiagnostics(): TrainingDiagnostics {
  const ctx = useContext(TrainingDiagnosticsContext)
  if (!ctx) {
    throw new Error('useTrainingDiagnostics must be used within TrainingDiagnosticsProvider')
  }
  return ctx
}

interface ProviderProps {
  samples: SamplesData | null
  datasetInfo: DatasetInfo | null
  epochHistory: EpochData[]
  config: TrainingConfig
  result: TrainingResult | null
  children: ReactNode
}

export function TrainingDiagnosticsProvider({
  samples,
  datasetInfo,
  epochHistory,
  config,
  result,
  children,
}: ProviderProps) {
  const diagnostics = useMemo(() => {
    const reasons = analyzeDiagnostics(result, samples, datasetInfo, epochHistory)
    const accuracy = result?.final_accuracy ?? 0

    return {
      samples,
      datasetInfo,
      epochHistory,
      config,
      result,
      shouldShowRemediation: accuracy < 0.5 || (accuracy < 0.7 && reasons.length > 0),
      reasons,
    }
  }, [samples, datasetInfo, epochHistory, config, result])

  return (
    <TrainingDiagnosticsContext.Provider value={diagnostics}>
      {children}
    </TrainingDiagnosticsContext.Provider>
  )
}

function analyzeDiagnostics(
  result: TrainingResult | null,
  samples: SamplesData | null,
  datasetInfo: DatasetInfo | null,
  epochHistory: EpochData[]
): DiagnosticReason[] {
  if (!result) return []

  const accuracy = result.final_accuracy
  const reasons: DiagnosticReason[] = []

  // 1. Check for data imbalance (column classifier only)
  if (samples && isColumnClassifierSamples(samples)) {
    const counts = Object.values(samples.digits).map((d) => d.count)
    const max = Math.max(...counts)
    const min = Math.min(...counts)

    if (max > min * 5 && min < 10) {
      const underrepresented = Object.entries(samples.digits)
        .filter(([, d]) => d.count < max / 3)
        .map(([digit]) => parseInt(digit, 10))

      reasons.push({
        type: 'imbalance',
        severity: 'error',
        title: 'Data imbalance',
        description: `Some digits have very few samples (${min}) while others have many (${max})`,
        action: `Collect more samples for digits: ${underrepresented.join(', ')}`,
        details: { underrepresented, minCount: min, maxCount: max },
      })
    }
  }

  // 2. Check for insufficient total data
  const total = datasetInfo
    ? isColumnClassifierDatasetInfo(datasetInfo)
      ? datasetInfo.total_images
      : datasetInfo.total_frames
    : samples
      ? isColumnClassifierSamples(samples)
        ? samples.totalImages
        : samples.totalFrames
      : 0
  if (total < 200) {
    reasons.push({
      type: 'insufficient-data',
      severity: total < 100 ? 'error' : 'warning',
      title: 'Insufficient training data',
      description: `Only ${total} images available`,
      action: 'Collect at least 200 images (20+ per digit)',
      details: { totalImages: total },
    })
  }

  // 3. Check for poor convergence (accuracy barely improved during training)
  if (epochHistory.length >= 2) {
    const firstAcc = epochHistory[0]?.val_accuracy ?? 0
    const lastAcc = epochHistory[epochHistory.length - 1]?.val_accuracy ?? 0
    const improvement = lastAcc - firstAcc

    if (improvement < 0.1 && accuracy < 0.5) {
      reasons.push({
        type: 'poor-convergence',
        severity: 'warning',
        title: 'Model failed to learn',
        description: 'Accuracy barely improved during training',
        action: 'Check data quality - images may be too noisy or inconsistent',
      })
    }
  }

  // 4. Unknown issue if accuracy is bad but no clear reason
  if (accuracy < 0.5 && reasons.length === 0) {
    reasons.push({
      type: 'unknown',
      severity: 'warning',
      title: 'Unexpected low accuracy',
      description: 'Data appears adequate but accuracy is poor',
      action: 'Try training again or review captured images for quality issues',
    })
  }

  return reasons
}