Hands-On: Complex Bug Investigation

// Stage 1: Set up automatic capture on CAN bus-off event

// Trigger: CAN error passive → error bus-off transition
// Aurix CAN SFR: CAN0.TGFC.BOFF = 1 when bus-off occurs
Break.Set SFR:CAN0.NODE0SR.BOFF /Write /Hardware /CONDition Data.Byte(SFR:CAN0.NODE0SR.BOFF)==1

// Enable ETM trace to capture execution leading up to fault
ETM.ON
ETM.DataTrace NONE
Trace.METHOD Analyzer
Trace.Size 256MB

// Run — will halt when bus-off bit is set
PRINT "Waiting for CAN bus-off event..."
Go
WAIT !STATE.RUN() 600s   // wait up to 10 minutes

IF STATE.RUN()
    PRINT "Event not detected in 10 min — extend wait or verify trigger"
ELSE
(
    PRINT "BUS-OFF EVENT DETECTED at " FORMAT.ADDRESS(Register(PC))
    Trace.SAVE canbus_crash_trace.t32t   // save full trace for analysis
    Data.SAVE.Binary sram_snapshot.bin 0x70000000--0x700FFFFF
)

// Stage 2: Analyse saved trace to find what triggered bus-off

// Load saved trace
Trace.LOAD canbus_crash_trace.t32t
Data.LOAD.Binary sram_snapshot.bin 0x70000000

// Step 1: Find the last CAN error counter increment before bus-off
// CAN error passive threshold: 128 errors; bus-off: 256 errors
// Look for rapid error count growth in trace
Trace.Find /Address SFR:CAN0.NODE0ECR    // find all accesses to error counter register

// Step 2: Examine execution ~5ms before bus-off
// Trace.Chart.TASK shows what was running at T-5ms
Trace.Chart.TASK /ZOOM -5ms 0ms         // last 5ms before halt

// Step 3: Find the anomaly
// Expected: ETM shows OsTask_10ms briefly calling CAN transmit at 50kHz rate
// instead of normal 1kHz rate
Trace.Statistics.Func "Can_Write"        // call count for Can_Write in trace
// FINDING: Can_Write called 483 times in last 1ms (expected: ~10)

// Step 4: Trace back to what caused the call rate spike
Trace.Find /Entry "Can_Write"            // find every call to Can_Write
// All calls originate from: Nm_MainFunction → CanNm_TxConfirmation loop
// Bug: CanNm missing return guard — callback loops under error condition

// Expected finding: CanNm_TxConfirmation called recursively when CAN TX queue full
// CAN flood → bus error counter hits 256 → bus-off
// Fix: add one-shot guard in CanNm_TxConfirmation: if (s_txActive) return;

// Stage 3: Verify fix and add regression guard

// After fix applied and reflashed:
Data.LOAD.Elf build/app_fixed.elf /RELPATH
SYStem.Reset

// Re-run bus-off detection for 30 minutes
Break.Set SFR:CAN0.NODE0SR.BOFF /Write /Hardware /CONDition Data.Byte(SFR:CAN0.NODE0SR.BOFF)==1
ETM.ON
Trace.METHOD Analyzer
Trace.Size 256MB

PRINT "Running regression for 30 min..."
Go
WAIT !STATE.RUN() 1800s

IF STATE.RUN()
    PRINT "PASS: No bus-off event in 30 minutes"
ELSE
    PRINT %ERROR "FAIL: bus-off still occurring after fix"

// Regression test: add Can_Write call rate monitor to CI suite
// (catches if call rate exceeds 50/ms in any 1ms window)
// Add to hil_results: {"id":"CAN-001","name":"Can_Write rate < 50/ms"...}

// Root cause summary:
PRINT "Root cause: CanNm_TxConfirmation recursive entry under CAN TX error"
PRINT "Fix: added s_txActive guard flag; one-shot callback pattern"
PRINT "Regression test: Can_Write call rate monitor added to HIL CI suite"
PRINT "MCSS: TARA entry TS-CAN-03 updated: 'CanNm reentrance' risk closed"