/* REXX */
/*--------------------------------------------------------------------*/
/* CPUDIAG - CPU first-response diagnostic                            */
/*                                                                    */
/* Author  : SARATH S                                                 */
/* Purpose : Sample CPU repeatedly through SDSF SYS and DA ALL,       */
/*           identify persistent CPU consumers, and add four-hour     */
/*           MSU context to the current CPU condition.                */
/*                                                                    */
/* Invocation examples:                                              */
/*   TSO CPUDIAG                                                      */
/*   TSO CPUDIAG 15                                                   */
/*   TSO CPUDIAG 15 5 10                                             */
/*   TSO CPUDIAG SAMPLE                                              */
/*   TSO CPUDIAG HELP                                                */
/*                                                                    */
/* Parameters:                                                       */
/*   1 - Top rows in each ranking. Default 10.                        */
/*   2 - Number of samples. Default 5.                                */
/*   3 - Seconds between samples. Default 10.                         */
/*                                                                    */
/* Design principles:                                                */
/* - Single member. No site-specific data sets or job names.          */
/* - Read-only. No system or workload changes are issued.             */
/* - Uses SDSF SYS and explicit DA ALL.                               */
/* - Takes repeated samples instead of trusting one CPU snapshot.     */
/* - Uses AvgMSU only as supporting four-hour workload context.       */
/* - Under ISPF, presents results in Browse for F7/F8/F9 support.     */
/* - Report data sets are retained after Browse is exited.            */
/*--------------------------------------------------------------------*/

signal on syntax name FatalSyntax

topN = 10
sampleCount = 5
intervalSec = 10

rpt. = ''
rpt.0 = 0
rptCount = 0
ispfActive = 0
sdsfOn = 0
ussOn = 0
reportMode = 'LIVE'
localSys = translate(MVSVAR('SYSNAME'))

if sysvar('SYSISPF') = 'ACTIVE' then ispfActive = 1

parse upper arg pTop pSamples pInterval .

if pTop = 'HELP' | pTop = 'H' | pTop = '?' then do
  reportMode = 'HELP'
  call HelpReport
  call PresentReport
  exit 0
end

if pTop = 'SAMPLE' | pTop = 'S' then do
  reportMode = 'SAMPLE'
  call SampleReport
  call PresentReport
  exit 0
end

if pTop <> '' then topN = pTop
if pSamples <> '' then sampleCount = pSamples
if pInterval <> '' then intervalSec = pInterval

if datatype(topN,'W') = 0 then do
  call Emit 'CPUDIAG: Top-row value must be a whole number.'
  call PresentReport
  exit 12
end

if datatype(sampleCount,'W') = 0 then do
  call Emit 'CPUDIAG: Sample count must be a whole number.'
  call PresentReport
  exit 12
end

if datatype(intervalSec,'W') = 0 then do
  call Emit 'CPUDIAG: Sample interval must be a whole number.'
  call PresentReport
  exit 12
end

if topN < 1 then topN = 10
if topN > 25 then topN = 25

if sampleCount < 3 then sampleCount = 3
if sampleCount > 12 then sampleCount = 12

if intervalSec < 5 then intervalSec = 5
if intervalSec > 60 then intervalSec = 60

startTime = time('N')

sampleTime. = ''
sampleCPU. = ''
samplePage. = ''
sampleSIO. = ''
sampleUIC. = ''
sampleRealPct. = ''
sampleRealAFC. = ''
sampleAuxPct. = ''
sampleOK. = 0

idx. = 0
aName. = ''
aJobid. = ''
aType. = ''
aAsid. = ''
aSum. = 0
aPeak. = 0
aSeen. = 0
aCpuTime. = ''
aPaging. = ''
aSio. = ''
aggCount = 0

cpuValid = 0
cpuSum = 0
cpuMin = 999
cpuMax = 0
cpuHi80 = 0
cpuHi90 = 0
cpuHi95 = 0
pageSum = 0
pagePeak = 0
pageActive = 0
sioSum = 0
sioPeak = 0
uicMin = 999999999
uicValid = 0
realPctPeak = 0
realAvailMin = 999999999999
realAvailValid = 0
auxPctPeak = 0
daValid = 0
daAvailable = 1

lastSYSLEVEL = 'N/A'
lastCPUPR = 'N/A'
lastSIO = 'N/A'
lastAUXPCT = 'N/A'
lastPAGERATE = 'N/A'
lastUIC = 'N/A'
lastREALPCT = 'N/A'
lastREAL = 'N/A'
lastREALAFC = 'N/A'
lastWTORNUM = 'N/A'
lastSTCNUM = 'N/A'
lastJOBNUM = 'N/A'
lastTSUNUM = 'N/A'
lastMSU = 'N/A'
lastSYSMSU = 'N/A'
lastAVGMSU = 'N/A'
lastCPUNUM = 'N/A'
lastZIIPNUM = 'N/A'
lastIPLDATE = 'N/A'
lastIPLDAYS = 'N/A'
lastSTATDATE = 'N/A'

rc = isfcalls('ON')
if rc <> 0 then do
  call Emit 'CPUDIAG: Unable to establish SDSF REXX environment.'
  call Emit 'RC='rc
  call PresentReport
  exit 20
end
sdsfOn = 1

/* Match JOBINFO style: set filters explicitly for this invocation. */
isfprefix = '**'
isfowner = '*'
isfsysname = localSys

call FetchSYS 0
if result <> 0 then do
  call Emit 'CPUDIAG: SDSF SYS data could not be obtained.'
  call Cleanup
  call PresentReport
  exit 20
end

ussRc = syscalls('ON')
if ussRc > 4 then do
  call Emit 'CPUDIAG: Timed sampling could not be initialized.'
  call Emit 'SYSCALLS RC='ussRc
  call Cleanup
  call PresentReport
  exit 20
end
ussOn = 1

/* Warm up DA measurement interval and deliberately discard it. */
call FetchDA 0
if result <> 0 then do
  daAvailable = 0
  call Emit 'CPUDIAG WARNING: DA ALL data is unavailable.'
  call Emit 'CPU sampling will continue without consumer rankings.'
end
else call SleepSeconds intervalSec

do s = 1 to sampleCount
  sampleTime.s = time('N')

  call FetchSYS s
  sysRc = result

  if sysRc = 0 then do
    sampleOK.s = 1
    v = Num(sampleCPU.s)

    cpuValid = cpuValid + 1
    cpuSum = cpuSum + v

    if v < cpuMin then cpuMin = v
    if v > cpuMax then cpuMax = v

    if v >= 80 then cpuHi80 = cpuHi80 + 1
    if v >= 90 then cpuHi90 = cpuHi90 + 1
    if v >= 95 then cpuHi95 = cpuHi95 + 1

    p = Num(samplePage.s)
    pageSum = pageSum + p
    if p > pagePeak then pagePeak = p
    if p > 0 then pageActive = pageActive + 1

    io = Num(sampleSIO.s)
    sioSum = sioSum + io
    if io > sioPeak then sioPeak = io

    if IsNum(sampleUIC.s) then do
      uv = Num(sampleUIC.s)
      uicValid = uicValid + 1
      if uv < uicMin then uicMin = uv
    end

    if IsNum(sampleRealPct.s) then do
      rv = Num(sampleRealPct.s)
      if rv > realPctPeak then realPctPeak = rv
    end

    if IsNum(sampleRealAFC.s) then do
      av = Num(sampleRealAFC.s)
      realAvailValid = realAvailValid + 1
      if av < realAvailMin then realAvailMin = av
    end

    if IsNum(sampleAuxPct.s) then do
      ax = Num(sampleAuxPct.s)
      if ax > auxPctPeak then auxPctPeak = ax
    end
  end

  if daAvailable then do
    call FetchDA s
    if result = 0 then daValid = daValid + 1
  end

  if s < sampleCount then call SleepSeconds intervalSec
end

if cpuValid = 0 then do
  call Emit 'CPUDIAG: No valid CPU samples were collected.'
  call Cleanup
  call PresentReport
  exit 20
end

cpuAvg = cpuSum / cpuValid
needHigh = trunc(((cpuValid * 4) + 4) / 5)
if needHigh < 2 then needHigh = 2

select
  when cpuAvg >= 95 & cpuHi95 >= needHigh then do
    cpuState = 'SUSTAINED VERY HIGH'
    severity = 'HIGH'
  end
  when cpuAvg >= 90 & cpuHi90 >= needHigh then do
    cpuState = 'SUSTAINED HIGH'
    severity = 'HIGH'
  end
  when cpuHi90 > 0 then do
    cpuState = 'INTERMITTENT HIGH'
    severity = 'MEDIUM'
  end
  when cpuAvg >= 80 then do
    cpuState = 'ELEVATED'
    severity = 'MEDIUM'
  end
  otherwise do
    cpuState = 'NORMAL'
    severity = 'LOW'
  end
end

pageAvg = pageSum / cpuValid
sioAvg = sioSum / cpuValid

if realPctPeak >= 95 & pageActive >= needHigh then
  storageState = 'POSSIBLE REAL-STORAGE PRESSURE'
else if realPctPeak >= 95 & pageActive > 0 then
  storageState = 'WATCH - PAGING WITH LOW REAL HEADROOM'
else if pageActive = 0 then
  storageState = 'NO PAGING PRESSURE OBSERVED'
else
  storageState = 'PAGING PRESENT - NO CLEAR SHORTAGE SIGNAL'

msuRatioValid = 0
msuRatio = 0
sysMsuNum = Num(lastSYSMSU)
avgMsuNum = Num(lastAVGMSU)

if sysMsuNum > 0 then do
  msuRatio = avgMsuNum * 100 / sysMsuNum
  msuRatioValid = 1
end

msuMature = MSUMature()

if \msuMature then
  msuState = 'NOT INTERPRETED - IPL AGE IS BELOW FOUR HOURS'
else if \msuRatioValid then
  msuState = 'UNAVAILABLE'
else if msuRatio > 100 then
  msuState = 'ABOVE IMAGE MSU RATING - REVIEW CAPACITY CONTEXT'
else if msuRatio >= 95 then
  msuState = 'VERY HIGH FOUR-HOUR MSU LOAD'
else if msuRatio >= 85 then
  msuState = 'HIGH FOUR-HOUR MSU LOAD'
else if msuRatio >= 70 then
  msuState = 'ELEVATED FOUR-HOUR MSU LOAD'
else
  msuState = 'LOW/MODERATE FOUR-HOUR MSU LOAD'

top3Cpu = TopAverageSum(3)

if cpuAvg > 0 then top3Share = top3Cpu * 100 / cpuAvg
else top3Share = 0

if top3Share >= 60 then distribution = 'CONCENTRATED'
else if top3Share > 0 then distribution = 'DISTRIBUTED'
else distribution = 'UNKNOWN'

call Header
call TriageSummary
call SystemContext
call ResourceCorrelation
call Analysis

if daAvailable & daValid > 0 then do
  call PrintTop 'ALL', 'TOP CPU CONSUMERS - ALL ADDRESS SPACES'
  call PrintTop 'JOB', 'TOP CPU CONSUMING BATCH JOBS'
  call PrintTop 'STC', 'TOP CPU CONSUMING STARTED TASKS'
  call PrintTop 'TSU', 'TOP CPU CONSUMING TSO USERS'
end
else do
  call Section 'AFFECTED WORKLOAD / TOP CPU CONSUMERS'
  call Emit 'DA ALL data was unavailable for this invocation.'
end

call RecentEvents
call Recommendations
call OperatorCommands
call SampleHistory
call Notes
call Footer
call Cleanup
call PresentReport
exit 0


/*====================================================================*/
/* SDSF DATA COLLECTION                                               */
/*====================================================================*/

FetchSYS:
parse arg fsSample

drop SYSNAME. SYSLEVEL. CPUPR. SIO. AUXPCT. PAGERATE. UIC.
drop REALPCT. REAL. REALAFC. WTORNUM. STCNUM. JOBNUM.
drop TSUNUM. MSU. SYSMSU. AVGMSU. CPUNUM. ZIIPNUM.
drop IPLDATE. IPLDAYS. STATDATE.

isfcols = 'SYSNAME SYSLEVEL CPUPR SIO AUXPCT PAGERATE UIC' ,
          'REALPCT REAL REALAFC WTORNUM STCNUM JOBNUM TSUNUM MSU' ,
          'SYSMSU AVGMSU CPUNUM ZIIPNUM IPLDATE IPLDAYS STATDATE'

Address SDSF "ISFEXEC SYS"
fsRc = rc

if fsRc <> 0 then do
  call Emit 'CPUDIAG: SDSF SYS access failed. RC='fsRc
  call SdsfMessages
  return fsRc
end

rows = isfrows
if rows = 0 then return 8

pick = 0
do i = 1 to rows
  if translate(strip(SYSNAME.i)) = localSys then do
    pick = i
    leave
  end
end

if pick = 0 then pick = 1

if fsSample > 0 then do
  sampleCPU.fsSample = strip(CPUPR.pick)
  samplePage.fsSample = strip(PAGERATE.pick)
  sampleSIO.fsSample = strip(SIO.pick)
  sampleUIC.fsSample = strip(UIC.pick)
  sampleRealPct.fsSample = strip(REALPCT.pick)
  sampleRealAFC.fsSample = strip(REALAFC.pick)
  sampleAuxPct.fsSample = strip(AUXPCT.pick)
end

lastSYSLEVEL = ValueOr(SYSLEVEL.pick,'N/A')
lastCPUPR = ValueOr(CPUPR.pick,'N/A')
lastSIO = ValueOr(SIO.pick,'N/A')
lastAUXPCT = ValueOr(AUXPCT.pick,'N/A')
lastPAGERATE = ValueOr(PAGERATE.pick,'N/A')
lastUIC = ValueOr(UIC.pick,'N/A')
lastREALPCT = ValueOr(REALPCT.pick,'N/A')
lastREAL = ValueOr(REAL.pick,'N/A')
lastREALAFC = ValueOr(REALAFC.pick,'N/A')
lastWTORNUM = ValueOr(WTORNUM.pick,'N/A')
lastSTCNUM = ValueOr(STCNUM.pick,'N/A')
lastJOBNUM = ValueOr(JOBNUM.pick,'N/A')
lastTSUNUM = ValueOr(TSUNUM.pick,'N/A')
lastMSU = ValueOr(MSU.pick,'N/A')
lastSYSMSU = ValueOr(SYSMSU.pick,'N/A')
lastAVGMSU = ValueOr(AVGMSU.pick,'N/A')
lastCPUNUM = ValueOr(CPUNUM.pick,'N/A')
lastZIIPNUM = ValueOr(ZIIPNUM.pick,'N/A')
lastIPLDATE = ValueOr(IPLDATE.pick,'N/A')
lastIPLDAYS = ValueOr(IPLDAYS.pick,'N/A')
lastSTATDATE = ValueOr(STATDATE.pick,'N/A')
return 0


FetchDA:
parse arg fdSample

drop JNAME. JOBID. JTYPE. ASID. CPUPR. CPU. PAGING. EXCPRT.

isfcols = 'JNAME JOBID JTYPE ASID CPUPR CPU PAGING EXCPRT'

Address SDSF "ISFEXEC DA ALL"
fdRc = rc

if fdRc <> 0 then do
  call Emit 'CPUDIAG: SDSF DA ALL access failed. RC='fdRc
  call SdsfMessages
  return fdRc
end

if fdSample = 0 then return 0

rows = isfrows

do i = 1 to rows
  jn = strip(JNAME.i)
  if jn = '' then iterate

  jid = strip(JOBID.i)
  typ = translate(strip(JTYPE.i))
  asidVal = strip(ASID.i)

  if jid = '' then jid = '-'
  if typ = '' then typ = 'N/A'
  if asidVal = '' then asidVal = '-'

  key = c2x(jn'|'jid'|'asidVal)

  if idx.key = 0 then do
    aggCount = aggCount + 1
    ai = aggCount
    idx.key = ai

    aName.ai = jn
    aJobid.ai = jid
    aType.ai = typ
    aAsid.ai = asidVal
    aSum.ai = 0
    aPeak.ai = 0
    aSeen.ai = 0
  end
  else ai = idx.key

  cpuVal = Num(CPUPR.i)
  aSum.ai = aSum.ai + cpuVal

  if cpuVal > aPeak.ai then aPeak.ai = cpuVal

  aSeen.ai = aSeen.ai + 1
  aCpuTime.ai = ValueOr(CPU.i,'N/A')
  aPaging.ai = ValueOr(PAGING.i,'N/A')
  aSio.ai = ValueOr(EXCPRT.i,'N/A')
end
return 0


/*====================================================================*/
/* REPORT                                                             */
/*====================================================================*/

Header:
call Emit ''
call Emit copies('=',78)
call Emit ' CPUDIAG - CPU FIRST-RESPONSE DIAGNOSTIC'
call KV 'Author', 'SARATH S'
call KV 'Date', DisplayDate()
call KV 'Time', time('N')
call KV 'System', localSys
call KV 'Mode', reportMode
call Emit copies('=',78)
return


TriageSummary:
call Section 'TRIAGE SUMMARY'
call KV 'Severity', severity
call KV 'CPU condition', cpuState
call KV 'Average sampled CPU', FormatPct(cpuAvg)
call KV 'Minimum sampled CPU', FormatPct(cpuMin)
call KV 'Peak sampled CPU', FormatPct(cpuMax)
call KV 'Samples >= 90%', cpuHi90 'of' cpuValid
call KV 'Samples >= 95%', cpuHi95 'of' cpuValid
call KV 'CPU distribution', distribution
call KV 'Storage-pressure signal', storageState

if top3Share > 0 then
  call KV 'Top 3 share of busy CPU', FormatPct(top3Share)

if msuRatioValid then
  call KV 'AvgMSU / SysMSU', FormatPct(msuRatio)

call KV 'Four-hour MSU assessment', msuState
return


SampleHistory:
call Section 'SUPPORTING DATA - CPU / RESOURCE SAMPLE HISTORY'
call Emit 'Samp Time      CPU% PageRate   UIC Real% Aux%      SIO'
call Emit copies('-',65)

do i = 1 to sampleCount
  if sampleOK.i then do
    line = right(i,4) ,
           left(sampleTime.i,8) ,
           right(FormatPct(Num(sampleCPU.i)),5) ,
           right(samplePage.i,8) ,
           right(sampleUIC.i,5) ,
           right(AddPct(sampleRealPct.i),5) ,
           right(AddPct(sampleAuxPct.i),5) ,
           right(sampleSIO.i,9)
  end
  else do
    line = right(i,4) left(sampleTime.i,8) ,
           right('N/A',5) right('N/A',8) ,
           right('N/A',5) right('N/A',5) ,
           right('N/A',5) right('N/A',9)
  end
  call Emit line
end
return


ResourceCorrelation:
call Section 'RESOURCE CORRELATION / SECONDARY EVIDENCE'
call KV 'Storage-pressure signal', storageState
call KV 'Average paging rate', FmtNum(pageAvg)
call KV 'Peak paging rate', FmtNum(pagePeak)
call KV 'Paging-active samples', pageActive 'of' cpuValid

if uicValid > 0 then
  call KV 'Minimum sampled UIC', strip(uicMin)
else
  call KV 'Minimum sampled UIC', 'N/A'

call KV 'Peak real storage used', FormatPct(realPctPeak)

if realAvailValid > 0 then
  call KV 'Minimum available frames', strip(realAvailMin)
else
  call KV 'Minimum available frames', 'N/A'

call KV 'Peak auxiliary used', FormatPct(auxPctPeak)
call KV 'Average SIO rate', FmtNum(sioAvg)
call KV 'Peak SIO rate', FmtNum(sioPeak)

call Emit ''
call Emit 'Interpretation:'
if storageState = 'NO PAGING PRESSURE OBSERVED' then do
  call Emit '  No paging activity was seen during the sample window.'
  call Emit '  Current evidence does not indicate paging pressure.'
end
else if storageState = 'POSSIBLE REAL-STORAGE PRESSURE' then do
  call Emit '  Paging persisted while real-storage usage was very high.'
  call Emit '  Do not attribute the slowdown to CPU alone.'
end
else if storageState = ,
  'WATCH - PAGING WITH LOW REAL HEADROOM' then do
  call Emit '  Paging and high real-storage use occurred together.'
  call Emit '  Storage pressure is possible but not yet proven.'
end
else do
  call Emit '  Paging occurred, but the short sample does not show a'
  call Emit '  clear real-storage shortage pattern.'
end

call Emit '  UIC is supporting evidence only. A low UIC must be'
call Emit '  correlated with low available frames over time.'
call Emit '  SIO shows I/O activity, not I/O response time.'
return


SystemContext:
call Section 'CURRENT RESOURCE STATUS'
call KV 'Current CPU busy', AddPct(lastCPUPR)
call KV 'Processor MSU rating', lastMSU
call KV 'System image MSU rating', lastSYSMSU
call KV 'Four-hour Avg MSU', lastAVGMSU

if msuRatioValid then
  call KV 'AvgMSU / SysMSU', FormatPct(msuRatio)

call KV 'IPL date/time', lastIPLDATE
call KV 'Days since IPL', lastIPLDAYS
call KV 'Four-hour MSU mature', YesNo(msuMature)
call KV 'Online general CPUs', lastCPUNUM
call KV 'Online zIIPs', lastZIIPNUM
call KV 'Paging rate', lastPAGERATE
call KV 'High unreferenced interval', lastUIC
call KV 'SIO rate', lastSIO
call KV 'Aux storage used', AddPct(lastAUXPCT)
call KV 'Real storage used', AddPct(lastREALPCT)
call KV 'Real frames online', lastREAL
call KV 'Real frames available', lastREALAFC
call KV 'Active started tasks', lastSTCNUM
call KV 'Active batch jobs', lastJOBNUM
call KV 'Active TSO users', lastTSUNUM
call KV 'Outstanding WTORs', lastWTORNUM
call KV 'SYS statistics time', lastSTATDATE
return


PrintTop:
parse arg ptType, ptTitle
call Section ptTitle

call Emit 'Rank Jobname  Type JobID     ASID AvgCPU PeakCPU Seen'
call Emit copies('-',65)

picked. = 0
shown = 0

do rank = 1 to topN
  best = 0
  bestAvg = -1

  do i = 1 to aggCount
    if picked.i then iterate

    if ptType <> 'ALL' then
      if aType.i <> ptType then iterate

    if daValid > 0 then avg = aSum.i / daValid
    else avg = 0

    if avg > bestAvg then do
      best = i
      bestAvg = avg
    end
  end

  if best = 0 then leave
  if bestAvg <= 0 then leave

  picked.best = 1
  shown = shown + 1
  seenText = aSeen.best'/'daValid

  line = right(shown,4) ,
         left(aName.best,8) ,
         left(aType.best,4) ,
         left(aJobid.best,8) ,
         left(aAsid.best,4) ,
         right(FormatPct(bestAvg),7) ,
         right(FormatPct(aPeak.best),7) ,
         right(seenText,5)

  call Emit line
end

if shown = 0 then
  call Emit 'No non-zero CPU consumers were found in this category.'
return


Analysis:
call Section 'KEY FINDINGS / INTERPRETATION'

select
  when cpuState = 'SUSTAINED VERY HIGH' then do
    call Emit 'Very high CPU persisted through most samples.'
    call Emit 'CPU contention is a strong candidate for current'
    call Emit 'system or workload slowdown.'
  end

  when cpuState = 'SUSTAINED HIGH' then do
    call Emit 'High CPU persisted through most samples.'
    call Emit 'CPU pressure is a credible contributor to slowdown.'
  end

  when cpuState = 'INTERMITTENT HIGH' then do
    call Emit 'High CPU was observed, but it was not sustained.'
    call Emit 'Treat this as intermittent pressure, not confirmed'
    call Emit 'continuous CPU saturation.'
  end

  when cpuState = 'ELEVATED' then do
    call Emit 'CPU was elevated but sustained high-CPU criteria'
    call Emit 'were not met during the observation window.'
  end

  otherwise do
    call Emit 'Current measurements do not support CPU saturation'
    call Emit 'as the primary cause of the performance problem.'
  end
end

if msuMature & msuRatioValid then do
  if msuRatio >= 85 & cpuAvg >= 90 then do
    call Emit ''
    call Emit 'The four-hour rolling MSU is also high relative to'
    call Emit 'the image MSU rating. This supports longer-duration'
    call Emit 'high processor demand, not only a short spike.'
  end
  else if msuRatio < 70 & cpuAvg >= 90 then do
    call Emit ''
    call Emit 'Current CPU is high while four-hour MSU is lower.'
    call Emit 'The severe CPU condition may be relatively recent.'
  end
  else if msuRatio >= 85 & cpuAvg < 80 then do
    call Emit ''
    call Emit 'CPU is not currently saturated, but four-hour MSU'
    call Emit 'remains high. Earlier processor demand may be subsiding.'
  end
end

if \msuMature then do
  call Emit ''
  call Emit 'Four-hour AvgMSU is not interpreted because the system'
  call Emit 'has been up for less than four hours.'
end

call Emit ''
if cpuAvg >= 90 then do
  if storageState = 'NO PAGING PRESSURE OBSERVED' then do
    call Emit 'No paging pressure was observed with the high CPU.'
    call Emit 'Current evidence favors processor demand over'
    call Emit 'real-storage contention.'
  end
  else if storageState = ,
    'POSSIBLE REAL-STORAGE PRESSURE' then do
    call Emit 'CPU is high and storage-pressure signals are also'
    call Emit 'present. Do not assume CPU is the only bottleneck.'
  end
end
else if storageState = ,
  'POSSIBLE REAL-STORAGE PRESSURE' then do
  call Emit 'CPU is not saturated, while paging/storage signals'
  call Emit 'are present. Storage deserves higher investigation'
  call Emit 'priority than CPU for the current slowdown.'
end

if top3Share >= 60 then do
  call Emit ''
  call Emit 'CPU consumption is concentrated in a small number'
  call Emit 'of address spaces.'
end
else if top3Share > 0 then do
  call Emit ''
  call Emit 'CPU consumption is relatively distributed.'
end
return


Recommendations:
call Section 'RECOMMENDED DEBUGGING'

if cpuAvg >= 90 then do
  call Emit '1. Review persistent top CPU consumers above.'
  call Emit '2. Confirm whether the dominant workload is expected.'
  call Emit '3. Investigate a dominant subsystem/job specifically'
  call Emit '   before considering any system-wide action.'
  call Emit '4. If CPU is distributed, review WLM/service classes.'
  call Emit '5. Correlate paging and SIO with the reported slowdown.'
end
else do
  call Emit '1. CPU saturation is not currently confirmed.'
  call Emit '2. If slowness persists, investigate paging, storage,'
  call Emit '   I/O, ENQ contention and WLM delays.'
  call Emit '3. Rerun CPUDIAG when an intermittent slowdown recurs.'
end
return


RecentEvents:
call Section 'RECENT RELATED EVENTS'
call Emit 'CPUDIAG does not currently scan SYSLOG or OPERLOG.'
call Emit 'This section is reserved for future event correlation.'
call Emit 'Use subsystem/job logs if a top consumer needs follow-up.'
return


OperatorCommands:
call Section 'POSSIBLE MITIGATION / OPERATOR COMMANDS'
call Emit 'CPUDIAG does not issue any of the commands below.'
call Emit ''
call Emit 'Additional read-only verification commands:'
call Emit '  D M=CPU'
call Emit '      Display processor configuration and CPU status.'
call Emit '  D WLM'
call Emit '      Display the active WLM service policy.'
call Emit '  D WLM,SYSTEMS'
call Emit '      Display WLM status for systems in the sysplex.'
call Emit '  D A,LIST'
call Emit '      Display active address spaces.'
call Emit ''
call Emit 'Possible state-changing actions after confirmation:'

if cpuAvg >= 90 & distribution = 'CONCENTRATED' then do
  call Emit '  CANCEL <jobname>'
  call Emit '      Consider only for a confirmed runaway batch job'
  call Emit '      after application/change approval.'
  call Emit ''
  call Emit '  RESET <jobname>,SRVCLASS=<service-class>'
  call Emit '      Consider only when WLM classification/importance'
  call Emit '      is confirmed to be inappropriate.'
end
else do
  call Emit '  No immediate state-changing action is suggested by'
  call Emit '  the current CPUDIAG evidence.'
end

call Emit ''
call Emit 'Never use FORCE as a routine first response to high CPU.'
return


Notes:
call Section 'IMPORTANT NOTES'
call Emit '  * CPUDIAG uses SDSF SYS and explicit DA ALL.'
call Emit '  * The first DA ALL read is a discarded warm-up sample.'
call Emit '  * DA CPU%, paging and SIO are SDSF interval measures.'
call Emit '  * UIC is interpreted only with other storage evidence.'
call Emit '  * SIO is activity, not device response-time evidence.'
call Emit '  * AvgMSU is supporting four-hour context only.'
call Emit '  * AvgMSU > SysMSU is not treated as a CPU-full flag.'
call Emit '  * No corrective commands are automatically issued.'
call Emit '  * Under ISPF the report opens in Browse.'
call Emit '  * The report data set is retained after Browse is exited.'
return


Footer:
call Emit ''
call Emit copies('=',78)
call Emit ' CPUDIAG completed. No system or workload changes were made.'
call Emit copies('=',78)
return


Section:
parse arg title
call Emit ''
call Emit title
call Emit copies('-',78)
return


SampleReport:
call Emit ''
call Emit copies('=',78)
call Emit ' CPUDIAG - CPU FIRST-RESPONSE DIAGNOSTIC'
call KV 'Author', 'SARATH S'
call KV 'Date', DisplayDate()
call KV 'Time', time('N')
call KV 'System', localSys
call KV 'Mode', 'SAMPLE'
call Emit copies('=',78)
call Emit ''
call Emit '*** SAMPLE REPORT - INCIDENT VALUES ARE FABRICATED ***'
call Emit '*** NO LIVE SDSF CPU OR DA DATA WAS COLLECTED          ***'

call Section 'TRIAGE SUMMARY'
call KV 'Severity', 'HIGH'
call KV 'CPU condition', 'SUSTAINED VERY HIGH'
call KV 'Average sampled CPU', '97.4%'
call KV 'Minimum sampled CPU', '95.0%'
call KV 'Peak sampled CPU', '99.0%'
call KV 'Samples >= 90%', '5 of 5'
call KV 'Samples >= 95%', '5 of 5'
call KV 'CPU distribution', 'CONCENTRATED'
call KV 'Storage-pressure signal', 'NO PAGING PRESSURE OBSERVED'
call KV 'Top 3 share of busy CPU', '68.2%'
call KV 'AvgMSU / SysMSU', '94.4%'
call KV 'Four-hour MSU assessment', 'HIGH FOUR-HOUR MSU LOAD'

call Section 'CURRENT RESOURCE STATUS'
call KV 'Current CPU busy', '98%'
call KV 'Processor MSU rating', '640'
call KV 'System image MSU rating', '500'
call KV 'Four-hour Avg MSU', '472'
call KV 'AvgMSU / SysMSU', '94.4%'
call KV 'IPL date/time', '18-AUG-2026 04:15:22'
call KV 'Days since IPL', '1'
call KV 'Four-hour MSU mature', 'YES'
call KV 'Online general CPUs', '8'
call KV 'Online zIIPs', '2'
call KV 'Paging rate', '0'
call KV 'High unreferenced interval', '248'
call KV 'SIO rate', '12840'
call KV 'Aux storage used', '14%'
call KV 'Real storage used', '79%'
call KV 'Real frames online', '25000000'
call KV 'Real frames available', '5250000'
call KV 'Active started tasks', '318'
call KV 'Active batch jobs', '84'
call KV 'Active TSO users', '37'
call KV 'Outstanding WTORs', '1'

call Section 'RESOURCE CORRELATION / SECONDARY EVIDENCE'
call KV 'Storage-pressure signal', 'NO PAGING PRESSURE OBSERVED'
call KV 'Average paging rate', '0'
call KV 'Peak paging rate', '0'
call KV 'Paging-active samples', '0 of 5'
call KV 'Minimum sampled UIC', '244'
call KV 'Peak real storage used', '80.0%'
call KV 'Minimum available frames', '5100000'
call KV 'Peak auxiliary used', '14.0%'
call KV 'Average SIO rate', '12792'
call KV 'Peak SIO rate', '13240'
call Emit ''
call Emit 'Interpretation:'
call Emit '  CPU is persistently high, but no paging pressure is'
call Emit '  visible. Current evidence favors processor demand over'
call Emit '  a real-storage shortage.'

call Section 'KEY FINDINGS / INTERPRETATION'
call Emit '1. CPU remained between 95% and 99% for all five samples.'
call Emit '2. PAYJOB1 and DB2MSTR are persistent top CPU consumers.'
call Emit '3. Paging remained at zero and real storage stayed stable.'
call Emit '4. Four-hour AvgMSU is high relative to SysMSU.'
call Emit '5. Evidence supports sustained processor demand rather'
call Emit '   than a momentary CPU spike or paging-driven slowdown.'
call Emit ''
call Emit 'Probable impact:'
call Emit '  Batch elapsed time and lower-priority workloads may'
call Emit '  experience delayed dispatching while CPU remains full.'

call Section 'TOP CPU CONSUMERS - ALL ADDRESS SPACES'
call Emit 'Rank Jobname  Type JobID     ASID AvgCPU PeakCPU Seen'
call Emit copies('-',65)
call Emit '   1 PAYJOB1  JOB  JOB12345 01A2  28.4%   31.2%  5/5'
call Emit '   2 DB2MSTR  STC  STC00010 00B4  24.8%   27.1%  5/5'
call Emit '   3 CICSAA   STC  STC00022 00D2  15.0%   16.4%  5/5'
call Emit '   4 SORTX    JOB  JOB12346 0130   5.8%   18.9%  2/5'

call Section 'TOP CPU CONSUMING BATCH JOBS'
call Emit 'Rank Jobname  Type JobID     ASID AvgCPU PeakCPU Seen'
call Emit copies('-',65)
call Emit '   1 PAYJOB1  JOB  JOB12345 01A2  28.4%   31.2%  5/5'
call Emit '   2 SORTX    JOB  JOB12346 0130   5.8%   18.9%  2/5'

call Section 'TOP CPU CONSUMING STARTED TASKS'
call Emit 'Rank Jobname  Type JobID     ASID AvgCPU PeakCPU Seen'
call Emit copies('-',65)
call Emit '   1 DB2MSTR  STC  STC00010 00B4  24.8%   27.1%  5/5'
call Emit '   2 CICSAA   STC  STC00022 00D2  15.0%   16.4%  5/5'

call Section 'TOP CPU CONSUMING TSO USERS'
call Emit 'No significant TSO CPU consumer in this sample incident.'

call Section 'RECENT RELATED EVENTS'
call Emit 'Not collected by the current CPUDIAG live diagnostic.'
call Emit 'For this SAMPLE, assume the reported slowdown began shortly'
call Emit 'after PAYJOB1 started. This statement is fabricated.'

call Section 'RECOMMENDED DEBUGGING'
call Emit '1. Review PAYJOB1 and confirm whether its CPU demand is'
call Emit '   expected for the current processing window.'
call Emit '2. Review DB2MSTR activity with the DB2 support team if the'
call Emit '   CPU level is unusual for the workload.'
call Emit '3. Confirm processor configuration with D M=CPU.'
call Emit '4. Review active WLM policy using D WLM.'
call Emit '5. Review sysplex WLM state using D WLM,SYSTEMS.'
call Emit '6. Use D A,LIST for an independent active-work view.'
call Emit '7. Use RMF for precise performance analysis if required.'

call Section 'POSSIBLE MITIGATION / OPERATOR COMMANDS'
call Emit 'The following commands are NOT issued by CPUDIAG.'
call Emit 'They are shown so the operator knows possible next actions.'
call Emit ''
call Emit 'Read-only verification:'
call Emit '  D M=CPU'
call Emit '  D WLM'
call Emit '  D WLM,SYSTEMS'
call Emit '  D A,LIST'
call Emit ''
call Emit 'State-changing examples - use only after confirmation:'
call Emit '  CANCEL <jobname>'
call Emit '      For a confirmed runaway batch job after approval.'
call Emit ''
call Emit '  RESET <jobname>,SRVCLASS=<service-class>'
call Emit '      If executing work is confirmed in an inappropriate'
call Emit '      WLM service class and the change is approved.'
call Emit ''
call Emit 'Do not use FORCE as a routine first response to high CPU.'

call Section 'SUPPORTING DATA - CPU / RESOURCE SAMPLE HISTORY'
call Emit 'Samp Time      CPU% PageRate   UIC Real% Aux%      SIO'
call Emit copies('-',65)
call Emit '   1 10:15:00  96.0%        0   252   78%  14%    12410'
call Emit '   2 10:15:10  98.0%        0   249   79%  14%    12780'
call Emit '   3 10:15:20  99.0%        0   247   79%  14%    13240'
call Emit '   4 10:15:30  99.0%        0   244   80%  14%    12990'
call Emit '   5 10:15:40  95.0%        0   248   79%  14%    12540'

call Section 'IMPORTANT NOTES'
call Emit '  * This is a SAMPLE report. Incident values are fabricated.'
call Emit '  * The System value above is the actual local LPAR name.'
call Emit '  * SAMPLE does not query live SDSF SYS or DA information.'
call Emit '  * Commands shown are genuine operator investigation/action'
call Emit '    patterns, but CPUDIAG does not issue them automatically.'
call Emit '  * Never execute a state-changing command solely because'
call Emit '    it appears in this demonstration report.'
call Emit ''
call Emit copies('=',78)
call Emit ' SAMPLE completed. No system or workload changes were made.'
call Emit copies('=',78)
return


HelpReport:
call Emit ''
call Emit copies('=',78)
call Emit ' CPUDIAG - README / HELP'
call Emit copies('=',78)
call Emit ''
call Emit 'WHAT IS CPUDIAG?'
call Emit copies('-',78)
call Emit 'CPUDIAG is a read-only z/OS CPU first-response diagnostic'
call Emit 'utility. It automates common SDSF checks used for system'
call Emit 'slowness, high CPU, or jobs suspected of running long'
call Emit 'because processor resources may be under pressure.'
call Emit ''
call Emit 'WHAT DOES IT CHECK?'
call Emit copies('-',78)
call Emit '  * Repeated system CPU samples, not one snapshot.'
call Emit '  * Sustained versus intermittent high CPU.'
call Emit '  * Top CPU-consuming jobs, STCs and TSO address spaces.'
call Emit '  * Paging rate, UIC, real and auxiliary storage.'
call Emit '  * SIO activity as supporting I/O context.'
call Emit '  * MSU, SysMSU and four-hour rolling AvgMSU.'
call Emit ''
call Emit 'HOW DOES IT WORK?'
call Emit copies('-',78)
call Emit 'The default run takes five samples at ten-second intervals.'
call Emit 'The first DA ALL read is discarded as a warm-up. Remaining'
call Emit 'samples are combined to identify persistent CPU consumers.'
call Emit ''
call Emit 'USAGE'
call Emit copies('-',78)
call Emit '  TSO CPUDIAG'
call Emit '  TSO CPUDIAG topRows'
call Emit '  TSO CPUDIAG topRows samples intervalSeconds'
call Emit '  TSO CPUDIAG SAMPLE'
call Emit '  TSO CPUDIAG HELP'
call Emit '  TSO CPUDIAG H'
call Emit '  TSO CPUDIAG ?'
call Emit ''
call Emit 'DEFAULTS / LIMITS'
call Emit copies('-',78)
call KV 'Top rows', 'Default 10. Allowed 1-25.'
call KV 'Samples', 'Default 5. Allowed 3-12.'
call KV 'Sample interval', 'Default 10 sec. Allowed 5-60 sec.'
call Emit ''
call Emit 'EXAMPLE'
call Emit copies('-',78)
call Emit '  TSO CPUDIAG 10 3 5'
call Emit '  Quick test: 10 rows, 3 samples, 5-second interval.'
call Emit ''
call Emit '  TSO CPUDIAG SAMPLE'
call Emit '  Generate a demonstration report with fabricated CPU data.'
call Emit '  SAMPLE uses the actual local system name but does not'
call Emit '  collect live SDSF CPU or address-space measurements.'
call Emit ''
call Emit 'REPORT OUTPUT'
call Emit copies('-',78)
call Emit 'Under ISPF, CPUDIAG writes the report to a uniquely named'
call Emit 'sequential data set and opens it in Browse.'
call Emit 'F7/F8 scroll and F9 SWAP work normally.'
call Emit ''
call Emit 'The report data set is RETAINED after Browse is exited.'
call Emit 'Its full data set name is displayed before Browse opens.'
call Emit ''
call Emit 'SAFETY'
call Emit copies('-',78)
call Emit 'CPUDIAG is read-only. It does not cancel, start, stop or'
call Emit 'modify workloads, WLM, JES or system configuration.'
call Emit 'Run one normal invocation at a time per LPAR.'
call Emit ''
call Emit 'LIMITATIONS'
call Emit copies('-',78)
call Emit '  * SDSF access is required.'
call Emit '  * It is a first-response diagnostic, not a replacement'
call Emit '    for detailed RMF performance or capacity analysis.'
call Emit '  * AvgMSU is supporting four-hour context only.'
call Emit '  * AvgMSU > SysMSU is not treated as a CPU-full flag.'
call Emit ''
call Emit copies('=',78)
return


KV:
procedure expose rpt. rptCount
parse arg label, value
labelWidth = 28
line = ' 'left(label,labelWidth)' : 'value
call Emit line
return


/*====================================================================*/
/* HELPERS                                                            */
/*====================================================================*/

TopAverageSum:
parse arg howMany
if daValid = 0 then return 0

picked. = 0
total = 0

do rank = 1 to howMany
  best = 0
  bestAvg = -1

  do i = 1 to aggCount
    if picked.i then iterate
    avg = aSum.i / daValid

    if avg > bestAvg then do
      best = i
      bestAvg = avg
    end
  end

  if best = 0 then leave
  if bestAvg <= 0 then leave

  picked.best = 1
  total = total + bestAvg
end
return total


MSUMature:
if datatype(strip(lastIPLDAYS),'N') then do
  if Num(lastIPLDAYS) >= 1 then return 1

  if Num(lastIPLDAYS) = 0 then do
    t = right(strip(lastIPLDATE),8)
    sec = ClockSeconds(t)

    if sec >= 0 then do
      age = time('S') - sec
      if age < 0 then age = age + 86400

      if age >= 14400 then return 1
      return 0
    end
  end
end

/* If IPL age cannot be parsed, show AvgMSU but do not suppress it. */
return 1


ClockSeconds:
procedure
parse arg t
t = strip(t)

if length(t) <> 8 then return -1
if substr(t,3,1) <> ':' then return -1
if substr(t,6,1) <> ':' then return -1

hh = substr(t,1,2)
mm = substr(t,4,2)
ss = substr(t,7,2)

if datatype(hh,'W') = 0 then return -1
if datatype(mm,'W') = 0 then return -1
if datatype(ss,'W') = 0 then return -1

if hh > 23 then return -1
if mm > 59 then return -1
if ss > 59 then return -1

return hh * 3600 + mm * 60 + ss


SleepSeconds:
parse arg seconds
remain = seconds

do while remain > 0
  Address SYSCALL "sleep" remain

  if symbol('RETVAL') = 'LIT' then leave

  if datatype(retval,'W') then remain = retval
  else remain = 0
end
return


ValueOr:
procedure
parse arg v, fallback
v = strip(v)
if v = '' then return fallback
return v


Num:
procedure
parse arg v
v = strip(v)

if v = '' then return 0
if right(v,1) = '%' then v = left(v,length(v)-1)

if datatype(v,'N') then return v
return 0


IsNum:
procedure
parse arg v
v = strip(v)
if v = '' then return 0
if right(v,1) = '%' then v = left(v,length(v)-1)
return datatype(v,'N')


FmtNum:
procedure
parse arg n
n = Num(n)
if n = trunc(n) then return strip(format(n,12,0))
return strip(format(n,12,1))


FormatPct:
procedure
parse arg n
n = Num(n)
return strip(format(n,7,1))'%'


AddPct:
procedure
parse arg v
v = strip(v)
if v = '' then return 'N/A'
if v = 'N/A' then return v
if right(v,1) = '%' then return v
return v'%'


YesNo:
procedure
parse arg v
if v then return 'YES'
return 'NO'


DisplayDate:
procedure
s = date('S')
yyyy = substr(s,1,4)
mm = substr(s,5,2)
dd = substr(s,7,2)
months = 'JAN FEB MAR APR MAY JUN JUL AUG SEP OCT NOV DEC'
return dd'-'word(months,mm + 0)'-'yyyy


SdsfMessages:
if symbol('ISFMSG') <> 'LIT' then do
  if strip(isfmsg) <> '' then call Emit ' SDSF:' isfmsg
end

if symbol('ISFMSG2.0') <> 'LIT' then do
  do sm = 1 to isfmsg2.0
    call Emit ' SDSF:' isfmsg2.sm
  end
end
return


/*====================================================================*/
/* REPORT PRESENTATION - SAME PATTERN AS JOBINFO                      */
/*====================================================================*/

Emit:
procedure expose rpt. rptCount
out = ''

do ei = 1 to arg()
  piece = arg(ei)

  if ei = 1 then out = piece
  else out = out piece
end

rptCount = rptCount + 1
rpt.rptCount = out
rpt.0 = rptCount
return


PresentReport:
procedure expose rpt. rptCount ispfActive reportMode
/* ZDIAG BATCH BYPASS */
if sysvar('SYSISPF') <> 'ACTIVE' then do
  if rptCount = 0 then return
  address TSO 'EXECIO 'rptCount' DISKW ZREPORT (STEM rpt. FINIS)'
  zdbw = rc
  if zdbw <> 0 then do
    say 'ZDIAG BATCH: ZREPORT write failed RC='zdbw
    exit 20
  end
  return
end
if rptCount = 0 then return

if \ispfActive then do
  do pi = 1 to rptCount
    say rpt.pi
  end
  return
end

uid = sysvar('SYSUID')
dsDate = date('S')
dsDate = substr(dsDate,3,6)
dsTime = space(translate(time('N'),'',':'),0)

if reportMode = 'SAMPLE' then
  rptDsn = uid'.CPUSAMP.D'dsDate'.T'dsTime
else if reportMode = 'HELP' then
  rptDsn = uid'.CPUHELP.D'dsDate'.T'dsTime
else
  rptDsn = uid'.CPUDRPT.D'dsDate'.T'dsTime

ddn = 'CPRPT001'

allocCmd = "ALLOC DA('"rptDsn"') F("ddn") NEW CATALOG"
allocCmd = allocCmd "TRACKS SPACE(5 5) DSORG(PS)"
allocCmd = allocCmd "RECFM(F B) LRECL(133) BLKSIZE(0)"

Address TSO allocCmd
aRc = rc

if aRc <> 0 then do
  do pi = 1 to rptCount
    say rpt.pi
  end
  return
end

Address TSO "EXECIO "rptCount" DISKW" ddn "(STEM rpt. FINIS)"
wRc = rc
Address TSO "FREE F("ddn")"

if wRc <> 0 then do
  Address TSO "DELETE '"rptDsn"'"

  do pi = 1 to rptCount
    say rpt.pi
  end
  return
end

Address ISPEXEC "CONTROL ERRORS RETURN"
say 'CPUDIAG report saved in:' rptDsn
Address ISPEXEC "BROWSE DATASET('"rptDsn"')"
bRc = rc
Address ISPEXEC "CONTROL ERRORS CANCEL"

if bRc > 4 then do
  say 'CPUDIAG: ISPF Browse failed RC='bRc

  do pi = 1 to rptCount
    say rpt.pi
  end
end
return


Cleanup:
if ussOn = 1 then do
  rc = syscalls('OFF')
  ussOn = 0
end

if sdsfOn = 1 then do
  rc = isfcalls('OFF')
  sdsfOn = 0
end
return


FatalSyntax:
say
say 'CPUDIAG: REXX syntax/runtime error.'
say 'Condition:' condition('C')
say 'Description:' condition('D')
say 'Source line:' sigl

if datatype(sigl,'W') then
  say 'Statement:' sourceline(sigl)

if symbol('SDSFON') <> 'LIT' then call Cleanup
exit 20
