1import * as _ from 'lodash-es'
2import { read2Resource } from './read2ResourceF.mts'
3import { getAppPods } from './podsF/k8sPodsF.mts'
4import { sleep } from '../../sharedF/sleepF.mts'
5import { debug1Sleep } from '../../serverF/sleepF/debugSleepF.mts'
6import { ret1PodLogs } from './logsF/retPodLogsF.mts'
7import { throwTimedOut, k8sWaitTimeoutMs, imagePullTimeoutMs } from './throwTimedOutAI.mts'
8import { chalkRed } from '../../serverF/libChalkF.mts'
9import { throwDebugH } from '../../serverF/throwErrorF/throwDebugHF.mts'
10import { mkProgressLogger } from '../cliF/cliProgressAI.mts'
11import { getPendingReason, pendingTimeoutMs, pvcPendingTimeoutMs } from './podPendingStatusAI.mts'
12import type { WithClusterAndResource } from './ctxF/klusterCtxF.mts'
13
14type WaitForRolloutProps = WithClusterAndResource & { rolloutTimeoutMs?: number }
15
16type PodStatusResult = 'InitError' | 'InitImageErr' | 'Creating' | 'Error' | 'ImageErr' | 'Pending' | 'Running' | 'Unknown'
17
18type ContainerState = {
19  running?: {startedAt?: string}
20  terminated?: {reason?: string, exitCode?: number, message?: string}
21  waiting?: {reason?: string, message?: string}
22}
23type ContainerStatus = {name?: string, state?: ContainerState}
24type PodCondition = {type: string, status: string, reason?: string, message?: string}
25type PodResource = {
26  metadata: {name: string, labels: {git_sha: string}}
27  status: {phase?: string, conditions?: PodCondition[], containerStatuses?: ContainerStatus[], initContainerStatuses?: ContainerStatus[]}
28}
29
30const getPodStatus = (pods: PodResource[]): PodStatusResult => {
31  for (const pod of pods) {
32    // Check init containers first (they run before main containers)
33    for (const cs of pod.status?.initContainerStatuses || []) {
34      const reason = cs.state?.waiting?.reason || cs.state?.terminated?.reason
35      if (reason === 'CrashLoopBackOff' || reason === 'Error') return 'InitError'
36      if (reason === 'ErrImagePull' || reason === 'ImagePullBackOff') return 'InitImageErr'
37    }
38    for (const cs of pod.status?.containerStatuses || []) {
39      const reason = cs.state?.waiting?.reason || cs.state?.terminated?.reason
40      if (reason === 'ContainerCreating') return 'Creating'
41      if (reason === 'CrashLoopBackOff' || reason === 'Error') return 'Error'
42      if (reason === 'ErrImagePull' || reason === 'ImagePullBackOff') return 'ImageErr'
43    }
44    if (pod.status?.phase === 'Pending') return 'Pending'
45    if (pod.status?.phase === 'Running') return 'Running'
46  }
47  return 'Unknown'
48}
49
50export const waitForDeploymentRollout = async ({resource, cluster_name, rolloutTimeoutMs}: WaitForRolloutProps) => {
51  const name = resource.metadata?.name
52  const effectiveTimeoutMs = rolloutTimeoutMs ?? k8sWaitTimeoutMs
53  const logProgress = mkProgressLogger(`awaitrollout ${name}`, effectiveTimeoutMs)
54  const startTime = Date.now()
55  let pendingStartTime: number | null = null
56  let debugLoggedAt: number | null = null
57  let prevStatus: PodStatusResult | null = null, statusChangedAt = startTime
58  while(true) {
59    let deployment = await read2Resource({resource, cluster_name}) as {
60      status: {updatedReplicas?: number, replicas?: number, unavailableReplicas?: number}
61      spec: {template: {metadata: {labels: {git_sha: string}}}}
62      metadata: {name: string}
63    } | undefined
64
65    if (!deployment) {
66      await sleep(2000)
67      deployment = await read2Resource({resource, cluster_name}) as typeof deployment
68    }
69    if (!deployment) {
70      throwDebugH({name, cluster_name, message: 'deployment not found after apply'})
71    }
72
73    const {status} = deployment
74    const {updatedReplicas, replicas, unavailableReplicas} = status
75
76    const pods = await getAppPods({cluster_name, name: deployment.metadata.name}) as PodResource[]
77    const curPods = _.filter(pods, ({metadata: {labels: {git_sha}}}) => git_sha == deployment.spec.template.metadata.labels.git_sha)
78    const oldPods = _.difference(pods, curPods)
79    const oldPodsLen = oldPods.length
80  
81    const runCurPodsLen = _.filter(curPods, (pod) => _.get(pod, 'status.phase') == 'Running').length
82
83    const curStatus = getPodStatus(curPods)
84
85    // Timeout from last state change; ContainerCreating (image pull) gets a longer budget —
86    // first multi-GB pull legitimately exceeds the flat 90s (k8s progressDeadline defaults to 600s)
87    if (curStatus !== prevStatus) { prevStatus = curStatus; statusChangedAt = Date.now() }
88    throwTimedOut(statusChangedAt, `awaitrollout ${name}`, curStatus === 'Creating' ? imagePullTimeoutMs : effectiveTimeoutMs)
89
90    // Fail fast if pods stuck in Pending for too long (scheduling/volume issues)
91    if (curStatus === 'Pending') {
92      pendingStartTime ||= Date.now()
93      const pendingPod = curPods.find(p => p.status?.phase === 'Pending')
94      const pendingInfo = getPendingReason(pendingPod as any)
95      const timeout = pendingInfo?.isPvcPending ? pvcPendingTimeoutMs : pendingTimeoutMs
96      if (Date.now() - pendingStartTime > timeout) {
97        console.error(chalkRed(`\nPending timeout: ${pendingPod?.metadata?.name}`))
98        console.error(chalkRed(`Reason: ${pendingInfo?.reason || 'Unknown'}`))
99        console.error(pendingInfo?.message || 'Pod stuck in Pending state')
100        throwDebugH({pendingPod: pendingPod?.metadata?.name, reason: pendingInfo?.reason, message: pendingInfo?.message})
101      }
102    } else {
103      pendingStartTime = null // reset if pod progressed past Pending
104    }
105
106    // Debug logging when approaching timeout (after 60s) and in Creating state
107    const elapsedMs = Date.now() - startTime
108    if (curStatus === 'Creating' && elapsedMs > 60_000 && (!debugLoggedAt || Date.now() - debugLoggedAt > 30_000)) {
109      debugLoggedAt = Date.now()
110      const creatingPods = curPods.filter(p => 
111        p.status?.containerStatuses?.some(cs => cs.state?.waiting?.reason === 'ContainerCreating'))
112      console.log(`\n[debug] Creating for ${Math.round(elapsedMs/1000)}s - deployment: updated=${updatedReplicas}/${replicas} unavail=${unavailableReplicas} oldPods=${oldPodsLen}`)
113      for (const pod of creatingPods) {
114        const conditions = pod.status?.conditions?.map(c => `${c.type}=${c.status}`).join(', ') || 'none'
115        console.log(`[debug] pod ${pod.metadata.name}: conditions=[${conditions}]`)
116      }
117    }
118    
119    if (updatedReplicas && updatedReplicas > 0) {
120      type CrashedContainer = {name: string, reason: string, hasLogs: boolean, isInit: boolean, message?: string}
121      const erroredPods = _.chain(curPods).map((resource) => {
122        // Check both init containers and main containers
123        const allStatuses = [
124          ...(resource.status.initContainerStatuses || []).map(s => ({...s, isInit: true})),
125          ...(resource.status.containerStatuses || []).map(s => ({...s, isInit: false})),
126        ]
127        const crashedContainers: CrashedContainer[] = []
128        for (const {name, state, isInit} of allStatuses) {
129          if (!name) continue
130          if (state?.running) continue
131          if (state?.terminated?.reason === 'Completed') continue // kaniko successful build
132          if (state?.waiting?.reason == 'ContainerCreating' || state?.waiting?.reason == 'PodInitializing') continue
133          const reason = state?.terminated?.reason || state?.waiting?.reason
134          if (reason == 'Error' || reason == 'CrashLoopBackOff') {
135            crashedContainers.push({name, reason, hasLogs: true, isInit})
136          } else if (reason == 'ErrImagePull' || reason == 'ImagePullBackOff') {
137            crashedContainers.push({name, reason, message: state?.waiting?.message || reason, hasLogs: false, isInit})
138          }
139          // Unknown state (e.g., Pending with no containerStatuses) - don't treat as error
140        }
141        if (crashedContainers.length == 0) return 
142        return {podName: resource.metadata.name, crashedContainers}
143      }).compact().value()
144  
145      const erroredPod = erroredPods[0]
146      if (erroredPod) {
147        const {podName, crashedContainers} = erroredPod
148        const firstContainer = crashedContainers[0]
149        const containerType = firstContainer.isInit ? 'init container' : 'container'
150        if (firstContainer.hasLogs) {
151          await debug1Sleep(2000, `showing logs for first failed pod ( ${podName} ): `)
152          await ret1PodLogs({
153            cluster_name, podName, containerName: firstContainer.name, follow: false, 
154            streamStdout: true, timestamps: true, pretty: true
155          })
156        } else {
157          console.error(`\nPod ${podName} ${containerType} ${firstContainer.name} failed: ${firstContainer.reason}`)
158          if (firstContainer.message) console.error(firstContainer.message)
159        }
160        throwDebugH({podName, containerType, container: firstContainer.name, reason: firstContainer.reason})
161      }
162    }
163
164    if (!unavailableReplicas && updatedReplicas == replicas && replicas && replicas > 0) {
165
166      const containerCreatingStatus = _.find(curPods, (pod) => {
167        return _.find(pod.status.containerStatuses, (containerStatus) => {
168          return _.get(containerStatus, 'state.waiting.reason') == 'ContainerCreating'
169        })
170      })
171      if (!containerCreatingStatus) {
172
173        const jsonStr = JSON.stringify(curPods, null, 2)
174        const debugContainerCreating = _.includes(jsonStr, 'ContainerCreating')
175        if (debugContainerCreating) {
176          console.log(jsonStr, {debugContainerCreating})
177          throw '2j3f0293jf0293fj'
178        }
179        // oldPodsLen==0 check removed: caused 90s timeouts waiting for old pod termination
180        // k8s handles traffic routing once new pods ready + unavailableReplicas==0
181        if (runCurPodsLen > 0) {
182          console.log(' ready')
183          break  
184        }
185      }
186    } // https://stackoverflow.com/questions/67981362/command-to-check-reconfigured-kubernetes-deployment-is-ready-or-not
187    logProgress(curStatus)
188    await sleep(10 * 1000)
189  }
190}
191