1import * as _ from 'lodash-es'
2import { getCreds } from './dockRegF.mts'
3import { deleteRegManifest, getRegCatalog, getRegDigest, getRegTags, resolveRegApiHost } from './regApiCoreF.mts'
4import { getDockRegKubeResA } from './dockRegF/getDockRegKubeResAF.mts'
5import { isRegistryDiskFullError } from './dockRegErrorsAI.mts'
6import { getKlusterCtx } from '../k8sF/ctxF/klusterCtxF.mts'
7import { getKubeApis } from '../k8sF/getApisF.mts'
8import { getMostRecentPod } from '../k8sF/podsF/k8sPodsF.mts'
9import { getKubeConfigPath } from '../../serverF/kubePortFwdF/kubeConfigPathAI.mts'
10import { spawnSync } from 'child_process'
11import { chalkGreen, chalkYellow } from '../../serverF/libChalkF.mts'
12import { BatchV1Api } from '@kubernetes/client-node'
13
14type DockRegGcProps = {
15  dockreg_host: string
16  dockLanHost?: string
17}
18
19const getDiskUsage = (cluster_name: string, podName: string) => {
20  const result = spawnSync('kubectl', ['exec', podName, '--', 'sh', '-c', 'df -h /var/lib/registry | awk \'NR>1{for(i=1;i<=NF;i++) if($i ~ /%/) {pct=$i; used=$(i-2); total=$(i-3)}} END{print total, used, pct}\''], {
21    encoding: 'utf-8', env: {...process.env, KUBECONFIG: getKubeConfigPath(cluster_name)},
22  })
23  const parts = (result.stdout || '').trim().split(/\s+/)
24  return {used: parts[1] || '?', total: parts[0] || '?', pct: parts[2] || '?'}
25}
26
27export const sweepStaleRegManifests = async ({apiHost, altHost, activeImageRefs, auth}: {
28  apiHost: string, altHost?: string, activeImageRefs: Set<string>, auth?: string
29}) => {
30  const catalogs = await getRegCatalog({apiHost, auth})
31  console.log(`${catalogs.length} repos in registry`)
32
33  let staleCount = 0, deletedCount = 0, failedCount = 0
34
35  for (const repo of catalogs) {
36    const tags = await getRegTags({apiHost, repo, auth})
37    for (const tag of tags) {
38      const fullRef = `${apiHost}/${repo}:${tag}`
39      const altRef = `${altHost}/${repo}:${tag}`
40      const isActive = _.some([...activeImageRefs], ref => ref === fullRef || ref === altRef || ref.includes(`${repo}:${tag}`))
41      if (isActive) continue
42
43      const digest = await getRegDigest({apiHost, repo, tag, auth})
44      if (!digest) { failedCount++; continue }
45
46      staleCount++
47      const ok = await deleteRegManifest({apiHost, repo, digest, auth})
48      if (ok) {
49        deletedCount++
50        console.log(`  deleted ${repo}:${tag}`)
51      } else {
52        failedCount++
53        console.log(chalkYellow(`  failed to delete ${repo}:${tag}`))
54      }
55    }
56  }
57
58  console.log(`\nstale manifests: ${staleCount}, deleted: ${deletedCount}, failed: ${failedCount}`)
59  return {staleCount, deletedCount, failedCount}
60}
61
62const runRegistryGc = (cluster_name: string, podName: string) => {
63  console.log('running registry garbage-collect...')
64  const result = spawnSync('kubectl', ['exec', podName, '--', 'registry', 'garbage-collect', '/etc/docker/registry/config.yml'], {
65    stdio: 'inherit', env: {...process.env, KUBECONFIG: getKubeConfigPath(cluster_name)},
66  })
67  return result.status === 0
68}
69
70const deleteDiskFullKanikoJobs = async ({cluster_name}: {cluster_name: string}) => {
71  const {kubeConfig} = await getKubeApis({cluster_name})
72  const batchApi = kubeConfig.makeApiClient(BatchV1Api)
73  const {items: jobs} = await batchApi.listNamespacedJob({namespace: 'default', labelSelector: `cluster_name=${cluster_name}`})
74  const kanikoJobs = jobs.filter(job => job.metadata?.name?.startsWith('kaniko2-') && job.status?.failed)
75  if (!kanikoJobs.length) return 0
76
77  let deletedCount = 0
78  for (const job of kanikoJobs) {
79    const jobName = job.metadata?.name!
80    const podLogs = spawnSync('kubectl', ['logs', '-l', `job-name=${jobName}`, '--tail=50', '-c', 'kaniko'], {
81      encoding: 'utf-8', env: {...process.env, KUBECONFIG: getKubeConfigPath(cluster_name)}, timeout: 15_000,
82    })
83    const logs = podLogs.stdout || ''
84    if (!isRegistryDiskFullError(logs)) continue
85    try {
86      await batchApi.deleteNamespacedJob({name: jobName, namespace: 'default', propagationPolicy: 'Background'})
87      console.log(`  deleted disk-full kaniko job ${jobName}`)
88      deletedCount++
89    } catch {}
90  }
91  return deletedCount
92}
93
94export const dockRegGc = async ({dockreg_host, dockLanHost}: DockRegGcProps) => {
95  const {cluster_name} = getKlusterCtx()
96  const regName = dockreg_host.split('.')[0]
97
98  const podName = await getMostRecentPod({cluster_name, name: regName})
99
100  const diskBefore = getDiskUsage(cluster_name, podName)
101  console.log(`disk before: ${diskBefore.used} / ${diskBefore.total} (${diskBefore.pct})`)
102
103  const activeImageRefs = new Set<string>()
104  const kubeResA = await getDockRegKubeResA({dockReg: regName})
105  for (const res of kubeResA) {
106    const containers = _.get(res, 'containers', [])
107    for (const c of containers) {
108      const image: string = c.image || ''
109      if (image.includes(regName)) activeImageRefs.add(image)
110    }
111  }
112  console.log(`${kubeResA.length} deployments using this registry, ${activeImageRefs.size} active image refs`)
113
114  const {auth} = getCreds({dockreg_host})
115  const apiHost = await resolveRegApiHost({host: dockreg_host, lanHost: dockLanHost, auth})
116
117  const {deletedCount} = await sweepStaleRegManifests({apiHost, altHost: dockreg_host, activeImageRefs, auth})
118
119  if (deletedCount > 0) {
120    const gcOk = runRegistryGc(cluster_name, podName)
121    if (!gcOk) console.log(chalkYellow('registry garbage-collect exited non-zero'))
122  } else {
123    console.log('no stale manifests to delete, skipping garbage-collect')
124  }
125
126  const deletedJobs = await deleteDiskFullKanikoJobs({cluster_name})
127  if (deletedJobs) console.log(`cleaned up ${deletedJobs} disk-full kaniko job(s)`)
128
129  const diskAfter = getDiskUsage(cluster_name, podName)
130  console.log(`\ndisk after:  ${diskAfter.used} / ${diskAfter.total} (${diskAfter.pct})`)
131
132  const freed = diskBefore.pct !== '?' && diskAfter.pct !== '?'
133  if (freed) {
134    console.log(chalkGreen(`freed: ${diskBefore.pct} → ${diskAfter.pct} usage`))
135  } else if (deletedCount === 0) {
136    console.log(chalkYellow('no space freed — all images are referenced by active deployments'))
137    console.log('consider: increase PVC size, redeploy apps to another registry, or manually delete stale deployments')
138  }
139}
140
141dockRegGc.cliDescript = 'delete unreferenced manifests and run registry garbage-collect'
142