1import * as _ from 'lodash-es'
2import { genRegCredSecName } from './dockRegF.mts'
3import { ptDir } from '../../serverF/ptDirF.mts'
4import { doesDockImg2Exist } from './dockRegApi.mts'
5import { write1File } from '../../serverF/libFs/write1FileF.mts'
6import { deployActionBuildsDf } from './deployActionGensDfF.mts'
7import { isKanikojob, getDfBuildLabel } from './dockerUtilsF.mts'
8import { getDockBuildCtx } from '../../serverF/dockNameF.mts'
9import { read1File } from '../../serverF/libFs/read1FileF.mts'
10import { liveSpawn } from '../../serverF/libFs/live1SpawnF.mts'
11import { liveSpawnThrow, printSpawnFailContext } from '../../serverF/fileutilsF/live1SpawnThrowF.mts'
12
13import { envHome } from '../../sharedF/isDevPcF.mts'
14import { getReqJsonSecret } from '../../serverF/secretsF/getSecretF.mts'
15import { sleep } from '../../sharedF/sleepF.mts'
16import { kubeJobKaniko } from '../k8sF/kanikoF/epKubeJobKanikoF.mts'
17import { getAppCfg } from '../k8sF/ctxF/appCfgCtxF.mts'
18import { getKlusterCtx } from '../k8sF/ctxF/klusterCtxF.mts'
19import { throwIf } from '../../sharedF/throwErrorF/throwIfF.mts'
20import { assertDefined } from '../../sharedF/assertsF/assertDefinedF.mts'
21import { rewriteDfHostForLocalBuild, getDockerPushHost } from './getDockerPushHostAI.mts'
22import { genLocalDockerCtxTar } from './localDockerCtxTarAI.mts'
23import { ensureVmLanRoute } from './ensureVmLanRouteAI.mts'
24import { getDockBuildCaps, getDockBuildPrefix } from './dockImplCapsAI.mts'
25import { createRegistry, deleteFailedKanikoJobs } from './dockRegHealthAI.mts'
26import { isRegistryDiskFullError, isRegistryNotFoundError } from './dockRegErrorsAI.mts'
27import { regcacheGcAndRestart } from '../k8sF/initialSetupF/regcacheF/regcacheGcAndRestartAI.mts'
28import { throwDebugH } from '../../serverF/throwErrorF/throwDebugHF.mts'
29import type { absFileDirPath } from '../../ptDirF.mts'
30import type { secretNameType } from '../../serverF/secretsF/getPlainSecAI.mts'
31
32const dockerBuildTimeoutMs = 15 * 60 * 1000
33const dockerPushTimeoutDefaultMs = 45 * 60 * 1000
34
35const restartDockerDesktop = async () => {
36  if (process.platform !== 'darwin') return false
37  console.log('Restarting Docker Desktop to fix QEMU emulation...')
38  await liveSpawn({cmd: `osascript -e 'tell application "Docker Desktop" to quit'`, isQuiet: true})
39  await sleep(2000)
40  await liveSpawn({cmd: `open -a "Docker Desktop"`, isQuiet: true})
41  for (let i = 0; i < 90; i++) {
42    const {isSuccess} = await liveSpawn({cmd: 'docker info', isQuiet: true})
43    if (isSuccess) {
44      console.log(`Docker Desktop ready after ${i}s`)
45      return true
46    }
47    await sleep(1000)
48  }
49  console.log('Docker Desktop failed to restart in time')
50  return false
51}
52
53export const dockerBuildPush = async ({reqDockPush, ...props}: {
54  reqDockPush?: boolean, cluster_name?: string, dockreg_host?: string, dockLanHost?: string,
55  repo_name_tag: string, dockerfileContent: string, tarSpecsA?: object[],
56  git_sha?: string, git_repo_name?: string, action?: string, name?: string, dfLabel?: string, includeShaInTag?: boolean,
57  gitsshHost?: string, gitsshRepoPath?: string,
58}) => {
59  let {cluster_name, dockreg_host, dockLanHost, repo_name_tag, dockerfileContent, tarSpecsA, git_sha, action, name, dfLabel, includeShaInTag} = props
60
61  if (action == 'builddf') {
62    reqDockPush = false
63  }
64
65  if (deployActionBuildsDf({action})) {
66    const builtDockerImageExists = await doesDockImg2Exist({git_sha, dockerfileContent, reqDockPush, includeShaInTag, repo_name_tag})
67    if (builtDockerImageExists) return {imageReused: true}
68  }
69
70  if (!deployActionBuildsDf({action})) return
71
72  const {dockregPoolA} = getKlusterCtx()
73  const registryPool = dockregPoolA?.length ? dockregPoolA : [await getDockerPushHost({dockreg_host, dockLanHost, builderClusterName: isKanikojob() ? cluster_name : undefined})]
74
75  const rewriteRepoNameTag = (tag: string, newRegistry: string) => {
76    const parts = tag.split('/')
77    if (parts.length >= 2) {
78      parts[0] = newRegistry
79      return parts.join('/')
80    }
81    return tag
82  }
83
84  const tryBuildPush = async (registry: string) => {
85    const tag = rewriteRepoNameTag(repo_name_tag, registry)
86    if (!isKanikojob()) {
87      await doLocalDockerBuild({dockreg_host: registry, dockLanHost, repo_name_tag: tag, dockerfileContent, tarSpecsA, action, name, dfLabel, reqDockPush})
88    } else {
89      await doKanikoBuild({dockreg_host: registry, dockLanHost, repo_name_tag: tag, dockerfileContent, tarSpecsA, git_sha, action, name, dfLabel, cluster_name: cluster_name!})
90    }
91  }
92
93  let diskFullRecovered = false
94  for (const currentRegistry of registryPool) {
95    try {
96      await tryBuildPush(currentRegistry)
97      return
98    } catch (err) {
99      const isDiskFull = isRegistryDiskFullError(err)
100      if (isDiskFull) {
101        if (!diskFullRecovered) {
102          const {regcacheEnabled, regcacheHost, regcacheLanHost} = getKlusterCtx()
103          if (regcacheEnabled && regcacheHost) {
104            console.log(`\n>>> Registry ${currentRegistry} disk full — sweeping stale manifests + restarting regcache to reclaim blobs...`)
105            diskFullRecovered = true
106            const recovered = await regcacheGcAndRestart({regcacheHost, regcacheLanHost, cluster_name})
107            if (recovered) {
108              try { await tryBuildPush(currentRegistry); return }
109              catch (err2) { if (!isRegistryDiskFullError(err2) && !isRegistryNotFoundError(err2)) throw err2 }
110            }
111          }
112        }
113        continue
114      }
115
116      const isNotFound = isRegistryNotFoundError(err)
117      if (isNotFound) {
118        console.log(`\n>>> Registry ${currentRegistry} not found, creating...`)
119        if (name) await deleteFailedKanikoJobs({name, cluster_name: cluster_name!})
120        const created = await createRegistry({dockreg_host: currentRegistry, cluster_name: cluster_name!})
121        if (created) {
122          try { await tryBuildPush(currentRegistry); return }
123          catch (err2) { if (!isRegistryDiskFullError(err2) && !isRegistryNotFoundError(err2)) throw err2 }
124        } else {
125          console.log(`\n>>> Registry ${currentRegistry} creation failed, trying next...`)
126        }
127      } else {
128        throw err
129      }
130    }
131  }
132
133  throwDebugH({reason: 'all registries exhausted', dockregPoolA: registryPool})
134}
135
136type LocalDockerBuildProps = {
137  dockreg_host: string
138  dockLanHost?: string
139  repo_name_tag: string
140  dockerfileContent: string
141  tarSpecsA?: object[]
142  action?: string
143  name?: string
144  dfLabel?: string
145  reqDockPush?: boolean
146}
147
148const doLocalDockerBuild = async ({dockreg_host, dockLanHost, repo_name_tag, dockerfileContent, tarSpecsA, action, name, dfLabel, reqDockPush}: LocalDockerBuildProps) => {
149  const {dockName} = getDockBuildCtx()
150  if (action == 'delete') return
151
152  dockerfileContent = await rewriteDfHostForLocalBuild(dockerfileContent)
153
154  await ensureVmLanRoute({hostNamesA: [dockreg_host, dockLanHost]})
155
156  assertDefined(dockreg_host)
157  const dockConfigPath = [envHome, '.docker/config.json'].join('/') as absFileDirPath
158  const dockerConfigJson = JSON.parse(await read1File(dockConfigPath))
159  const authH = getReqJsonSecret(genRegCredSecName(dockreg_host) as secretNameType)
160  _.set(dockerConfigJson, ['auths', dockreg_host], authH)
161  if (dockLanHost) _.set(dockerConfigJson, ['auths', dockLanHost], authH)
162  dockerConfigJson.credsStore = ''
163
164  await write1File(dockConfigPath, JSON.stringify(dockerConfigJson, null, 2))
165
166  // minimal context = Dockerfile + the baked tmpdockertars/*.tar.gz it ADDs (mirrors kaniko's
167  // localctx); fall back to full ptDir only for standalone/raw-COPY Dockerfiles with no tarSpecs.
168  const isStandalone = !tarSpecsA?.length && !/^\s*(COPY|ADD)\b/im.test(dockerfileContent)
169  const ctxTar = (tarSpecsA?.length && !isStandalone)
170    ? await genLocalDockerCtxTar({dockerfileContent, tarSpecsA: tarSpecsA as {tarPath: string}[]})
171    : null
172
173  const tryBuild = async () => {
174    const buildLabel = getDfBuildLabel({dfLabel, name, repo_name_tag})
175    console.log(`${buildLabel} build`)
176    const progressFlag = getDockBuildCaps(dockName).supportsProgress ? ' --progress plain' : ''
177    let cmd = `${getDockBuildPrefix(dockName)}${progressFlag} --platform=linux/amd64 -t ${repo_name_tag}`
178    if (ctxTar) {
179      cmd += ` - < ${ctxTar}`
180    } else {
181      cmd += ` -f - ${ptDir} <<'EOF'
182${dockerfileContent}
183EOF`
184    }
185    const ret = await liveSpawn({cmd, noOutCmd: true, timeoutAfterSec: Math.round(dockerBuildTimeoutMs/1000)})
186    if (!reqDockPush) return ret
187    const {isSuccess} = ret
188    if (isSuccess) {
189      console.log(`${buildLabel} push`)
190      const pushTimeoutMs = (getAppCfg() as Record<string, unknown>)?.dockerPushTimeoutMs as number || dockerPushTimeoutDefaultMs
191      await liveSpawnThrow({cmd: `${dockName} push ${repo_name_tag}`, noOutCmd: true, timeoutAfterSec: Math.round(pushTimeoutMs/1000)})
192    }
193    return ret
194  }
195  let res = await tryBuild()
196  let {stderr, isSuccess, stdout} = res
197  if (isSuccess) return
198  const combinedOutput = stdout + stderr
199  const isExecFormatErr = combinedOutput.includes('exec /bin/sh: exec format error')
200  const isSegfault = res.exitCode === 139
201  const isLocalDiskFull = stderr.includes('no space left on device') || stdout.includes('no space left on device') || stdout.includes(`You don't have enough free space in`)
202  
203  // Check if this is a registry disk full (during push) vs local disk full (during build)
204  // Registry disk full errors typically mention the registry host or have specific patterns
205  const isRegistryFull = isLocalDiskFull && (combinedOutput.includes('Err:28') || combinedOutput.includes('blobs/uploads'))
206  if (isRegistryFull) {
207    throw new Error(`Registry disk full: ${combinedOutput.slice(-500)}`)
208  }
209
210  if (
211    stderr.includes('At least one invalid signature was encountered.') ||
212    isLocalDiskFull ||
213    isExecFormatErr ||
214    isSegfault
215  ) {
216    if (isSegfault) console.log('Segmentation fault detected, retrying...')
217    else if (isExecFormatErr) console.log('Detected architecture mismatch, pruning build cache and retrying...')
218    await liveSpawnThrow({cmd: `${dockName} system prune --force && ${dockName} builder prune --force && ${dockName} image prune -a -f`})
219    ;({isSuccess} = await tryBuild())
220    if (!isSuccess && isExecFormatErr) {
221      const restarted = await restartDockerDesktop()
222      if (restarted) {
223        ;({isSuccess} = await tryBuild())
224      }
225    }
226  }
227  if (!isSuccess) {
228    printSpawnFailContext(res, true)
229    throw 'DockerBuildFailed'
230  }
231}
232
233type KanikoBuildProps = {
234  dockreg_host: string
235  dockLanHost?: string
236  repo_name_tag: string
237  dockerfileContent: string
238  tarSpecsA?: object[]
239  git_sha?: string
240  action?: string
241  name?: string
242  dfLabel?: string
243  cluster_name: string
244}
245
246const doKanikoBuild = async ({dockreg_host, dockLanHost, repo_name_tag, dockerfileContent, tarSpecsA, git_sha, action, name, dfLabel, cluster_name}: KanikoBuildProps) => {
247  const kanikoLocalContext = getAppCfg()?.kanikoLocalContext ?? true
248  if (!kanikoLocalContext) {
249    const {gitsshHost: gitsshHostCfg, git_repo_name: gitRepoNameCfg} = getAppCfg()
250    throwIf(() => !(gitsshHostCfg || process.env.GITSSH_HOST || gitRepoNameCfg), {gitsshHostCfg, gitRepoNameCfg})
251  }
252
253  assertDefined(name)
254  assertDefined(cluster_name)
255  assertDefined(dockreg_host)
256
257  await kubeJobKaniko({name, dockerfileContent, tarSpecsA: tarSpecsA as any, repo_name_tag, dfLabel: dfLabel || 'dfbuild', cluster_name, dockreg_host, dockLanHost, action, git_sha})
258}
259