🌳 
$ ptnode pt0/devpconlyF/zhongF/epZhong.mjs --ignoreRecent

mcbackup { isRecent: true } 7m / 1d
â–¸ mcbackup
world unchanged since last backup, skipping download
✓ 0.173s mcbackup
backup2db_donatedbprod_qs { isRecent: true } 7m / 1d
â–¸ backup2db_donatedbprod_qs
doKubeExec hetzn1 exec -it donatedbprod-1 pg_dump --format=custom postgresql://app:****@donatedbprod-rw.default:5432/app?application_name=Petes-MBP.localdomain --file=/var/lib/postgresql/data/donatedbprod_1785511219840.sql -Upostgres --verbose
doKubeExecCapture hetzn1 exec donatedbprod-1 stat -c %s /var/lib/postgresql/data/donatedbprod_1785511219840.sql
kubectl exec -i donatedbprod-1 -c postgres -- cat /var/lib/postgresql/data/donatedbprod_1785511219840.sql
doKubeExec hetzn1 exec -it donatedbprod-1 rm /var/lib/postgresql/data/donatedbprod_1785511219840.sql
backed up 14.81kb ~/ptdisk/backups/donatedbprod/donatedbprod_1785511219840.sql
✓ 3.241s backup2db_donatedbprod_qs
backup2db_gameshetzdb_qs { isRecent: true } 7m / 1d
â–¸ backup2db_gameshetzdb_qs
doKubeExec hetzn1 exec -it gameshetzdb-1 pg_dump --format=custom postgresql://app:****@gameshetzdb-rw.default:5432/app?application_name=Petes-MBP.localdomain --file=/var/lib/postgresql/data/gameshetzdb_1785511223081.sql -Upostgres --verbose
doKubeExecCapture hetzn1 exec gameshetzdb-1 stat -c %s /var/lib/postgresql/data/gameshetzdb_1785511223081.sql
kubectl exec -i gameshetzdb-1 -c postgres -- cat /var/lib/postgresql/data/gameshetzdb_1785511223081.sql
kubeExecCat attempt 1/5 code=0 localSize=14417920 expectedSize=16587928
kubectl exec -i gameshetzdb-1 -c postgres -- tail -c +14417921 /var/lib/postgresql/data/gameshetzdb_1785511223081.sql
doKubeExec hetzn1 exec -it gameshetzdb-1 rm /var/lib/postgresql/data/gameshetzdb_1785511223081.sql
backed up 15.82mb ~/ptdisk/backups/gameshetzdb/gameshetzdb_1785511223081.sql
✓ 12.345s backup2db_gameshetzdb_qs
ipfs3pins { isRecent: true } 4m / 1h
â–¸ ipfs3pins
pinHealFail {
  label: 'donateapp/ipfs_mfsroot_donatedbprod_qs',
  cid: 'QmTc8jL3...',
  host: 'ipfs-api.harv3lan.demokluster.com',
  errMsg: '<html>\r\n<head><title>504 Gateway Time-out</title></head>\r\n<body>\r\n<center><h1>504 Gateway Time-out</h1></center>\r\n<hr><center>nginx</center>\r\n</body>\r\n</html>\r\n'
}
err.toString(): Error: ipfsMfsRootMissing err.stack: PtErr: ipfsMfsRootMissing
at noThrowNotifErr (pt0/serverF/throwErrorF/noThrowNotifErrF.mts:10)
at verifyIpfsPins (pt0/ipfsF/verifyIpfsPinsAI.mts:49)
at runFnc (pt0/ipfsF/ipfsPinsJobAI.mts:30)
at zhongRunOnce (pt0/devpconlyF/zhongF/zhongF.mts:101)
at eptZhong (pt0/devpconlyF/zhongF/zhongF.mts:66)
at <anonymous> (pt0/devpconlyF/zhongF/epZhong.mjs:64) {
  uniqDebugH: {
    totalRoots: 2,
    hosts: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ],
    pinnedOnAll: 1,
    missing: 0,
    healed: 0,
    perRoot: {
      'gamesapp/ipfs_mfsroot_gameshetzdb_qs': { cid: 'QmSB4kGs...', pinnedOn: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ], missingOn: [], healed: [] },
      'donateapp/ipfs_mfsroot_donatedbprod_qs': { cid: 'QmTc8jL3...', pinnedOn: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com' ], missingOn: [ 'ipfs-api.harv3lan.demokluster.com' ], healed: [] }
    },
    eeToken: '[demo]-prod'
  },
  dumpDebugH: {},
  envCtx: { gpIsProd: undefined, appname: 'unknown', deploymentName: '', git_sha: 'devvv', hostname: 'Petes-MBP.localdomain', deployEp: '' },
  reqCtx: {},
  errCtx: { errHash: '730daf7bf3', backoffNo: 5, numSilenced: 1, isRlSilenced: true }
}
ipfsPinVerifyResult {
  totalRoots: 2,
  hosts: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ],
  pinnedOnAll: 1,
  missing: 0,
  healed: 0,
  perRoot: {
    'gamesapp/ipfs_mfsroot_gameshetzdb_qs': { cid: 'QmSB4kGs...', pinnedOn: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ], missingOn: [], healed: [] },
    'donateapp/ipfs_mfsroot_donatedbprod_qs': { cid: 'QmTc8jL3...', pinnedOn: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com' ], missingOn: [ 'ipfs-api.harv3lan.demokluster.com' ], healed: [] }
  }
}
mfsVerifyHealed { label: 'donateapp/ipfs_mfsroot_donatedbprod_qs', host: 'ipfs-api.harv3lan.demokluster.com', copiedCount: 2 }
mfsVerifyCidMismatch { label: 'donateapp/ipfs_mfsroot_donatedbprod_qs', host: 'ipfs-api.harv3lan.demokluster.com', primaryCid: 'QmTc8jL3...', secondaryCid: 'QmSTDQrp...' }
err.toString(): Error: mfsReplicationMismatch err.stack: PtErr: mfsReplicationMismatch
at noThrowNotifErr (pt0/serverF/throwErrorF/noThrowNotifErrF.mts:10)
at verifyMfsReplication (pt0/ipfsF/verifyMfsReplicationAI.mts:94)
at runFnc (pt0/ipfsF/ipfsPinsJobAI.mts:31)
at zhongRunOnce (pt0/devpconlyF/zhongF/zhongF.mts:101)
at eptZhong (pt0/devpconlyF/zhongF/zhongF.mts:66)
at <anonymous> (pt0/devpconlyF/zhongF/epZhong.mjs:64) {
  uniqDebugH: {
    totalRoots: 2,
    synced: 1,
    healed: 1,
    mismatches: 1,
    perRoot: {
      'gamesapp/ipfs_mfsroot_gameshetzdb_qs': { primaryCid: 'QmSB4kGs...', synced: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ], healed: [], mismatches: [] },
      'donateapp/ipfs_mfsroot_donatedbprod_qs': {
        primaryCid: 'QmTc8jL3...',
        synced: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com' ],
        healed: [ 'ipfs-api.harv3lan.demokluster.com' ],
        mismatches: [ 'ipfs-api.harv3lan.demokluster.com' ]
      }
    },
    eeToken: '[demo]-prod'
  },
  dumpDebugH: {},
  envCtx: { gpIsProd: undefined, appname: 'unknown', deploymentName: '', git_sha: 'devvv', hostname: 'Petes-MBP.localdomain', deployEp: '' },
  reqCtx: {},
  errCtx: { errHash: '796464d0bd', backoffNo: 2 }
}
noThrowNotifErr failed mfsReplicationMismatch !secretNotExist
mfsVerifyResult {
  totalRoots: 2,
  synced: 1,
  healed: 1,
  mismatches: 1,
  perRoot: {
    'gamesapp/ipfs_mfsroot_gameshetzdb_qs': { primaryCid: 'QmSB4kGs...', synced: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com', 'ipfs-api.harv3lan.demokluster.com' ], healed: [], mismatches: [] },
    'donateapp/ipfs_mfsroot_donatedbprod_qs': {
      primaryCid: 'QmTc8jL3...',
      synced: [ 'ipfs-api.do2demo.demokluster.com', 'ipfs-api.harv2.demokluster.com' ],
      healed: [ 'ipfs-api.harv3lan.demokluster.com' ],
      mismatches: [ 'ipfs-api.harv3lan.demokluster.com' ]
    }
  }
}
ipfsBackupResult { total: 8, new: 0, skipped: 8, failed: 0 }
✓ 184.999s ipfs3pins
â–¸ check1ipfs_gamesapp
checkGamesIpfsPins { total: 11, orphaned: 9 }
✗ check1ipfs_gamesapp PtErr: orphanCount > 0
    at throPtErr (pt0/sharedF/throwErrorF/sharedErrF.mts:55)
    at throwIf (pt0/sharedF/throwErrorF/throwIfF.mts:9)
    at checkGamesAppIpfsPins (pt0/ipfsF/checkIpfsPinsAI.mts:68)
    at <anonymous> (pt0/ipfsF/zhongCheckIpfsPinsF.mts:14)
    at <anonymous> (pt0/serverF/dbF/withDbF.mts:79)
    at <anonymous> (pt0/serverF/dbF/withDbF.mts:78)
    at <anonymous> (pt0/serverF/dbF/withDbF.mts:90)
    at dbConnWrapNoMap (pt0/serverF/dbF/dbConnWrapF.mts:60)
    at <anonymous> (pt0/serverF/dbF/withDbF.mts:88)
  uniqDebugH: {"orphanedCids":[{"cid":"QmePJhwe...","gameIds":["petesfam"]},{"cid":"QmVfZuSj...","gameIds":["petesfam"]},{"cid":"QmYqbTWn...","gameIds":["petesfam"]},{"cid":"QmQZFrkA...","gameIds":["petesfam"]},{"cid":"QmXqojGP...","gameIds":["petesfam"]},{"cid":"QmdXpRSs...","gameIds":["petesfam"]},{"cid":"Qme7XRYb...","gameIds":["petesfam"]},{"cid":"QmRi56Zp...","gameIds":["petesfam"]},{"cid":"QmS2BdYo...","gameIds":["petesfam"]}],"fixmfsCmd":"ptnode pt0/bartgames/bin/db_sync.mjs fixmfs"}
✓ 4.615s check1ipfs_gamesapp
check1ipfs_donateapp { isRecent: true } 7m / 1d
â–¸ check1ipfs_donateapp
checkDonateIpfsPins { total: 2, orphaned: 0 }
✓ 1.221s check1ipfs_donateapp
validatorAttest15576 { isRecent: true } 7m / 30m
â–¸ validatorAttest15576
✓ 0.884s validatorAttest15576
ethSyncMonitor { isRecent: false } 7d / 30m
â–¸ ethSyncMonitor
✗ ethSyncMonitor Error: eth sync: harv3demo nimbus-mainnet: -6238618 slots
    at runFnc (file:///Users/pete/code/monor/pt0/deployF/ethF/ethSyncMonitorAI.mts:57:13)
    at process.processTicksAndRejections (node:internal/process/task_queues:104:5)
    at async zhongRunOnce (file:///Users/pete/code/monor/pt0/devpconlyF/zhongF/zhongF.mts:101:21)
    at async eptZhong (file:///Users/pete/code/monor/pt0/devpconlyF/zhongF/zhongF.mts:66:3)
    at async file:///Users/pete/code/monor/pt0/devpconlyF/zhongF/epZhong.mjs:64:16
✓ 1.05s ethSyncMonitor
â–¸ kopia_backup
{ connName: 'b2legacy', isDriveConnected: false }
{
  didSwitch: false,
  drivePluggedIn: false,
  isB2Connected: true,
  isDriveConnected: false
}
kopia snapshot create ~



✓ 41.54s kopia_backup
ocModelsCache { isRecent: true } 7m / 1d
â–¸ ocModelsCache
✓ 0.123s ocModelsCache