Job 高级特性与批处理生态：Indexed Job、失败/成功策略、JobSet 与 Kueue

apiVersion: v1
kind: Service
metadata:
  name: trainer
spec:
  clusterIP: None
  selector:
    job-name: trainer
---
apiVersion: batch/v1
kind: Job
metadata:
  name: trainer
spec:
  completionMode: Indexed
  completions: 4
  parallelism: 4
  template:
    spec:
      subdomain: trainer          # 与无头 Service 同名，使主机名可解析
      restartPolicy: Never
      containers:
      - name: worker
        image: my-trainer:1.0
        env:
        - name: MASTER_ADDR
          value: trainer-0.trainer
        - name: RANK
          valueFrom:
            fieldRef:
              fieldPath: metadata.annotations['batch.kubernetes.io/job-completion-index']

apiVersion: batch/v1
kind: Job
metadata:
  name: etl
spec:
  completions: 10
  parallelism: 5
  backoffLimit: 6
  podFailurePolicy:
    rules:
    - action: FailJob             # 软件缺陷：立即失败，不浪费重试
      onExitCodes:
        containerName: main
        operator: In
        values: [42]
    - action: Ignore              # 基础设施打断：不计入 backoffLimit
      onPodConditions:
      - type: DisruptionTarget
    - action: Count               # 其他失败：按默认逻辑计数
      onExitCodes:
        operator: NotIn
        values: [0]
  template:
    spec:
      restartPolicy: Never        # 使用 podFailurePolicy 时必须为 Never
      containers:
      - name: main
        image: my-etl:1.0

spec:
  completionMode: Indexed
  completions: 100
  parallelism: 20
  backoffLimitPerIndex: 2      # 每个索引最多重试 2 次
  maxFailedIndexes: 5          # 超过 5 个索引失败则整个 Job 失败

spec:
  completionMode: Indexed
  completions: 8
  parallelism: 8
  successPolicy:
    rules:
    - succeededIndexes: "0"        # 索引 0（leader）成功即可
    - succeededCount: 6            # 或者任意 6 个索引成功

apiVersion: jobset.x-k8s.io/v1alpha2
kind: JobSet
metadata:
  name: pytorch-ddp
spec:
  successPolicy:
    operator: All
    targetReplicatedJobs: [workers]   # workers 全部完成即成功，自动清理其他角色
  failurePolicy:
    maxRestarts: 3                    # 任一 Job 失败则整体重启，最多 3 次
  network:
    enableDNSHostnames: true          # 自动创建无头 Service，Pod 可通过 <jobset>-<job>-<replica>-<index> 寻址
  replicatedJobs:
  - name: leader
    replicas: 1
    template:
      spec:
        backoffLimit: 0
        completions: 1
        parallelism: 1
        template:
          spec:
            containers:
            - name: leader
              image: my-trainer:1.0
              args: ["--role=leader"]
  - name: workers
    replicas: 1
    template:
      spec:
        backoffLimit: 0
        completionMode: Indexed
        completions: 4
        parallelism: 4
        template:
          spec:
            containers:
            - name: worker
              image: my-trainer:1.0
              args: ["--role=worker", "--leader=pytorch-ddp-leader-0-0.pytorch-ddp"]
              resources:
                limits:
                  nvidia.com/gpu: 1

apiVersion: kueue.x-k8s.io/v1beta2
kind: ResourceFlavor
metadata:
  name: h100
spec:
  nodeLabels:
    gpu-type: h100
---
apiVersion: kueue.x-k8s.io/v1beta2
kind: ClusterQueue
metadata:
  name: ml-team
spec:
  namespaceSelector: {}
  cohort: shared-gpu              # 同一 cohort 内的队列可以互相借用闲置配额
  preemption:
    reclaimWithinCohort: Any
  resourceGroups:
  - coveredResources: ["cpu", "memory", "nvidia.com/gpu"]
    flavors:
    - name: h100
      resources:
      - name: cpu
        nominalQuota: 200
      - name: memory
        nominalQuota: 1600Gi
      - name: nvidia.com/gpu
        nominalQuota: 64
---
apiVersion: kueue.x-k8s.io/v1beta2
kind: LocalQueue
metadata:
  namespace: ml-team
  name: default
spec:
  clusterQueue: ml-team

apiVersion: batch/v1
kind: Job
metadata:
  generateName: train-
  namespace: ml-team
  labels:
    kueue.x-k8s.io/queue-name: default
spec:
  suspend: true                   # Kueue 的 Webhook 会自动设置，这里显式写出便于理解
  parallelism: 8
  completions: 8
  completionMode: Indexed
  template:
    spec:
      restartPolicy: Never
      containers:
      - name: worker
        image: my-trainer:1.0
        resources:
          requests:
            cpu: "8"
            memory: "64Gi"
            nvidia.com/gpu: "1"