Lesson 22 — AI/ML platform: prefill/decode disaggregation & MoE serving topology on the fleet

# Pool nodes
kubectl get nodes -l pool=prefill -o wide
kubectl get nodes -l pool=decode -o wide
kubectl get nodes -l pool=moe-ep -o wide
 
# Workloads
kubectl get deploy,pods -n model-inference \
  -l 'role in (prefill,decode,moe-expert)' -o wide
 
# Pending under pressure (wrong taints / flavors)
kubectl get pods -n model-inference --field-selector=status.phase=Pending
kubectl describe pod -n model-inference -l role=prefill | sed -n '/Node-Selectors:/,/Events:/p'

# Prefill instance (shape — flag names drift by engine/version)
python -m sglang.launch_server \
  --model-path /models/deepseek-v3 \
  --disaggregation-mode prefill \
  --host 0.0.0.0 --port 30000 \
  --tp-size 8 \
  --trust-remote-code \
  --disaggregation-ib-device mlx5_bond_0 \
  --mem-fraction-static 0.85
 
# Decode instance
python -m sglang.launch_server \
  --model-path /models/deepseek-v3 \
  --disaggregation-mode decode \
  --host 0.0.0.0 --port 30000 \
  --tp-size 8 \
  --trust-remote-code \
  --disaggregation-ib-device mlx5_bond_0 \
  --mem-fraction-static 0.90
 
# MoE + EP (decode often wider EP) — interview vocabulary
#   --enable-dp-attention
#   --dp-size 8
#   --ep-size 64
#   --moe-a2a-backend deepep   # or mooncake-class backend
# Transfer backend talking points: Mooncake Transfer Engine, NIXL, NVLink path

apiVersion: apps/v1
kind: Deployment
metadata:
  name: sglang-prefill
  namespace: model-inference
  labels:
    role: prefill
    trust-domain: inference-prod
spec:
  replicas: 4
  selector:
    matchLabels:
      role: prefill
  template:
    metadata:
      labels:
        role: prefill
        trust-domain: inference-prod
        pool: prefill
    spec:
      serviceAccountName: prefill-readonly
      runtimeClassName: nvidia
      nodeSelector:
        sku: h100
        pool: prefill
      tolerations:
        - key: pool
          value: prefill
          effect: NoSchedule
      containers:
        - name: sglang
          image: registry.example.com/ml/sglang@sha256:REPLACE_PREFILL
          args:
            - "--model-path=/models/deepseek-v3"
            - "--disaggregation-mode=prefill"
            - "--tp-size=8"
            - "--disaggregation-ib-device=mlx5_bond_0"
          resources:
            limits:
              nvidia.com/gpu: "8"
              memory: 512Gi
            requests:
              nvidia.com/gpu: "8"
              cpu: "32"
              memory: 512Gi
          volumeMounts:
            - name: weights
              mountPath: /models
              readOnly: true
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: weights
          persistentVolumeClaim:
            claimName: deepseek-v3-weights-ro
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: sglang-decode
  namespace: model-inference
  labels:
    role: decode
    trust-domain: inference-prod
spec:
  replicas: 8
  selector:
    matchLabels:
      role: decode
  template:
    metadata:
      labels:
        role: decode
        trust-domain: inference-prod
        pool: decode
    spec:
      serviceAccountName: decode-readonly
      runtimeClassName: nvidia
      nodeSelector:
        sku: h100
        pool: decode
      tolerations:
        - key: pool
          value: decode
          effect: NoSchedule
      containers:
        - name: sglang
          image: registry.example.com/ml/sglang@sha256:REPLACE_DECODE
          args:
            - "--model-path=/models/deepseek-v3"
            - "--disaggregation-mode=decode"
            - "--tp-size=8"
            - "--disaggregation-ib-device=mlx5_bond_0"
          resources:
            limits:
              nvidia.com/gpu: "8"
              memory: 512Gi
            requests:
              nvidia.com/gpu: "8"
              cpu: "32"
              memory: 512Gi
          volumeMounts:
            - name: weights
              mountPath: /models
              readOnly: true
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: true
            capabilities:
              drop: ["ALL"]
      volumes:
        - name: weights
          persistentVolumeClaim:
            claimName: deepseek-v3-weights-ro

apiVersion: kueue.x-k8s.io/v1beta1
kind: ResourceFlavor
metadata:
  name: gpu-prefill-h100
spec:
  nodeLabels:
    pool: prefill
    sku: h100
---
apiVersion: kueue.x-k8s.io/v1beta1
kind: ResourceFlavor
metadata:
  name: gpu-decode-h100
spec:
  nodeLabels:
    pool: decode
    sku: h100
---
# ClusterQueues (sketch): prefill-cq and decode-cq each bind their flavor.
# Train gang CQ must NOT borrow these flavors ([L20]).

apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: pd-default-deny
  namespace: model-inference
spec:
  podSelector: {}
  policyTypes: ["Ingress", "Egress"]
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: prefill-to-decode-kv
  namespace: model-inference
spec:
  podSelector:
    matchLabels:
      role: prefill
  policyTypes: ["Egress"]
  egress:
    - to:
        - podSelector:
            matchLabels:
              role: decode
      # App control plane + transfer ports — pin real ports in-cluster
      ports:
        - protocol: TCP
          port: 30000
    - to:
        # RDMA / RoCE data plane CIDR for KV transfer only ([L9])
        - ipBlock:
            cidr: 10.50.0.0/16
    - to:
        - namespaceSelector:
            matchLabels:
              role: dns
---
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
  name: decode-from-prefill-kv
  namespace: model-inference
spec:
  podSelector:
    matchLabels:
      role: decode
  policyTypes: ["Ingress", "Egress"]
  ingress:
    - from:
        - podSelector:
            matchLabels:
              role: prefill
        - namespaceSelector:
            matchLabels:
              role: edge-gateway
  egress:
    - to:
        - podSelector:
            matchLabels:
              role: moe-expert
      ports:
        - protocol: TCP
          port: 30000
    - to:
        - ipBlock:
            cidr: 10.50.0.0/16   # RDMA fabric
    - to:
        - namespaceSelector:
            matchLabels:
              role: dns
# Explicitly NO path to train namespaces or weight-write Jobs

apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: sglang-prefill
  namespace: model-inference
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: sglang-prefill
  minReplicas: 4
  maxReplicas: 24
  metrics:
    - type: Pods
      pods:
        metric:
          name: prefill_waiting_requests
        target:
          type: AverageValue
          averageValue: "4"
    - type: Pods
      pods:
        metric:
          name: ttft_slo_burn_ratio
        target:
          type: AverageValue
          averageValue: "800m"
---
apiVersion: autoscaling/v2
kind: HorizontalPodAutoscaler
metadata:
  name: sglang-decode
  namespace: model-inference
spec:
  scaleTargetRef:
    apiVersion: apps/v1
    kind: Deployment
    name: sglang-decode
  minReplicas: 8
  maxReplicas: 48
  metrics:
    - type: Pods
      pods:
        metric:
          name: kv_cache_usage_ratio
        target:
          type: AverageValue
          averageValue: "750m"
    - type: Pods
      pods:
        metric:
          name: tpot_slo_burn_ratio
        target:
          type: AverageValue
          averageValue: "800m"

apiVersion: admissionregistration.k8s.io/v1
kind: ValidatingAdmissionPolicy
metadata:
  name: pd-serve-floor
spec:
  failurePolicy: Fail
  matchConstraints:
    resourceRules:
      - apiGroups: ["apps"]
        apiVersions: ["v1"]
        operations: ["CREATE", "UPDATE"]
        resources: ["deployments"]
  validations:
    - expression: >
        !has(object.metadata.labels) ||
        object.metadata.labels["trust-domain"] != "inference-prod" ||
        (
          object.metadata.labels["role"] == "prefill" &&
          object.spec.template.spec.serviceAccountName == "prefill-readonly"
        ) ||
        (
          object.metadata.labels["role"] == "decode" &&
          object.spec.template.spec.serviceAccountName == "decode-readonly"
        ) ||
        (
          object.metadata.labels["role"] == "moe-expert" &&
          object.spec.template.spec.serviceAccountName == "expert-readonly"
        )
      message: "inference-prod PD/MoE roles require matching *-readonly SA"

# Sketch: PD / MoE serving-path weight & image floor
apiVersion: policy.platform/v1
kind: MlServeGate
metadata:
  name: pd-moe-weight-floor
spec:
  match:
    labels:
      trust-domain: inference-prod
  require:
    imageDigestPin: true
    cosignVerify: true
    runtimeClass: nvidia
    roleServiceAccounts:
      prefill: prefill-readonly
      decode: decode-readonly
      moe-expert: expert-readonly
    workloadIdentity: true
    weightMountReadOnly: true
    weightBucketPublicAccess: false
    cmek: true
    serveSaPutObject: false
    trainSaOnServePods: false
    networkPolicy:
      defaultDeny: true
      allowKvPathCidr: "10.50.0.0/16"
      denyTrainNamespace: true
    edgeRateLimit: true
    apiKeyInImage: false