Kubernetes 기반 NPU 서빙 플랫폼 구축

apiVersion: apps/v1
kind: DaemonSet
metadata:
  name: x330-device-plugin-daemonset
  namespace: sapeon-device-plugin
spec:
  selector:
    matchLabels:
      name: sapeon-device-plugin
  template:
    metadata:
      labels:
        name: sapeon-device-plugin
    spec:
      containers:
      - name: aix-device-plugin
        image: x330_k8s_device_plugin:v1.0
        imagePullPolicy: IfNotPresent
        env:
        - name: GRPC_GO_LOG_VERBOSITY_LEVEL
          value: "99"
        - name: GRPC_GO_LOG_SEVERITY_LEVEL
          value: info
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
        volumeMounts:
        - name: device-plugin
          mountPath: /var/lib/kubelet/device-plugins
      tolerations:
      - key: CriticalAddonsOnly
        operator: Exists
      - key: app
        operator: Equal
        value: vps
        effect: NoSchedule
      volumes:
      - name: device-plugin
        hostPath:
          path: /var/lib/kubelet/device-plugins

apiVersion: serving.kserve.io/v1beta1
kind: InferenceService
metadata:
  name: kserve-llm
  namespace: ml-serving
spec:
  predictor:
    nodeSelector:
      kubernetes.io/hostname: gpu-node-1
    containers:
      - name: vllm
        image: vllm/vllm-openai:v0.5.4
        args:
          - --model
          - meta-llama/Meta-Llama-3.1-8B-Instruct
          - --tensor-parallel-size
          - "2"
          - --gpu-memory-utilization
          - "0.8"
        resources:
          limits:
            nvidia.com/gpu: 2

---
# VirtualService를 별도로 생성해야 함
apiVersion: networking.istio.io/v1
kind: VirtualService
metadata:
  name: kserve-llm-vs
spec:
  gateways:
    - kubeflow/kubeflow-gateway
  hosts:
    - kserve-llm.example.com
  http:
    - route:
        - destination:
            host: kserve-llm-predictor.ml-serving.svc.cluster.local

apiVersion: dudaji.com/v1
kind: NpuDeploy
metadata:
  name: llama-8b
  namespace: ml-serving
spec:
  # 동적 엔드포인트 (VirtualService 자동 생성)
  endpoint:
    gateway: kubeflow/kubeflow-gateway
    host: llama-8b.nufi.me

  # 추론 서버 설정
  inferenceServer:
    image: vllm/vllm-openai:v0.5.4
    inferencePort: 8000
    replicas: 1
    args:
      - --model
      - meta-llama/Meta-Llama-3.1-8B-Instruct
      - --tensor-parallel-size
      - "2"
      - --gpu-memory-utilization
      - "0.8"
    resources:
      limits:
        nvidia.com/gpu: 2
    volumeMounts:
      - name: models
        mountPath: /models
      - name: dshm
        mountPath: /dev/shm
    volumes:
      - name: models
        hostPath:
          path: /mnt/data/model-cache
      - name: dshm
        emptyDir:
          medium: Memory

func (r *NpuDeployReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
    logger := log.FromContext(ctx)

    nd := &dudajicomv1.NpuDeploy{}
    if err := r.Get(ctx, req.NamespacedName, nd); err != nil {
        if kerrors.IsNotFound(err) {
            logger.Info("NpuDeploy resource not found. Ignoring since object must be deleted")
            return ctrl.Result{}, nil
        }
        return ctrl.Result{}, err
    }

    // Status 업데이트를 위한 defer
    statusUpdater := NewStatusUpdater()
    originalStatus := nd.Status.DeepCopy()
    defer func() {
        if !reflect.DeepEqual(*originalStatus, nd.Status) {
            r.Status().Update(ctx, nd)
        }
    }()

    statusUpdater.SetReconciling(nd)

    // 모든 하위 리소스 Reconcile
    if err := r.reconcileAll(ctx, nd); err != nil {
        statusUpdater.SetError(nd, err)
        return ctrl.Result{}, err
    }

    statusUpdater.SetSuccess(nd)
    return ctrl.Result{}, nil
}

func (r *NpuDeployReconciler) reconcileAll(ctx context.Context, nd *dudajicomv1.NpuDeploy) error {
    // 1. Deployment 생성/업데이트
    if err := r.reconcileDeployment(ctx, nd); err != nil {
        r.Recorder.Event(nd, corev1.EventTypeWarning, "ReconcileError", err.Error())
        return fmt.Errorf("failed to reconcile Deployment: %w", err)
    }

    // 2. Service 생성/업데이트
    if err := r.reconcileService(ctx, nd); err != nil {
        r.Recorder.Event(nd, corev1.EventTypeWarning, "ReconcileError", err.Error())
        return fmt.Errorf("failed to reconcile Service: %w", err)
    }

    // 3. VirtualService (endpoint 필드 존재 시)
    if err := r.reconcileVirtualService(ctx, nd); err != nil {
        r.Recorder.Event(nd, corev1.EventTypeWarning, "ReconcileError", err.Error())
        return fmt.Errorf("failed to reconcile VirtualService: %w", err)
    }

    r.Recorder.Event(nd, corev1.EventTypeNormal, "ReconcileSuccess", "All sub-resources reconciled successfully")
    return nil
}

func (r *NpuDeployReconciler) reconcileVirtualService(
    ctx context.Context,
    npuDeploy *nufiv1.NpuDeploy,
) error {
    vs := &istionetv1beta1.VirtualService{
        ObjectMeta: metav1.ObjectMeta{
            Name:      npuDeploy.Name,
            Namespace: npuDeploy.Namespace,
        },
        Spec: istionetv1beta1api.VirtualService{
            Hosts:    []string{npuDeploy.Spec.Endpoint.Host},
            Gateways: []string{npuDeploy.Spec.Endpoint.Gateway},
            Http: []*istionetv1beta1api.HTTPRoute{{
                Route: []*istionetv1beta1api.HTTPRouteDestination{{
                    Destination: &istionetv1beta1api.Destination{
                        Host: fmt.Sprintf("%s.%s.svc.cluster.local",
                            npuDeploy.Name, npuDeploy.Namespace),
                        Port: &istionetv1beta1api.PortSelector{
                            Number: 80,
                        },
                    },
                }},
            }},
        },
    }

    return r.CreateOrUpdate(ctx, vs)
}

apiVersion: ray.io/v1
kind: RayService
metadata:
  name: vllm-ray-service
  namespace: llm
spec:
  serveConfigV2: |
    applications:
    - args:
        llm_configs:
          - model_loading_config:
              model_id: meta-llama/Llama-3.1-8B-Instruct
              model_source: meta-llama/Llama-3.1-8B-Instruct
            deployment_config:
              autoscaling_config:
                min_replicas: 1
                max_replicas: 1
            engine_kwargs:
              tensor_parallel_size: 2
              pipeline_parallel_size: 1
              max_model_len: 4096
      import_path: ray.serve.llm:build_openai_app
      name: llm_app
      route_prefix: "/"

  rayClusterConfig:
    rayVersion: '2.48.0'
    headGroupSpec:
      rayStartParams:
        num-cpus: "0"
        num-gpus: "0"
        dashboard-host: '0.0.0.0'
      template:
        spec:
          containers:
            - name: ray-head
              image: rayproject/ray-llm:2.48.0-py311-cu128
              ports:
              - containerPort: 8000
                name: serve
              - containerPort: 8265
                name: dashboard
              resources:
                limits:
                  cpu: "2"
                  memory: "4Gi"
    workerGroupSpecs:
      - groupName: gpu-group
        replicas: 1
        rayStartParams:
          num-gpus: "2"
        template:
          spec:
            containers:
              - name: ray-worker
                image: rayproject/ray-llm:2.48.0-py311-cu128
                resources:
                  limits:
                    cpu: "16"
                    memory: "32Gi"
                    nvidia.com/gpu: "2"

type HardwareFetcher interface {
    FetchMetrics(ctx context.Context, promAPI v1.API) (
        map[string]map[string]*HardwareDeviceStatus, error)
}

type HardwareDeviceStatus struct {
    Type        string   // "GPU" 또는 "RBLN"
    DeviceID    string   // 장치 ID
    Utilization *float64 // 사용률 (%)
    Temperature *float64 // 온도 (C)
    MemoryUsed  *float64 // 메모리 사용량 (GB)
    MemoryTotal *float64 // 총 메모리 (GB)
}

// GPU Fetcher 구현
type GPUFetcher struct{}

func (f *GPUFetcher) FetchMetrics(ctx context.Context, promAPI v1.API) (...) {
    utilizationQuery := `DCGM_FI_DEV_GPU_UTIL`
    temperatureQuery := `DCGM_FI_DEV_GPU_TEMP`
    // ... Prometheus 쿼리 수행
}

// RBLN Fetcher 구현
type RBLNFetcher struct{}

func (f *RBLNFetcher) FetchMetrics(ctx context.Context, promAPI v1.API) (...) {
    utilizationQuery := `rbln_device_utilization`
    temperatureQuery := `rbln_device_temperature`
    // ... Prometheus 쿼리 수행
}