VKE 的 VPC-CNI 网络插件

概述

本文调研 VKE 选择 VPC-CNI 网络插件的部署和配置情况（基于 VKE v1.30）：

YAML 清单

cello

apiVersion: apps/v1
kind: DaemonSet
metadata:
  annotations:
    deprecated.daemonset.template.generation: "1"
    meta.helm.sh/release-name: vpc-cni
    meta.helm.sh/release-namespace: kube-system
  labels:
    addon.vke.volcengine.com/status-check: vpc-cni
    app: cello
    app.kubernetes.io/managed-by: Helm
  name: cello
  namespace: kube-system
spec:
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app: cello
  template:
    metadata:
      creationTimestamp: null
      labels:
        addon.vke.volcengine.com/status-check: vpc-cni
        app: cello
    spec:
      affinity:
        nodeAffinity:
          requiredDuringSchedulingIgnoredDuringExecution:
            nodeSelectorTerms:
            - matchExpressions:
              - key: kubernetes.io/os
                operator: In
                values:
                - linux
              - key: kubernetes.io/arch
                operator: In
                values:
                - amd64
                - arm64
              - key: type
                operator: NotIn
                values:
                - virtual-kubelet
              - key: node.kubernetes.io/instance-type
                operator: NotIn
                values:
                - dcp-node
              - key: node.vke.volcengine.com/deletion
                operator: NotIn
                values:
                - "true"
      containers:
      - command:
        - cello-agent
        env:
        - name: KUBERNETES_SERVICE_HOST
          value: 10.10.3.200
        - name: KUBERNETES_SERVICE_PORT
          value: "6443"
        - name: NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        image: vke-cn-beijing.cr.volces.com/vke/cello:v1.7.10
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 6
          httpGet:
            path: /healthz
            port: 11414
            scheme: HTTP
          initialDelaySeconds: 30
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        name: cello
        readinessProbe:
          failureThreshold: 12
          httpGet:
            path: /healthz
            port: 11414
            scheme: HTTP
          initialDelaySeconds: 10
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources:
          limits:
            cpu: 250m
            memory: 256Mi
          requests:
            cpu: 100m
            memory: 100Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /var/run/cello
          name: host-run-cello
        - mountPath: /var/run/cilium
          name: host-cilium-run
        - mountPath: /var/run/netns
          mountPropagation: HostToContainer
          name: host-run-netns
        - mountPath: /opt/cni/bin
          name: host-cni-bin-dir
        - mountPath: /var/log/cello
          name: host-log-cello
        - mountPath: /var/log/cello/cni.log
          name: host-log-cello-cni
        - mountPath: /var/lib/kubelet/device-plugins
          name: host-device-plugins
        - mountPath: /etc/cello/cello-config
          name: configvolume
          subPath: cello.json
        - mountPath: /cello/secrets
          name: credential
      - command:
        - cilium-launcher
        env:
        - name: KUBERNETES_SERVICE_HOST
          value: 10.10.3.200
        - name: KUBERNETES_SERVICE_PORT
          value: "6443"
        - name: KUBE_PROXY_REPLACEMENT
          value: strict
        - name: NODENAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: K8S_NODE_NAME
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: spec.nodeName
        - name: CILIUM_K8S_NAMESPACE
          valueFrom:
            fieldRef:
              apiVersion: v1
              fieldPath: metadata.namespace
        - name: CILIUM_CNI_CHAINING_MODE
          value: cello-chainer
        image: vke-cn-beijing.cr.volces.com/vke/cello:v1.7.10
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 6
          initialDelaySeconds: 30
          periodSeconds: 10
          successThreshold: 1
          tcpSocket:
            host: localhost
            port: 9099
          timeoutSeconds: 1
        name: cilium
        readinessProbe:
          failureThreshold: 3
          periodSeconds: 10
          successThreshold: 1
          tcpSocket:
            host: localhost
            port: 9099
          timeoutSeconds: 1
        resources:
          requests:
            cpu: 250m
            memory: 256Mi
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /lib/modules
          name: host-lib-modules
        - mountPath: /etc/cni/net.d
          name: host-cni-conf-dir
        - mountPath: /opt/cni/bin/
          name: host-cni-bin-dir
        - mountPath: /sys/fs/bpf
          name: host-bpf-maps
        - mountPath: /var/run/cilium
          name: host-cilium-run
        - mountPath: /var/run/netns
          name: host-run-netns
        - mountPath: /run/xtables.lock
          name: host-xtables-lock
        - mountPath: /etc/cilium/cilium-config
          name: cilium-config-path
          readOnly: true
        - mountPath: /etc/cilium/cello-config
          name: configvolume
          subPath: cello.json
      dnsPolicy: ClusterFirst
      hostNetwork: true
      hostPID: true
      initContainers:
      - command:
        - /bin/bash
        - -c
        - /script/install_env.sh
        image: vke-cn-beijing.cr.volces.com/vke/cello:v1.7.10
        imagePullPolicy: IfNotPresent
        name: cello-init
        securityContext:
          privileged: true
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /opt/cni/bin/
          name: host-cni-bin-dir
        - mountPath: /etc/cni/net.d/
          name: host-cni-conf-dir
        - mountPath: /lib/modules
          name: host-lib-modules
        - mountPath: /etc/cello
          name: configvolume
      priorityClassName: system-node-critical
      restartPolicy: Always
      schedulerName: default-scheduler
      serviceAccount: cello-service-account
      serviceAccountName: cello-service-account
      terminationGracePeriodSeconds: 30
      tolerations:
      - operator: Exists
      volumes:
      - configMap:
          defaultMode: 420
          items:
          - key: conf
            path: cello.json
          - key: 10-cello.conflist
            path: net.d/10-cello.conflist
          - key: 01-cello-meta.conflist
            path: net.d/01-cello-meta.conflist
          name: cello-config
        name: configvolume
      - configMap:
          defaultMode: 420
          name: cilium-config
        name: cilium-config-path
      - hostPath:
          path: /opt/cni/bin/
          type: DirectoryOrCreate
        name: host-cni-bin-dir
      - hostPath:
          path: /etc/cni/net.d/
          type: DirectoryOrCreate
        name: host-cni-conf-dir
      - hostPath:
          path: /var/run/cello
          type: DirectoryOrCreate
        name: host-run-cello
      - hostPath:
          path: /var/run/netns
          type: DirectoryOrCreate
        name: host-run-netns
      - hostPath:
          path: /var/log/cello
          type: DirectoryOrCreate
        name: host-log-cello
      - hostPath:
          path: /var/log/cello/cni.log
          type: FileOrCreate
        name: host-log-cello-cni
      - hostPath:
          path: /var/lib/kubelet/device-plugins
          type: Directory
        name: host-device-plugins
      - hostPath:
          path: /lib/modules
          type: ""
        name: host-lib-modules
      - hostPath:
          path: /var/run/cilium
          type: DirectoryOrCreate
        name: host-cilium-run
      - hostPath:
          path: /sys/fs/bpf
          type: DirectoryOrCreate
        name: host-bpf-maps
      - hostPath:
          path: /run/xtables.lock
          type: FileOrCreate
        name: host-xtables-lock
      - name: credential
        secret:
          defaultMode: 420
          optional: true
          secretName: addon.network.token
  updateStrategy:
    rollingUpdate:
      maxSurge: 0
      maxUnavailable: 1
    type: RollingUpdate

cello-config

apiVersion: v1
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: vpc-cni
    meta.helm.sh/release-namespace: kube-system
  labels:
    app.kubernetes.io/managed-by: Helm
  name: cello-config
  namespace: kube-system
data:
  01-cello-meta.conflist: |
    {
      "cniVersion": "0.3.1",
      "name": "cello-meta",
      "plugins": [
        {
          "type": "cello-meta",
          "capabilities": {
            "bandwidth": true,
            "portMappings": true,
            "io.kubernetes.cri.pod-annotations": true
          },
          "defaultCni": "cello-chainer"
        }
      ]
    }
  10-cello.conflist: |
    {
      "cniVersion": "0.3.1",
      "name": "cello-chainer",
      "plugins": [
        {
          "type": "cello",
          "capabilities": {
            "bandwidth": true,
            "com.volcengine.k8s.network-interface": true
           },
          "redirectToHostCIDRs": ["169.254.0.0/16"]
        }
      ]
    }
  conf: |
    {
      "subnets": [
        "subnet-1a1ybnm8eqr5s8nvepl7tl6gg",
        "subnet-1a1yadwomszr48nvepkhdfr6y",
        "subnet-iit7ovtsl1c074o8cu5x2hr8",
        "subnet-iit7l1pmp72874o8cuhut0pn"
      ],
      "securityGroups": [
        "sg-1jox14x37qadc1n7ampakr4ma"
      ],
      "poolTarget": 3,
      "poolTargetMin": 5,
      "poolMaxCapProbe": true,
      "interfaceTagPrefixes": ["volc:vke:"],
      "additionalTags": {
        "volc:vke:createdby-vke-flag": "true"
      },
      "credentialFile": "/cello/secrets/addon_token_info",
      "ramRole": "KubernetesNodeRoleForECS",
      "openApiAddress": "open.volcengineapi.com",
      "poolMonitorIntervalSec": 120,
      "enableTrunk": false,
      "networkMode": "eni_shared",
      "ipFamily": "ipv4"
    }

cilium-config

apiVersion: v1
kind: ConfigMap
metadata:
  annotations:
    meta.helm.sh/release-name: vpc-cni
    meta.helm.sh/release-namespace: kube-system
  labels:
    app.kubernetes.io/managed-by: Helm
  name: cilium-config
  namespace: kube-system
data:
  debug: "false"
  enable-policy: never

CNI 配置

cello-meta

/etc/cni/net.d/01-cello-meta.conflist

{
  "cniVersion": "0.3.1",
  "name": "cello-meta",
  "plugins": [
    {
      "type": "cello-meta",
      "capabilities": {
        "bandwidth": true,
        "portMappings": true,
        "io.kubernetes.cri.pod-annotations": true
      },
      "defaultCni": "cello-chainer"
    }
  ]
}

cello-config

/etc/cni/net.d/10-cello.conflist

{
  "cniVersion": "0.3.1",
  "name": "cello-chainer",
  "plugins": [
    {
      "type": "cello",
      "capabilities": {
        "bandwidth": true,
        "com.volcengine.k8s.network-interface": true
       },
      "redirectToHostCIDRs": ["169.254.0.0/16"]
    }
  ]
}

CNI 二进制

root@iv-ye593xiz9c5i3z3kulq3:/opt/cni/bin# ls
cello-cni  cello-meta  cello-rdma  cilium-cni  loopback

进程

cello 这个 DaemonSet 中包含 cello 和 cilium 两个容器：

• cello 容器拉起的进程名是 cello-agent
• cilium 容器会使用 cilium-launcher 拉起 cilium 相关组件，进程树：

  $ pstree -apnT 30278
  cilium-launcher,30278
    └─cilium-agent,30333 --enable-ipv4-masquerade=false --kube-proxy-replacement=strict --node-port-mode=snat --disable-envoy-version-check=true --disable-cnp-status-updates=true --enable-local-node-route=false --datapath-mode=ipvlan --enable-bandwidth-manager=true --agent-health-port=9099 --ipv4-range=169.254.0.0/16 --enable-endpoint-health-checking=false --bpf-map-dynamic-size-ratio=0.0025 --enable-ipv4=true --debug=false --enable-policy=never --ipam=cluster-pool --ipvlan-master-device=eth0 --enable-host-legacy-routing=true --tunnel=disabled --direct-routing-device=eth0 --enable-ipv6=false
        └─cilium-operator-generic,30368 --k8s-namespace kube-system --identity-gc-interval 10m --identity-heartbeat-timeout 20m

cilium 版本信息

root@iv-ye593xiz9ccva4flha5g:/# cilium-operator-generic --version
Cilium-Operator 1.10.4 2a46fd6 2021-09-01T12:58:41-07:00 go version go1.16.7 linux/amd64
root@iv-ye593xiz9ccva4flha5g:/# cilium-agent --version
Cilium 1.10.4 2a46fd6 2021-09-01T12:58:41-07:00 go version go1.16.7 linux/amd64

cilium CRD

安装了以下 cilium 相关 CRD:

╰─ kubectl get crd | grep cilium
ciliumclusterwidenetworkpolicies.cilium.io                 2025-09-22T04:13:37Z
ciliumegressnatpolicies.cilium.io                          2025-09-22T04:13:36Z
ciliumendpoints.cilium.io                                  2025-09-22T04:13:36Z
ciliumexternalworkloads.cilium.io                          2025-09-22T04:13:36Z
ciliumidentities.cilium.io                                 2025-09-22T04:13:36Z
ciliumlocalredirectpolicies.cilium.io                      2025-09-22T04:13:36Z
ciliumnetworkpolicies.cilium.io                            2025-09-22T04:13:37Z
ciliumnodes.cilium.io                                      2025-09-22T04:13:36Z