Kubernetes 学习笔记之 Calico CNI Plugin 源码解析 (一)

func (m *kubeGenericRuntimeManager) createPodSandbox(pod *v1.Pod, attempt uint32) (string, string, error) {    // 生成pod相关配置数据    podSandboxConfig, err := m.generatePodSandboxConfig(pod, attempt)    // ...    // 这里会在宿主机上创建pod logs目录，在/var/log/pods/{namespace}_{pod_name}_{uid}目录下    err = m.osInterface.MkdirAll(podSandboxConfig.LogDirectory, 0755)    // ...    // 调用容器运行时创建sandbox container，我们生产k8s这里是docker创建    podSandBoxID, err := m.runtimeService.RunPodSandbox(podSandboxConfig, runtimeHandler)    // ...    return podSandBoxID, "", nil}

// 创建sandbox container，以及为该container创建networkfunc (ds *dockerService) RunPodSandbox(ctx context.Context, r *runtimeapi.RunPodSandboxRequest) (*runtimeapi.RunPodSandboxResponse, error) {    config := r.GetConfig()    // Step 1: Pull the image for the sandbox.    // 1. 拉取镜像    image := defaultSandboxImage    podSandboxImage := ds.podSandboxImage    if len(podSandboxImage) != 0 {        image = podSandboxImage    }    if err := ensureSandboxImageExists(ds.client, image); err != nil {        return nil, err    }        // Step 2: Create the sandbox container.    // 2. 创建sandbox container    createResp, err := ds.client.CreateContainer(*createConfig)    // ...    resp := &runtimeapi.RunPodSandboxResponse{PodSandboxId: createResp.ID}    ds.setNetworkReady(createResp.ID, false)        // Step 3: Create Sandbox Checkpoint.    // 3. 创建checkpoint    if err = ds.checkpointManager.CreateCheckpoint(createResp.ID, constructPodSandboxCheckpoint(config)); err != nil {        return nil, err    }        // Step 4: Start the sandbox container.    // Assume kubelet's garbage collector would remove the sandbox later, if    // startContainer failed.    // 4. 启动容器    err = ds.client.StartContainer(createResp.ID)    // ...    // Step 5: Setup networking for the sandbox.    // All pod networking is setup by a CNI plugin discovered at startup time.    // This plugin assigns the pod ip, sets up routes inside the sandbox,    // creates interfaces etc. In theory, its jurisdiction ends with pod    // sandbox networking, but it might insert iptables rules or open ports    // on the host as well, to satisfy parts of the pod spec that aren't    // recognized by the CNI standard yet.    // 5. 这一步为sandbox container创建网络，主要是调用calico cni插件创建路由和虚拟网卡，以及为pod分配pod ip，为该宿主机划分pod网段    cID := kubecontainer.BuildContainerID(runtimeName, createResp.ID)    networkOptions := make(map[string]string)    if dnsConfig := config.GetDnsConfig(); dnsConfig != nil {        // Build DNS options.        dnsOption, err := json.Marshal(dnsConfig)        if err != nil {            return nil, fmt.Errorf("failed to marshal dns config for pod %q: %v", config.Metadata.Name, err)        }        networkOptions["dns"] = string(dnsOption)    }    // 这一步调用网络插件来setup sandbox pod    // 由于我们网络插件都是cni(container network interface)，所以代码在 pkg/kubelet/dockershim/network/cni/cni.go    err = ds.network.SetUpPod(config.GetMetadata().Namespace, config.GetMetadata().Name, cID, config.Annotations, networkOptions)    // ...    return resp, nil}

func (plugin *cniNetworkPlugin) SetUpPod(namespace string, name string, id kubecontainer.ContainerID, annotations, options map[string]string) error {    // ...    netnsPath, err := plugin.host.GetNetNS(id.ID)    // ...    // Windows doesn't have loNetwork. It comes only with Linux    if plugin.loNetwork != nil {        // 添加loopback        if _, err = plugin.addToNetwork(cniTimeoutCtx, plugin.loNetwork, name, namespace, id, netnsPath, annotations, options); err != nil {            return err        }    }    // 调用网络插件创建网络相关资源    _, err = plugin.addToNetwork(cniTimeoutCtx, plugin.getDefaultNetwork(), name, namespace, id, netnsPath, annotations, options)    return err}func (plugin *cniNetworkPlugin) addToNetwork(ctx context.Context, network *cniNetwork, podName string, podNamespace string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (cnitypes.Result, error) {    // 这一步准备网络插件所需相关参数，这些参数最后会被calico插件使用    rt, err := plugin.buildCNIRuntimeConf(podName, podNamespace, podSandboxID, podNetnsPath, annotations, options)    // ...    // 这里会调用调用cni标准库里的AddNetworkList函数，最后会调用calico二进制命令    res, err := cniNet.AddNetworkList(ctx, netConf, rt)    // ...    return res, nil}// 这些参数主要包括container id，pod等相关参数func (plugin *cniNetworkPlugin) buildCNIRuntimeConf(podName string, podNs string, podSandboxID kubecontainer.ContainerID, podNetnsPath string, annotations, options map[string]string) (*libcni.RuntimeConf, error) {    rt := &libcni.RuntimeConf{        ContainerID: podSandboxID.ID,        NetNS:       podNetnsPath,        IfName:      network.DefaultInterfaceName,        CacheDir:    plugin.cacheDir,        Args: [][2]string{            {"IgnoreUnknown", "1"},            {"K8S_POD_NAMESPACE", podNs},            {"K8S_POD_NAME", podName},            {"K8S_POD_INFRA_CONTAINER_ID", podSandboxID.ID},        },    }    // port mappings相关参数    // ...       // dns 相关参数    // ...    return rt, nil}  

func (c *CNIConfig) addNetwork(ctx context.Context, name, cniVersion string, net *NetworkConfig, prevResult types.Result, rt *RuntimeConf) (types.Result, error) {    c.ensureExec()    pluginPath, err := c.exec.FindInPath(net.Network.Type, c.Path)    // ...    // pluginPath就是calico二进制文件路径，这里其实就是调用 calico ADD命令，并传递相关参数，参数也是上文描述的已经准备好了的    // 参数传递也是写入了环境变量，calico二进制文件可以从环境变量里取值    return invoke.ExecPluginWithResult(ctx, pluginPath, newConf.Bytes, c.args("ADD", rt), c.exec)}// AddNetworkList executes a sequence of plugins with the ADD commandfunc (c *CNIConfig) AddNetworkList(ctx context.Context, list *NetworkConfigList, rt *RuntimeConf) (types.Result, error) {    // ...    for _, net := range list.Plugins {        result, err = c.addNetwork(ctx, list.Name, list.CNIVersion, net, result, rt)        // ...    }    // ...    return result, nil}

{  "name": "k8s-pod-network",  "cniVersion": "0.3.1",  "plugins": [    {      "type": "calico",      "log_level": "debug",      "log_file_path": "/var/log/calico/cni/cni.log",      "datastore_type": "kubernetes",      "nodename": "minikube",      "mtu": 1440,      "ipam": {          "type": "calico-ipam"      },      "policy": {          "type": "k8s"      },      "kubernetes": {          "kubeconfig": "/etc/cni/net.d/calico-kubeconfig"      }    },    {      "type": "portmap",      "snat": true,      "capabilities": {"portMappings": true}    },    {      "type": "bandwidth",      "capabilities": {"bandwidth": true}    }  ]}