From 0674bfac4dd1ab812432334c779ab718dc54bc8b Mon Sep 17 00:00:00 2001
From: liuxu <liuxu156@huawei.com>
Date: Thu, 11 Apr 2024 11:02:19 +0800
Subject: [PATCH 68/69] cdi:invoke cdi operate when init isulad and create
 container

---
 src/cmd/isulad/main.c                         | 11 +++
 src/daemon/common/cri/v1/v1_cri_helpers.cc    | 79 +++++++++++++++++++
 src/daemon/common/cri/v1/v1_cri_helpers.h     |  3 +
 src/daemon/config/daemon_arguments.c          |  4 +
 src/daemon/config/isulad_config.c             |  8 ++
 .../v1/v1_cri_container_manager_service.cc    |  8 ++
 .../executor/container_cb/execution_create.c  |  9 +++
 .../modules/service/service_container.c       | 10 +++
 src/daemon/modules/spec/specs_mount.c         | 43 +++++++++-
 src/daemon/modules/spec/specs_mount.h         |  4 +
 src/daemon/modules/spec/verify.c              |  2 +-
 11 files changed, 179 insertions(+), 2 deletions(-)

diff --git a/src/cmd/isulad/main.c b/src/cmd/isulad/main.c
index 9fa87bdb..3e2249d7 100644
--- a/src/cmd/isulad/main.c
+++ b/src/cmd/isulad/main.c
@@ -83,6 +83,9 @@
 #endif
 #include "id_name_manager.h"
 #include "cgroup.h"
+#ifdef ENABLE_CDI
+#include "cdi_operate_api.h"
+#endif /* ENABLE_CDI */
 
 sem_t g_daemon_shutdown_sem;
 sem_t g_daemon_wait_shutdown_sem;
@@ -1400,6 +1403,14 @@ static int isulad_server_init_common()
     }
 #endif
 
+#ifdef ENABLE_CDI
+    if (args->json_confs->enable_cdi &&
+        cdi_operate_registry_init(args->json_confs->cdi_spec_dirs, args->json_confs->cdi_spec_dirs_len) != 0) {
+        ERROR("Failed to init CDI module");
+        goto out;
+    }
+#endif /* ENABLE_CDI */
+
     if (spec_module_init() != 0) {
         ERROR("Failed to init spec module");
         goto out;
diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.cc b/src/daemon/common/cri/v1/v1_cri_helpers.cc
index ea5c8bb5..520d23d4 100644
--- a/src/daemon/common/cri/v1/v1_cri_helpers.cc
+++ b/src/daemon/common/cri/v1/v1_cri_helpers.cc
@@ -22,6 +22,7 @@
 
 #include <isula_libutils/log.h>
 #include <isula_libutils/parse_common.h>
+#include <isula_libutils/auto_cleanup.h>
 
 #include "v1_cri_security_context.h"
 #include "cri_helpers.h"
@@ -33,6 +34,9 @@
 #include "isulad_config.h"
 #include "sha256.h"
 #include "v1_naming.h"
+#ifdef ENABLE_CDI
+#include "cdi_operate_api.h"
+#endif /* ENABLE_CDI */
 
 namespace CRIHelpersV1 {
 
@@ -666,4 +670,79 @@ std::unique_ptr<runtime::v1::ContainerStatus> GetContainerStatus(service_executo
     return contStatus;
 }
 
+#ifdef ENABLE_CDI
+static int InsertCDIDevices(std::unordered_set<std::string> &fromCRI, const std::string &devName,
+                            string_array *requested, Errors &err)
+{
+    if (fromCRI.find(devName) == fromCRI.end()) {
+        fromCRI.insert(devName);
+        if (util_append_string_array(requested, devName.c_str()) != 0) {
+            ERROR("Out of memory");
+            err.Errorf("Out of memory");
+            return -1;
+        }
+        DEBUG("Appended device: %s", devName.c_str());
+    } else {
+        INFO("Skipping duplicate CDI device %s", devName.c_str());
+    }
+    return 0;
+}
+ 
+void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err)
+{
+    std::unordered_set<std::string> fromCRI;
+    __isula_auto_string_array_t string_array *requested = nullptr;
+    __isula_auto_string_array_t string_array *keys = nullptr;
+    __isula_auto_string_array_t string_array *devices = nullptr;
+    json_map_string_string *annotations = nullptr;
+    __isula_auto_free char *error = nullptr;
+ 
+    if (hostconfig == nullptr) {
+        ERROR("Invalid input arguments");
+        err.Errorf("Invalid input arguments");
+        return;
+    }
+    
+    if (config.cdi_devices().empty() && config.annotations().empty()) {
+        return;
+    }
+    requested = (string_array *)util_common_calloc_s(sizeof(*requested));
+    if (requested == nullptr) {
+        ERROR("Out of memory");
+        err.Errorf("Out of memory");
+        return;
+    }
+    if (!config.cdi_devices().empty()) {
+        for (int i = 0; i < config.cdi_devices().size(); i++) {
+            if (InsertCDIDevices(fromCRI, config.cdi_devices(i).name(), requested, err) != 0) {
+                goto free_out;
+            }
+        }
+    }
+    if (!config.annotations().empty()) {
+        annotations = CRIHelpers::MakeAnnotations(config.annotations(), err);
+        if (err.NotEmpty()) {
+            goto free_out;
+        }
+        if (cdi_operate_parse_annotations(annotations, &keys, &devices, &error) != 0) {
+            ERROR("Failed to parse CDI annotations: %s", error);
+            err.Errorf("Failed to parse CDI annotations: %s", error);
+            goto free_out;
+        }
+        for (size_t i = 0; i < devices->len; i++) {
+            if (InsertCDIDevices(fromCRI, std::string(devices->items[i]), requested, err) != 0) {
+                goto free_out;
+            }
+        }
+    }
+    hostconfig->cdi_requested_devices = requested->items;
+    requested->items = nullptr;
+    hostconfig->cdi_requested_devices_len = requested->len;
+    requested->len = 0;
+ 
+free_out:
+    free_json_map_string_string(annotations);
+}
+#endif /* ENABLE_CDI */
+
 } // v1 namespace CRIHelpers
diff --git a/src/daemon/common/cri/v1/v1_cri_helpers.h b/src/daemon/common/cri/v1/v1_cri_helpers.h
index 1578c428..22cffd0d 100644
--- a/src/daemon/common/cri/v1/v1_cri_helpers.h
+++ b/src/daemon/common/cri/v1/v1_cri_helpers.h
@@ -79,6 +79,9 @@ std::string CRISandboxerConvert(const std::string &runtime);
 
 void ApplySandboxSecurityContextToHostConfig(const runtime::v1::LinuxSandboxSecurityContext &context, host_config *hc,
                                              Errors &error);
+#ifdef ENABLE_CDI
+void GenerateCDIRequestedDevices(const runtime::v1::ContainerConfig &config, host_config *hostconfig, Errors &err);
+#endif /* ENABLE_CDI */
 
 auto GetContainerStatus(service_executor_t *m_cb, const std::string &containerID, Errors &error)
 -> std::unique_ptr<runtime::v1::ContainerStatus>;
diff --git a/src/daemon/config/daemon_arguments.c b/src/daemon/config/daemon_arguments.c
index 0ae6268a..ef15934a 100644
--- a/src/daemon/config/daemon_arguments.c
+++ b/src/daemon/config/daemon_arguments.c
@@ -173,6 +173,10 @@ int service_arguments_init(struct service_arguments *args)
         goto free_out;
     }
 
+#ifdef ENABLE_CDI
+    args->json_confs->enable_cdi = false;
+#endif /* ENABLE_CDI */
+
     ret = 0;
 
 free_out:
diff --git a/src/daemon/config/isulad_config.c b/src/daemon/config/isulad_config.c
index 778ff921..695a0d95 100644
--- a/src/daemon/config/isulad_config.c
+++ b/src/daemon/config/isulad_config.c
@@ -1830,6 +1830,14 @@ int merge_json_confs_into_global(struct service_arguments *args)
     args->json_confs->metrics_port = tmp_json_confs->metrics_port;
 #endif
 
+#ifdef ENABLE_CDI
+    args->json_confs->enable_cdi = tmp_json_confs->enable_cdi;
+    args->json_confs->cdi_spec_dirs = tmp_json_confs->cdi_spec_dirs;
+    tmp_json_confs->cdi_spec_dirs = NULL;
+    args->json_confs->cdi_spec_dirs_len = tmp_json_confs->cdi_spec_dirs_len;
+    tmp_json_confs->cdi_spec_dirs_len = 0;
+#endif /* ENABLE_CDI */
+ 
 out:
     free(err);
     free_isulad_daemon_configs(tmp_json_confs);
diff --git a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
index e86dafae..1097c32c 100644
--- a/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
+++ b/src/daemon/entry/cri/v1/v1_cri_container_manager_service.cc
@@ -199,6 +199,14 @@ auto ContainerManagerService::GenerateCreateContainerHostConfig(
         }
     }
 
+#ifdef ENABLE_CDI
+    CRIHelpersV1::GenerateCDIRequestedDevices(containerConfig, hostconfig, error);
+    if (error.NotEmpty()) {
+        ERROR("Failed to generate CDI requested devices");
+        goto cleanup;
+    }
+#endif /* ENABLE_CDI */
+
     return hostconfig;
 
 cleanup:
diff --git a/src/daemon/executor/container_cb/execution_create.c b/src/daemon/executor/container_cb/execution_create.c
index a9102226..785b4e27 100644
--- a/src/daemon/executor/container_cb/execution_create.c
+++ b/src/daemon/executor/container_cb/execution_create.c
@@ -63,6 +63,7 @@
 #include "runtime_api.h"
 #include "id_name_manager.h"
 #include "mailbox.h"
+#include "specs_mount.h"
 
 #ifdef ENABLE_CRI_API_V1
 static bool validate_sandbox_info(const container_sandbox_info *sandbox)
@@ -512,6 +513,14 @@ static oci_runtime_spec *generate_oci_config(host_config *host_spec, const char
         goto error_out;
     }
 
+#ifdef ENABLE_CDI
+    ret = inject_CDI_devcies_for_oci_spec(oci_spec, host_spec);
+    if (ret != 0) {
+        ERROR("Failed to inject CDI devices");
+        goto error_out;
+    }
+#endif /* ENABLE_CDI */
+
     return oci_spec;
 
 error_out:
diff --git a/src/daemon/modules/service/service_container.c b/src/daemon/modules/service/service_container.c
index eb7ce4f4..b19a134a 100644
--- a/src/daemon/modules/service/service_container.c
+++ b/src/daemon/modules/service/service_container.c
@@ -2003,6 +2003,16 @@ static defs_process *make_exec_process_spec(const container_config *container_sp
         }
 
         spec->no_new_privileges = oci_spec->process->no_new_privileges;
+
+#ifdef ENABLE_CDI
+        // extend step: merge env from oci_spec which comes from injected devices
+        ret = defs_process_add_multiple_env(spec, (const char **)oci_spec->process->env,
+            oci_spec->process->env_len);
+        if (ret != 0) {
+            ERROR("Failed to dup oci env for exec process spec");
+            goto err_out;
+        }
+#endif /* ENABLE_CDI */
     }
 
     // for oci runtime:
diff --git a/src/daemon/modules/spec/specs_mount.c b/src/daemon/modules/spec/specs_mount.c
index 50ee9a85..12bd261b 100644
--- a/src/daemon/modules/spec/specs_mount.c
+++ b/src/daemon/modules/spec/specs_mount.c
@@ -28,6 +28,7 @@
 #include <isula_libutils/container_config_v2.h>
 #include <isula_libutils/json_common.h>
 #include <isula_libutils/oci_runtime_config_linux.h>
+#include <isula_libutils/auto_cleanup.h>
 #include <limits.h>
 #include <stdint.h>
 
@@ -54,6 +55,9 @@
 #include "volume_api.h"
 #include "parse_volume.h"
 #include "specs_api.h"
+#ifdef ENABLE_CDI
+#include "cdi_operate_api.h"
+#endif /* ENABLE_CDI */
 
 enum update_rw {
     update_rw_untouch,
@@ -3582,6 +3586,15 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon
         oci_spec->linux->resources->devices_len += 1;
     }
 
+    // extend step: inject CDI devcies
+#ifdef ENABLE_CDI
+    ret = inject_CDI_devcies_for_oci_spec(oci_spec, hostconfig);
+    if (ret != 0) {
+        ERROR("Failed to inject CDI devices");
+        return -1;
+    }
+#endif /* ENABLE_CDI */
+
     // Step8: do update devices and cgroup device rules at here
     if (hostconfig->privileged) {
         // Step8.1: for priviledged container, we should merge all devices under /dev
@@ -3592,4 +3605,32 @@ int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostcon
     }
 
     return ret;
-}
\ No newline at end of file
+}
+
+#ifdef ENABLE_CDI
+int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig)
+{
+    int ret = 0;
+    string_array devices_array = { 0 };
+    __isula_auto_free char *error = NULL;
+    
+    if (oci_spec == NULL || hostconfig == NULL) {
+        ERROR("Invalid params");
+        return -1;
+    }
+    if (hostconfig->cdi_requested_devices == NULL) {
+        return 0;
+    }
+    devices_array.items = hostconfig->cdi_requested_devices;
+    devices_array.len = hostconfig->cdi_requested_devices_len;
+    devices_array.cap = hostconfig->cdi_requested_devices_len;
+    if (cdi_operate_refresh() != 0) {
+        WARN("CDI registry has errors, please check past logs");
+    }
+    if (cdi_operate_inject_devices(oci_spec, &devices_array) != 0) {
+        ERROR("Failed to inject CDI devices");
+        ret = -1;
+    }
+    return ret;
+}
+#endif /* ENABLE_CDI */
\ No newline at end of file
diff --git a/src/daemon/modules/spec/specs_mount.h b/src/daemon/modules/spec/specs_mount.h
index b742ca35..1406c557 100644
--- a/src/daemon/modules/spec/specs_mount.h
+++ b/src/daemon/modules/spec/specs_mount.h
@@ -49,6 +49,10 @@ int setup_ipc_dirs(host_config *host_spec, container_config_v2_common_config *v2
 
 int update_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig);
 
+#ifdef ENABLE_CDI
+int inject_CDI_devcies_for_oci_spec(oci_runtime_spec *oci_spec, host_config *hostconfig);
+#endif /* ENABLE_CDI */
+
 #ifdef __cplusplus
 }
 #endif
diff --git a/src/daemon/modules/spec/verify.c b/src/daemon/modules/spec/verify.c
index af790d6e..617b7f23 100644
--- a/src/daemon/modules/spec/verify.c
+++ b/src/daemon/modules/spec/verify.c
@@ -1518,7 +1518,7 @@ static int verify_custom_mount(defs_mount **mounts, size_t len)
 
     for (i = 0; i < len; ++i) {
         iter = *(mounts + i);
-        if (iter == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) {
+        if (iter == NULL || iter->type == NULL || strcmp(iter->type, MOUNT_TYPE_BIND)) {
             continue;
         }
 
-- 
2.34.1