[RFC PATCH 2/2] vmrd: add implementation of memory plugin interfaces

[Sudarshan Rajagopalan]

Implementation of memory plugin interfaces such as memory_plug_request,
memory_unplug_request etc. using virtio_mem kernel driver.

The userspace daemon makes ioctl calls to kernel requesting for adding/
removing memory to the VM. The size request is aligned to virtio-mem
device size. Modified version of virtio-mem driver is used that supports
memory_on_hotplug feature and add/remove memory requests via ioctl calls.
Link to the ioctl handling function is below:
https://git.codelinaro.org/clo/la/kernel/msm-5.15/-/blob/kernel.lnx.5.15.r33-rel/drivers/virtio/qti_virtio_mem.c#L185
modified virtio-mem driver: https://git.codelinaro.org/clo/la/kernel/msm-5.15/-/blob/kernel.lnx.5.15.r33-rel/drivers/virtio/virtio_mem.c

We use a kernel driver called mem-buf for communication between the VMs.
The memory plug request by virtio-mem is made by calling mem_buf_alloc each of
virtio-mem block size iteratively to accommodate the requested size.
https://git.codelinaro.org/clo/la/kernel/msm-5.15/-/blob/kernel.lnx.5.15.r33-rel/drivers/virtio/virtio_mem.c#L1373

The mem-buf driver uses to Linux Gunyah driver (mem_buf_map_mem_s2) to
communicate with Hypervisor requesting to map the memory into S2 page-tables.
This is currently tested only on Gunyah Hypervisor and not on other Hypervisors.
mem_buf_map_mem_s2: https://git.codelinaro.org/clo/la/kernel/msm-5.15/-/blob/kernel.lnx.5.15.r33-rel/drivers/soc/qcom/mem_buf/mem-buf-dev-gh.c#L207

The daemon also gets virtio-mem device configuration details such as
device_bloc_size, max_threshold of resizing the VM etc. from the kernel
via sysfs nodes of virtio-mem device.
---
 vmrd.cpp | 129 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 117 insertions(+), 12 deletions(-)

diff --git a/vmrd.cpp b/vmrd.cpp
index 1bf5812..090f90a 100644
--- a/vmrd.cpp
+++ b/vmrd.cpp
@@ -19,11 +19,18 @@
 #include <sys/epoll.h>
 #include <sys/types.h>
 #include <sys/time.h>
+#include <sys/ioctl.h>
 #include <sys/syslog.h>
 #include <fcntl.h>
 #include <inttypes.h>
 #include <pthread.h>
+#include <cutils/memory.h>
 #include <atomic>
+#include <vector>
+#include <fcntl.h>
+
+/* qti_virtio_mem uapi header */
+#include <linux/qti_virtio_mem.h>
 
 #define SIZE_1MB    0x00100000
 #define SIZE_1KB    0x00000400
@@ -205,23 +212,71 @@ using namespace std;
  * are needed to support the functionality of vmrd.
  */
 
+/* qti_virtio_mem device fd */
+static int virtio_mem_fd = -1;
+
+/* mem_buf fds returned by virtio-mem driver */
+static vector<int> array_memfd;
+
+#define QVM_VIRTIO_MEM_DEV_PATH     "/dev/qti_virtio_mem"
+#define QVM_SYS_DEVICE_PATH         "/sys/devices/virtual/qti_virtio_mem/qti_virtio_mem"
+#define QVM_BLOCK_SIZE_PATH         QVM_SYS_DEVICE_PATH"/device_block_size"
+#define QVM_MAX_PLUGIN_THRES_PATH   QVM_SYS_DEVICE_PATH"/max_plugin_threshold"
+#define QVM_NUM_BLOCK_PLUGGED_PATH  QVM_SYS_DEVICE_PATH"/device_block_plugged"
+
+static int virtio_mem_plug_memory(int64_t size, const std::string& name)
+{
+    struct qti_virtio_mem_ioc_hint_create_arg arg = {};
+    int ret;
+
+    if (virtio_mem_fd < 0)
+        return -ENOTTY;
+
+    arg.size = size;
+    strlcpy(arg.name, name.c_str(), sizeof(arg.name));
+
+    ret = ioctl(virtio_mem_fd, QTI_VIRTIO_MEM_IOC_HINT_CREATE, &arg);
+    if (ret) {
+        LOGE("MemorySizeHint() failed");
+        return ret;
+    }
+
+    return arg.fd;
+}
+
 static int memory_plug_init(void) {
-    LOGE("memory plug request not supported");
-    return -ENOTTY;
+    virtio_mem_fd = open(QVM_VIRTIO_MEM_DEV_PATH, O_RDONLY | O_CLOEXEC);
+    if (virtio_mem_fd < 0) {
+        LOGE("Unable to open %s: %s", QVM_VIRTIO_MEM_DEV_PATH, strerror(errno));
+        return errno;
+    }
+
+    return 0;
 }
 
 static void memory_plug_deinit(void) {
-    LOGE("memory plug request not supported");
+    if (virtio_mem_fd >= 0)
+        close(virtio_mem_fd);
 }
 
 /*
  * Plugs in memory of given size into the system by requesting it from host VM.
  * This call is expected to be blocking call.
  */
+
 static int memory_plug_request(uint64_t size) {
-    (void) size;
-    LOGE("Memory plug request not supported");
-    return -ENOTTY;
+    int memfd;
+
+    memfd = virtio_mem_plug_memory(size * SIZE_1MB, "vmrd");
+    if (memfd < 0) {
+        LOGE("failed to suggest memory size hint");
+        return -1;
+    }
+
+    LOGI("Memory of size %lu MB plugged-in successfully", size);
+    array_memfd.push_back(memfd);
+
+    return 0;
 }
 
 /*
@@ -230,8 +285,18 @@ static int memory_plug_request(uint64_t size) {
  * his call is expected to be blocking call.
  */
 static int memory_unplug_request(uint64_t size) {
-    (void) size;
-    LOGE("Memory unplug request not supported");
+    int res;
+
+    if (array_memfd.size()) {
+        res = close(array_memfd.back());
+        array_memfd.pop_back();
+        if (res)
+            LOGE("Failed to unplug one memory chunk of size %lu MB", size);
+
+        return res;
+    }
+
+    LOGE("No memory available to unplug");
     return -ENOTTY;
 }
 
@@ -243,8 +308,32 @@ static int memory_unplug_request(uint64_t size) {
  * This call is expected to be blocking call.
  */
 static int __unused memory_unplug_all_request(void) {
-    LOGE("Memory unplug all request not supported");
-    return -ENOTTY;
+    uint64_t initial_count, unplugged_count = 0, res;
+
+    initial_count = array_memfd.size();
+    if (!initial_count) {
+        LOGE("No memory available to unplug");
+        return 0;
+    }
+
+    while (array_memfd.size()) {
+        LOGI("releasing one memory chunk to host VM");
+        res = close(array_memfd.back());
+        array_memfd.pop_back();
+        if (res)
+            LOGE("failed to unplug one memory chunk");
+        else
+            unplugged_count++;
+    }
+
+    if (unplugged_count < initial_count)
+        LOGI("not all memory chunks were unplugged. initial_count: %lu unplugged_count: %lu",
+                initial_count, unplugged_count);
+    else
+        LOGI("Successfully unplugged all memory chunks. unplugged_count: %lu",
+                unplugged_count);
+
+    return unplugged_count;
 }
 
 static int write_file(const char *file_path, char *s) {
@@ -305,7 +394,15 @@ static char *read_file(const char *file_path) {
  * Memory block size or resolution.
  */
 static int get_memory_plugin_resolution(uint64_t *plugin_resolution_mb) {
-    *plugin_resolution_mb = DEFAULT_PLUGIN_RESOLUTION_MB;
+    char *buf;
+
+    buf = read_file(QVM_BLOCK_SIZE_PATH);
+    if (!buf)
+        return -EINVAL;
+
+    *plugin_resolution_mb = strtoul(buf, 0, 10);
+    *plugin_resolution_mb /= SIZE_1MB;
+
     return 0;
 }
 
@@ -313,7 +410,15 @@ static int get_memory_plugin_resolution(uint64_t *plugin_resolution_mb) {
  * Total max memory that the system (guest VM) allows to be pluuged-in.
  */
 static int get_max_memory_plugin_allowed(uint64_t *max_memory_plugin_mb) {
-    *max_memory_plugin_mb = DEFAULT_MAX_MEMORY_PLUGIN_MB;
+    char *buf;
+
+    buf = read_file(QVM_MAX_PLUGIN_THRES_PATH);
+    if (!buf)
+        return -EINVAL;
+
+    *max_memory_plugin_mb = strtoul(buf, 0, 10);
+    *max_memory_plugin_mb /= SIZE_1MB;
+
     return 0;
 }
 
-- 
2.7.4