From 9707219f4ec2b12012620a3e5cec90b61a3ca2c4 Mon Sep 17 00:00:00 2001
From: liubo <liubo254@huawei.com>
Date: Tue, 18 Jan 2022 11:03:13 +0800
Subject: [PATCH 16/33] etmem: add sysmem_threshold and swap_threshold
 parameters to etmem

As a memory expansion tool, etmem performs memory swap operation
for the target process by default after it is enabled. However,
in some specific scenarios, in order to obtain the ultimate
performance of the business, it is necessary to consider
the timing of memory swapping by etmem.

Add the sysmem_threshold parameter to control the start and
stop of memory swap out when the system available memory
meets the requirements.

Add the swap_threshold parameter to limit the absolute
value of the memory size reserved by the process in DRAM.
When the target process memory is lower than this value, do not
swap out any memory.

Signed-off-by: liubo <liubo254@huawei.com>
---
 etmem/conf/slide_conf.yaml                |   2 +
 etmem/inc/etmemd_inc/etmemd_common.h      |  10 +++
 etmem/inc/etmemd_inc/etmemd_project_exp.h |   1 +
 etmem/inc/etmemd_inc/etmemd_slide.h       |   1 +
 etmem/src/etmemd_src/etmemd_common.c      |  58 +++++++++++++-
 etmem/src/etmemd_src/etmemd_project.c     |  32 ++++++--
 etmem/src/etmemd_src/etmemd_slide.c       | 123 ++++++++++++++++++++++++++++++
 7 files changed, 220 insertions(+), 7 deletions(-)

diff --git a/etmem/conf/slide_conf.yaml b/etmem/conf/slide_conf.yaml
index 75f0220..511d657 100644
--- a/etmem/conf/slide_conf.yaml
+++ b/etmem/conf/slide_conf.yaml
@@ -4,6 +4,7 @@ scan_type=page
 loop=1
 interval=1
 sleep=1
+sysmem_threshold=50
 
 [engine]
 name=slide
@@ -17,3 +18,4 @@ type=name
 value=mysql
 T=1
 max_threads=1
+swap_threshold=10g
diff --git a/etmem/inc/etmemd_inc/etmemd_common.h b/etmem/inc/etmemd_inc/etmemd_common.h
index 8d18f8a..576d38a 100644
--- a/etmem/inc/etmemd_inc/etmemd_common.h
+++ b/etmem/inc/etmemd_inc/etmemd_common.h
@@ -21,6 +21,7 @@
 
 #define PROC_PATH "/proc/"
 #define STATUS_FILE "/status"
+#define PROC_MEMINFO "meminfo"
 #define SWAPIN "SwapIN"
 #define VMRSS "VmRSS"
 #define VMSWAP "VmSwap"
@@ -30,14 +31,21 @@
 #define ETMEMD_MAX_PARAMETER_NUM 6
 #define BYTE_TO_KB(s) ((s) >> 10)
 #define KB_TO_BYTE(s) ((s) << 10)
+#define CONVERT_GB_2_KB (1024 * 1024)
 
 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
 
 /* in some system the max length of pid may be larger than 5, so we use 10 herr */
 #define PID_STR_MAX_LEN 10
+#define SWAP_THRESHOLD_MAX_LEN          10
 
 #define PIPE_FD_LEN 2
 
+enum swap_type {
+    DONT_SWAP = 0,
+    DO_SWAP,
+};
+
 /*
  * function: parse cmdline passed to etmemd server.
  *
@@ -65,4 +73,6 @@ int get_mem_from_proc_file(const char *pid, const char *file_name, unsigned long
 
 int dprintf_all(int fd, const char *format, ...);
 
+int get_swap_threshold_inKB(char *string);
+
 #endif
diff --git a/etmem/inc/etmemd_inc/etmemd_project_exp.h b/etmem/inc/etmemd_inc/etmemd_project_exp.h
index 8740f7e..fc3c85e 100644
--- a/etmem/inc/etmemd_inc/etmemd_project_exp.h
+++ b/etmem/inc/etmemd_inc/etmemd_project_exp.h
@@ -42,6 +42,7 @@ struct project {
     char *name;
     enum scan_type type;
     void *scan_param;
+    int sysmem_threshold;
     bool start;
     struct engine *engs;
 
diff --git a/etmem/inc/etmemd_inc/etmemd_slide.h b/etmem/inc/etmemd_inc/etmemd_slide.h
index 93de502..7c80502 100644
--- a/etmem/inc/etmemd_inc/etmemd_slide.h
+++ b/etmem/inc/etmemd_inc/etmemd_slide.h
@@ -22,6 +22,7 @@
 struct slide_params {
     struct task_executor *executor;
     int t;          /* watermark */
+    int swap_threshold;
     uint8_t dram_percent;
 };
 
diff --git a/etmem/src/etmemd_src/etmemd_common.c b/etmem/src/etmemd_src/etmemd_common.c
index ebf6232..caa5826 100644
--- a/etmem/src/etmemd_src/etmemd_common.c
+++ b/etmem/src/etmemd_src/etmemd_common.c
@@ -256,7 +256,7 @@ static char *etmemd_get_proc_file_str(const char *pid, const char *file)
     char *file_name = NULL;
     size_t file_str_size;
 
-    file_str_size = strlen(PROC_PATH) + strlen(pid) + strlen(file) + 1;
+    file_str_size = strlen(PROC_PATH) + strlen(file) + 1 + (pid == NULL ? 0 : strlen(pid));
     file_name = (char *)calloc(file_str_size, sizeof(char));
     if (file_name == NULL) {
         etmemd_log(ETMEMD_LOG_ERR, "malloc for %s path fail\n", file);
@@ -264,7 +264,7 @@ static char *etmemd_get_proc_file_str(const char *pid, const char *file)
     }
 
     if (snprintf_s(file_name, file_str_size, file_str_size - 1, 
-                    "%s%s%s", PROC_PATH, pid, file) == -1) {
+                    "%s%s%s", PROC_PATH, pid ? pid : "", file) == -1) {
         etmemd_log(ETMEMD_LOG_ERR, "snprintf for %s fail\n", file);
         free(file_name);
         return NULL;
@@ -495,3 +495,57 @@ unsigned long get_pagesize(void)
 
     return (unsigned long)pagesize;
 }
+
+int get_swap_threshold_inKB(char *string)
+{
+    int len;
+    int i;
+    int ret = -1;
+    char *swap_threshold_string = NULL;
+    int swap_threshold_inGB;
+    int swap_threshold_inKB;
+
+    if (string == NULL) {
+        goto out;
+    }
+
+    len = strlen(string);
+    if (len > SWAP_THRESHOLD_MAX_LEN) {
+        etmemd_log(ETMEMD_LOG_ERR, "swap_threshold string is too long.\n");
+        goto out;
+    }
+
+    swap_threshold_string = (char *)calloc(len, sizeof(char));
+    if (swap_threshold_string == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "calloc swap_threshold_string fail.\n");
+        goto out;
+    }
+
+    for (i = 0; i < len - 1; i++) {
+        if (isdigit(string[i])) {
+            swap_threshold_string[i] = string[i];
+            continue;
+        }
+        etmemd_log(ETMEMD_LOG_ERR, "the swap_threshold contain wrong parameter.\n");
+        goto free_out;
+    }
+
+    if (string[i] != 'g' && string[i] != 'G') {
+        etmemd_log(ETMEMD_LOG_ERR, "the swap_threshold should in G or g.\n");
+        goto free_out;
+    }
+
+    if (get_int_value(swap_threshold_string, &swap_threshold_inGB) != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get_int_value swap_threshold faild.\n");
+        goto free_out;
+    }
+
+    swap_threshold_inKB = swap_threshold_inGB * CONVERT_GB_2_KB;
+    ret = swap_threshold_inKB;
+
+free_out:
+    free(swap_threshold_string);
+
+out:
+    return ret;
+}
diff --git a/etmem/src/etmemd_src/etmemd_project.c b/etmem/src/etmemd_src/etmemd_project.c
index fa4293b..459e140 100644
--- a/etmem/src/etmemd_src/etmemd_project.c
+++ b/etmem/src/etmemd_src/etmemd_project.c
@@ -30,12 +30,13 @@
 #include "etmemd_file.h"
 #include "etmemd_log.h"
 
-#define MAX_INTERVAL_VALUE  1200
-#define MAX_SLEEP_VALUE     1200
-#define MAX_LOOP_VALUE      120
+#define MAX_INTERVAL_VALUE              1200
+#define MAX_SLEEP_VALUE                 1200
+#define MAX_LOOP_VALUE                  120
+#define MAX_SYSMEM_THRESHOLD_VALUE      100
 
-#define MAX_OBJ_NAME_LEN    64
-#define MIN_NR_MIN_VAL      3
+#define MAX_OBJ_NAME_LEN                64
+#define MIN_NR_MIN_VAL                  3
 
 static SLIST_HEAD(project_list, project) g_projects = SLIST_HEAD_INITIALIZER(g_projects);
 
@@ -630,9 +631,27 @@ static int fill_project_scan_type(void *obj, void *val)
     return 0;
 }
 
+/* fill the project parameter: sysmem_threshold
+ * sysmem_threshold: [0, 100]. do not swap any memory out if system free memory is higher than sysmem_threshold */
+static int fill_project_sysmem_threshold(void *obj, void *val)
+{
+    struct project *proj = (struct project *)obj;
+    int sysmem_threshold = parse_to_int(val);
+
+    if (sysmem_threshold < 0 || sysmem_threshold > MAX_SYSMEM_THRESHOLD_VALUE) {
+        etmemd_log(ETMEMD_LOG_WARN, "invaild project sysmem_threshold value %d, it must between 0 and 100.\n",
+                   sysmem_threshold, MAX_SYSMEM_THRESHOLD_VALUE);
+        sysmem_threshold = -1;
+    }
+
+    proj->sysmem_threshold = sysmem_threshold;
+    return 0;
+}
+
 static struct config_item g_project_config_items[] = {
     {"name", STR_VAL, fill_project_name, false},
     {"scan_type", STR_VAL, fill_project_scan_type, false},
+    {"sysmem_threshold", INT_VAL, fill_project_sysmem_threshold, true},
 };
 
 static void clear_project(struct project *proj)
@@ -688,6 +707,9 @@ enum opt_result etmemd_project_add(GKeyFile *config)
         etmemd_log(ETMEMD_LOG_ERR, "alloc memory for project fail\n");
         return OPT_INTER_ERR;
     }
+
+    proj->sysmem_threshold = -1;
+
     if (project_fill_by_conf(config, proj) != 0) {
         etmemd_log(ETMEMD_LOG_ERR, "fill project from configuration file fail\n");
         free(proj);
diff --git a/etmem/src/etmemd_src/etmemd_slide.c b/etmem/src/etmemd_src/etmemd_slide.c
index 8362224..cbc4b17 100644
--- a/etmem/src/etmemd_src/etmemd_slide.c
+++ b/etmem/src/etmemd_src/etmemd_slide.c
@@ -17,6 +17,7 @@
 #include <stdio.h>
 #include <string.h>
 #include <malloc.h>
+#include <unistd.h>
 
 #include "securec.h"
 #include "etmemd_log.h"
@@ -106,6 +107,104 @@ static int slide_do_migrate(unsigned int pid, const struct memory_grade *memory_
     return ret;
 }
 
+static int check_sysmem_lower_threshold(struct task_pid *tk_pid)
+{
+    unsigned long mem_total;
+    unsigned long mem_free;
+    int vm_cmp;
+    int ret;
+
+    ret = get_mem_from_proc_file(NULL, PROC_MEMINFO, &mem_total, "MemTotal");
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get memtotal fail\n");
+        return DONT_SWAP;
+    }
+
+    ret = get_mem_from_proc_file(NULL, PROC_MEMINFO, &mem_free, "MemFree");
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get memfree fail\n");
+        return DONT_SWAP;
+    }
+
+    /* Calculate the free memory percentage in 0 - 100 */
+    vm_cmp = (mem_free * 100) / mem_total;
+    if (vm_cmp < tk_pid->tk->eng->proj->sysmem_threshold) {
+        return DO_SWAP;
+    }
+
+    return DONT_SWAP;
+}
+
+static int check_pid_should_swap(const char *pid, unsigned long vmrss, const struct task_pid *tk_pid)
+{
+    unsigned long vmswap;
+    unsigned long vmcmp;
+    int ret;
+
+    ret = get_mem_from_proc_file(pid, STATUS_FILE, &vmswap, "VmSwap");
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get VmSwap fail\n");
+        return DONT_SWAP;
+    }
+
+    /* Calculate the total amount of memory that can be swappout for the current process
+     * and check whether the memory is larger than the current swapout amount.
+     * If true, continue swap-out; otherwise, abort the swap-out process. */
+    vmcmp = (vmrss + vmswap) / 100 * tk_pid->tk->eng->proj->sysmem_threshold;
+    if (vmcmp > vmswap) {
+        return DO_SWAP;
+    }
+
+    return DONT_SWAP;
+}
+
+static int check_pidmem_lower_threshold(struct task_pid *tk_pid)
+{
+    struct slide_params *params = NULL;
+    unsigned long vmrss;
+    int ret;
+    char pid_str[PID_STR_MAX_LEN] = {0};
+
+    params = (struct slide_params *)tk_pid->tk->params;
+    if (params == NULL) {
+        return DONT_SWAP;
+    }
+
+    if (snprintf_s(pid_str, PID_STR_MAX_LEN, PID_STR_MAX_LEN - 1, "%u", tk_pid->pid) <= 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "snprintf pid fail %u", tk_pid->pid);
+        return DONT_SWAP;
+    }
+
+    ret = get_mem_from_proc_file(pid_str, STATUS_FILE, &vmrss, "VmRSS");
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get VmRSS fail\n");
+        return DONT_SWAP;
+    }
+
+    if (params->swap_threshold == 0) {
+        return check_pid_should_swap(pid_str, vmrss, tk_pid);
+    }
+
+    if ((int)vmrss > params->swap_threshold) {
+        return DO_SWAP;
+    }
+
+    return DONT_SWAP;
+}
+
+static int check_should_swap(struct task_pid *tk_pid)
+{
+    if (tk_pid->tk->eng->proj->sysmem_threshold == -1) {
+        return DO_SWAP;
+    }
+
+    if (check_sysmem_lower_threshold(tk_pid) == DONT_SWAP) {
+        return DONT_SWAP;
+    }
+
+    return check_pidmem_lower_threshold(tk_pid);
+}
+
 static void *slide_executor(void *arg)
 {
     struct task_pid *tk_pid = (struct task_pid *)arg;
@@ -113,6 +212,10 @@ static void *slide_executor(void *arg)
     struct memory_grade *memory_grade = NULL;
     struct page_sort *page_sort = NULL;
 
+    if (check_should_swap(tk_pid) == DONT_SWAP) {
+        return NULL;
+    }
+
     /* register cleanup function in case of unexpected cancellation detected,
      * and register for memory_grade first, because it needs to clean after page_refs is cleaned */
     pthread_cleanup_push(clean_memory_grade_unexpected, &memory_grade);
@@ -194,8 +297,28 @@ static int fill_task_dram_percent(void *obj, void *val)
     return 0;
 }
 
+static int fill_task_swap_threshold(void *obj, void *val)
+{
+    struct slide_params *params = (struct slide_params *)obj;
+    char *swap_threshold_string = (char *)val;
+    int swap_threshold = get_swap_threshold_inKB(swap_threshold_string);
+
+    free(swap_threshold_string);
+
+    if (swap_threshold < 0) {
+        etmemd_log(ETMEMD_LOG_WARN,
+                   "parse swap_threshold failed.\n");
+        return -1;
+    }
+
+    params->swap_threshold = swap_threshold;
+
+    return 0;
+}
+
 static struct config_item g_slide_task_config_items[] = {
     {"T", INT_VAL, fill_task_threshold, false},
+    {"swap_threshold", STR_VAL, fill_task_swap_threshold, true},
     {"dram_percent", INT_VAL, fill_task_dram_percent, true},
 };
 
-- 
1.8.3.1