From 6ef46195753a4f383e931b7267cebedc40e756c5 Mon Sep 17 00:00:00 2001
From: liubo <liubo254@huawei.com>
Date: Wed, 18 Aug 2021 10:22:50 +0800
Subject: [PATCH 45/50] add dram_percent to etmem

---
 inc/etmemd_inc/etmemd_common.h  |  9 ++++
 inc/etmemd_inc/etmemd_exp.h     |  5 ++
 inc/etmemd_inc/etmemd_migrate.h |  3 +-
 inc/etmemd_inc/etmemd_scan.h    |  5 +-
 inc/etmemd_inc/etmemd_slide.h   |  1 +
 src/etmemd_src/etmemd_common.c  | 94 +++++++++++++++++++++++++++++++++
 src/etmemd_src/etmemd_migrate.c | 46 ++++++++++++++++
 src/etmemd_src/etmemd_scan.c    | 70 ++++++++++++++++++++++++
 src/etmemd_src/etmemd_slide.c   | 83 +++++++++++++++++++++++++----
 9 files changed, 304 insertions(+), 12 deletions(-)

diff --git a/inc/etmemd_inc/etmemd_common.h b/inc/etmemd_inc/etmemd_common.h
index 4127ccf..8d18f8a 100644
--- a/inc/etmemd_inc/etmemd_common.h
+++ b/inc/etmemd_inc/etmemd_common.h
@@ -20,10 +20,16 @@
 #include <stdbool.h>
 
 #define PROC_PATH "/proc/"
+#define STATUS_FILE "/status"
+#define SWAPIN "SwapIN"
+#define VMRSS "VmRSS"
+#define VMSWAP "VmSwap"
 #define FILE_LINE_MAX_LEN 1024
 #define KEY_VALUE_MAX_LEN 64
 #define DECIMAL_RADIX 10
 #define ETMEMD_MAX_PARAMETER_NUM 6
+#define BYTE_TO_KB(s) ((s) >> 10)
+#define KB_TO_BYTE(s) ((s) << 10)
 
 #define ARRAY_SIZE(array) (sizeof(array) / sizeof((array)[0]))
 
@@ -48,11 +54,14 @@ int etmemd_parse_cmdline(int argc, char *argv[], bool *is_help);
 bool check_str_format(char endptr);
 int get_int_value(const char *val, int *value);
 int get_unsigned_int_value(const char *val, unsigned int *value);
+int get_unsigned_long_value(const char *val, unsigned long *value);
 void etmemd_safe_free(void **ptr);
 
 FILE *etmemd_get_proc_file(const char *pid, const char *file, int flags, const char *mode);
 
 int get_keyword_and_value(const char *str, char *key, char *val);
+unsigned long get_pagesize(void);
+int get_mem_from_proc_file(const char *pid, const char *file_name, unsigned long *data, const char *cmpstr);
 
 int dprintf_all(int fd, const char *format, ...);
 
diff --git a/inc/etmemd_inc/etmemd_exp.h b/inc/etmemd_inc/etmemd_exp.h
index 8c57d9f..48a0018 100644
--- a/inc/etmemd_inc/etmemd_exp.h
+++ b/inc/etmemd_inc/etmemd_exp.h
@@ -39,4 +39,9 @@ struct page_refs {
     struct page_refs *next;     /* point to next page */
 };
 
+struct page_sort {
+    struct page_refs **page_refs_sort;
+    struct page_refs **page_refs;
+    int loop;
+};
 #endif
diff --git a/inc/etmemd_inc/etmemd_migrate.h b/inc/etmemd_inc/etmemd_migrate.h
index db61c69..ef20bde 100644
--- a/inc/etmemd_inc/etmemd_migrate.h
+++ b/inc/etmemd_inc/etmemd_migrate.h
@@ -17,6 +17,7 @@
 #define ETMEMD_MIGRATE_H
 
 #include "etmemd.h"
+#include "etmemd_task.h"
 
 #define COLD_PAGE   "/swap_pages"
 
@@ -26,5 +27,5 @@
 #define SWAP_ADDR_LEN   20
 
 int etmemd_grade_migrate(const char* pid, const struct memory_grade *memory_grade);
-
+unsigned long check_should_migrate(const struct task_pid *tk_pid);
 #endif
diff --git a/inc/etmemd_inc/etmemd_scan.h b/inc/etmemd_inc/etmemd_scan.h
index 09ad51c..9e5bcc4 100644
--- a/inc/etmemd_inc/etmemd_scan.h
+++ b/inc/etmemd_inc/etmemd_scan.h
@@ -77,9 +77,12 @@ struct vmas *get_vmas_with_flags(const char *pid, char **vmflags_array, int vmfl
 struct vmas *get_vmas(const char *pid);
 
 void clean_page_refs_unexpected(void *arg);
-
 void clean_memory_grade_unexpected(void *arg);
 
+void clean_page_sort_unexpected(void *arg);
+struct page_sort *alloc_page_sort(const struct task_pid *tk_pid);
+struct page_sort *sort_page_refs(struct page_refs **page_refs, const struct task_pid *tk_pid);
+
 struct page_refs *add_page_refs_into_memory_grade(struct page_refs *page_refs, struct page_refs **list);
 int init_g_page_size(void);
 int page_type_to_size(enum page_type type);
diff --git a/inc/etmemd_inc/etmemd_slide.h b/inc/etmemd_inc/etmemd_slide.h
index af48be7..93de502 100644
--- a/inc/etmemd_inc/etmemd_slide.h
+++ b/inc/etmemd_inc/etmemd_slide.h
@@ -22,6 +22,7 @@
 struct slide_params {
     struct task_executor *executor;
     int t;          /* watermark */
+    uint8_t dram_percent;
 };
 
 int fill_engine_type_slide(struct engine *eng, GKeyFile *config);
diff --git a/src/etmemd_src/etmemd_common.c b/src/etmemd_src/etmemd_common.c
index 8aad0eb..4595499 100644
--- a/src/etmemd_src/etmemd_common.c
+++ b/src/etmemd_src/etmemd_common.c
@@ -195,6 +195,19 @@ int get_unsigned_int_value(const char *val, unsigned int *value)
     return 0;
 }
 
+int get_unsigned_long_value(const char *val, unsigned long *value)
+{
+    char *pos = NULL;
+
+    errno = 0;
+    *value = strtoul(val, &pos, DECIMAL_RADIX);
+    if (check_str_format(pos[0])) {
+        etmemd_log(ETMEMD_LOG_ERR, "invalid value, must be type of unsigned long.\n");
+        return -1;
+    }
+
+    return 0;
+}
 void etmemd_safe_free(void **ptr)
 {
     if (ptr == NULL || *ptr == NULL) {
@@ -205,6 +218,36 @@ void etmemd_safe_free(void **ptr)
     *ptr = NULL;
 }
 
+static int get_status_num(char *getline, char *value, size_t value_len)
+{
+    size_t len = strlen(getline);
+    int start_cp_index = 0;
+    int end_cp_index = 0;
+    size_t i;
+
+    for (i = 0; i < len; i++) {
+        if (isdigit(getline[i])) {
+            start_cp_index = i;
+            end_cp_index = start_cp_index;
+            break;
+        }
+    }
+
+    for (; i < len; i++) {
+        if (!isdigit(getline[i])) {
+            end_cp_index = i - 1;
+            break;
+        }
+    }
+
+    if (strncpy_s(value, value_len, getline + start_cp_index, end_cp_index - start_cp_index + 1) != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "strncpy_s for result failed.\n");
+        return -1;
+    }
+
+    return 0;
+}
+
 static char *etmemd_get_proc_file_str(const char *pid, const char *file)
 {
     char *file_name = NULL;
@@ -398,3 +441,54 @@ int dprintf_all(int fd, const char *format, ...)
     va_end(args_in);
     return 0;
 }
+
+int get_mem_from_proc_file(const char *pid, const char *file_name, unsigned long *data, const char *cmpstr)
+{
+    FILE *file = NULL;
+    char value[KEY_VALUE_MAX_LEN] = {};
+    char get_line[FILE_LINE_MAX_LEN] = {};
+    unsigned long val;
+    int ret = -1;
+
+    file = etmemd_get_proc_file(pid, file_name, 0, "r");
+    if (file == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "cannot open %s for pid %s\n", file_name, pid);
+        return ret;
+    }
+
+    while (fgets(get_line, FILE_LINE_MAX_LEN - 1, file) != NULL) {
+        if (strstr(get_line, cmpstr) == NULL) {
+            continue;
+        }
+
+        if (get_status_num(get_line, value, KEY_VALUE_MAX_LEN) != 0) {
+            etmemd_log(ETMEMD_LOG_ERR, "get mem from /proc/%s/%s fail\n", pid, file_name);
+            break;
+        }
+
+        if (get_unsigned_long_value(value, &val) != 0) {
+            etmemd_log(ETMEMD_LOG_ERR, "get value with strtoul fail.\n");
+            break;
+        }
+
+        *data = val;
+        ret = 0;
+        break;
+    }
+
+    fclose(file);
+    return ret;
+}
+
+unsigned long get_pagesize(void)
+{
+    long pagesize;
+
+    pagesize = sysconf(_SC_PAGESIZE);
+    if (pagesize == -1) {
+        etmemd_log(ETMEMD_LOG_ERR, "get pageszie fail,error: %d\n", errno);
+        return -1;
+    }
+
+    return (unsigned long)pagesize;
+}
diff --git a/src/etmemd_src/etmemd_migrate.c b/src/etmemd_src/etmemd_migrate.c
index 2f29f31..639d570 100644
--- a/src/etmemd_src/etmemd_migrate.c
+++ b/src/etmemd_src/etmemd_migrate.c
@@ -20,6 +20,7 @@
 #include "etmemd.h"
 #include "etmemd_migrate.h"
 #include "etmemd_common.h"
+#include "etmemd_slide.h"
 #include "etmemd_log.h"
 
 static char *get_swap_string(struct page_refs **page_refs, int batchsize)
@@ -113,3 +114,48 @@ int etmemd_grade_migrate(const char *pid, const struct memory_grade *memory_grad
     return ret;
 }
 
+unsigned long check_should_migrate(const struct task_pid *tk_pid)
+{
+    int ret = -1;
+    unsigned long vm_rss;
+    unsigned long vm_swap;
+    unsigned long vm_cmp;
+    unsigned long need_to_swap_page_num;
+    char pid_str[PID_STR_MAX_LEN] = {0};
+    unsigned long pagesize;
+    struct slide_params *slide_params = NULL;
+
+    if (snprintf_s(pid_str, PID_STR_MAX_LEN, PID_STR_MAX_LEN - 1, "%u", tk_pid->pid) <= 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "snprintf pid fail %u", tk_pid->pid);
+        return 0;
+    }
+
+    ret = get_mem_from_proc_file(pid_str, STATUS_FILE, &vm_rss, VMRSS);
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get vmrss %s fail", pid_str);
+        return 0;
+    }
+
+    ret = get_mem_from_proc_file(pid_str, STATUS_FILE, &vm_swap, VMSWAP);
+    if (ret != 0) {
+        etmemd_log(ETMEMD_LOG_ERR, "get swapout %s fail", pid_str);
+        return 0;
+    }
+
+    slide_params = (struct slide_params *)tk_pid->tk->params;
+    if (slide_params == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "slide params is null");
+        return 0;
+    }
+
+    vm_cmp = (vm_rss + vm_swap) / 100 * slide_params->dram_percent;
+    if (vm_cmp > vm_rss) {
+        etmemd_log(ETMEMD_LOG_DEBUG, "migrate too much, stop migrate this time\n");
+        return 0;
+    }
+
+    pagesize = get_pagesize();
+    need_to_swap_page_num = KB_TO_BYTE(vm_rss - vm_cmp) / pagesize;
+
+    return need_to_swap_page_num;
+}
diff --git a/src/etmemd_src/etmemd_scan.c b/src/etmemd_src/etmemd_scan.c
index 0fb4fe6..fec6373 100644
--- a/src/etmemd_src/etmemd_scan.c
+++ b/src/etmemd_src/etmemd_scan.c
@@ -24,6 +24,7 @@
 #include "etmemd_project.h"
 #include "etmemd_engine.h"
 #include "etmemd_common.h"
+#include "etmemd_slide.h"
 #include "etmemd_log.h"
 #include "securec.h"
 
@@ -819,6 +820,46 @@ void clean_memory_grade_unexpected(void *arg)
     return;
 }
 
+void clean_page_sort_unexpected(void *arg)
+{
+    struct page_sort **msg = (struct page_sort **)arg;
+
+    if (*msg == NULL) {
+        return;
+    }
+
+    for (int i = 0; i < (*msg)->loop + 1; i++) {
+        clean_page_refs_unexpected(&((*msg)->page_refs_sort)[i]);
+    }
+
+    free(*msg);
+    *msg = NULL;
+
+    return;
+}
+
+struct page_sort *alloc_page_sort(const struct task_pid *tpid)
+{
+    struct page_sort *page_sort = NULL;
+
+    page_sort = (struct page_sort *)calloc(1, sizeof(struct page_sort));
+    if (page_sort == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "calloc page sort failed.\n");
+        return NULL;
+    }
+
+    page_sort->loop = tpid->tk->eng->proj->loop;
+
+    page_sort->page_refs_sort = (struct page_refs **)calloc((tpid->tk->eng->proj->loop + 1), sizeof(struct page_refs *));
+    if (page_sort->page_refs_sort == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "calloc page refs sort failed.\n");
+        free(page_sort);
+        return NULL;
+    }
+
+    return page_sort; 
+}
+
 struct page_refs *add_page_refs_into_memory_grade(struct page_refs *page_refs, struct page_refs **list)
 {
     struct page_refs *tmp = NULL;
@@ -851,3 +892,32 @@ void etmemd_scan_exit(void)
 {
     g_exp_scan_inited = false;
 }
+
+/* Move the colder pages by sorting page refs.
+ * Use original page_refs if dram_percent is not set.
+ * But, use the sorting result of page_refs, if dram_percent is set to (0, 100] */
+struct page_sort *sort_page_refs(struct page_refs **page_refs, const struct task_pid *tpid)
+{
+    struct slide_params *slide_params = NULL;
+    struct page_sort *page_sort = NULL;
+    struct page_refs *page_next = NULL;
+
+    page_sort = alloc_page_sort(tpid);
+    if (page_sort == NULL)
+        return NULL;
+
+    slide_params = (struct slide_params *)tpid->tk->params;
+    if (slide_params == NULL || slide_params->dram_percent == 0) {
+        page_sort->page_refs = page_refs;
+        return page_sort;
+    }
+
+    while (*page_refs != NULL) {
+        page_next = (*page_refs)->next;
+        (*page_refs)->next = (page_sort->page_refs_sort[(*page_refs)->count]);
+        (page_sort->page_refs_sort[(*page_refs)->count]) = *page_refs;
+        *page_refs = page_next;
+    }
+
+    return page_sort;
+}
diff --git a/src/etmemd_src/etmemd_slide.c b/src/etmemd_src/etmemd_slide.c
index 96d3dcc..a024178 100644
--- a/src/etmemd_src/etmemd_slide.c
+++ b/src/etmemd_src/etmemd_slide.c
@@ -28,12 +28,15 @@
 #include "etmemd_pool_adapter.h"
 #include "etmemd_file.h"
 
-static struct memory_grade *slide_policy_interface(struct page_refs **page_refs, void *params)
+static struct memory_grade *slide_policy_interface(struct page_sort **page_sort, const struct task_pid *tpid)
 {
-    struct slide_params *slide_params = (struct slide_params *)params;
+    struct slide_params *slide_params = (struct slide_params *)(tpid->tk->params);
+    struct page_refs **page_refs = NULL;
     struct memory_grade *memory_grade = NULL;
+    unsigned long need_2_swap_num;
+    volatile uint64_t count = 0;
 
-    if (params == NULL) {
+    if (slide_params == NULL) {
         etmemd_log(ETMEMD_LOG_ERR, "cannot get params for slide\n");
         return NULL;
     }
@@ -44,14 +47,41 @@ static struct memory_grade *slide_policy_interface(struct page_refs **page_refs,
         return NULL;
     }
 
-    while (*page_refs != NULL) {
-        if ((*page_refs)->count >= slide_params->t) {
-            *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->hot_pages);
-            continue;
+    if (slide_params->dram_percent == 0) {
+        page_refs = (*page_sort)->page_refs;
+
+        while (*page_refs != NULL) {
+            if ((*page_refs)->count >= slide_params->t) {
+                *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->hot_pages);
+                continue;
+            }
+            *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->cold_pages);
+        }
+
+        return memory_grade;
+    }
+
+    need_2_swap_num = check_should_migrate(tpid);
+    if (need_2_swap_num == 0)
+        goto count_out;
+
+    for (int i = 0; i < tpid->tk->eng->proj->loop + 1; i++) {
+        page_refs = &((*page_sort)->page_refs_sort[i]);
+
+        while (*page_refs != NULL) {
+            if ((*page_refs)->count >= slide_params->t) {
+                *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->hot_pages);
+                goto count_out;
+            }
+
+            *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->cold_pages);
+            count++;
+            if (count >= need_2_swap_num)
+                goto count_out;
         }
-        *page_refs = add_page_refs_into_memory_grade(*page_refs, &memory_grade->cold_pages);
     }
 
+count_out:
     return memory_grade;
 }
 
@@ -80,17 +110,32 @@ static void *slide_executor(void *arg)
     struct task_pid *tk_pid = (struct task_pid *)arg;
     struct page_refs *page_refs = NULL;
     struct memory_grade *memory_grade = NULL;
+    struct page_sort *page_sort = NULL;
 
     /* register cleanup function in case of unexpected cancellation detected,
      * and register for memory_grade first, because it needs to clean after page_refs is cleaned */
     pthread_cleanup_push(clean_memory_grade_unexpected, &memory_grade);
     pthread_cleanup_push(clean_page_refs_unexpected, &page_refs);
+    pthread_cleanup_push(clean_page_sort_unexpected, &page_sort);
 
     page_refs = etmemd_do_scan(tk_pid, tk_pid->tk);
-    if (page_refs != NULL) {
-        memory_grade = slide_policy_interface(&page_refs, tk_pid->tk->params);
+    if (page_refs == NULL) {
+        etmemd_log(ETMEMD_LOG_WARN, "pid %u cannot get page refs\n", tk_pid->pid);
+        goto scan_out;
     }
 
+    page_sort = sort_page_refs(&page_refs, tk_pid);
+    if (page_sort == NULL) {
+        etmemd_log(ETMEMD_LOG_ERR, "failed to alloc memory for page sort.", tk_pid->pid);
+        goto scan_out;
+    }
+
+    memory_grade = slide_policy_interface(&page_sort, tk_pid);
+
+scan_out:
+    /* clean up page_sort linked array */
+    pthread_cleanup_pop(1);
+
     /* no need to use page_refs any longer.
      * pop the cleanup function with parameter 1, because the items in page_refs list will be moved
      * into the at least on list of memory_grade after polidy function called if no problems happened,
@@ -131,8 +176,26 @@ static int fill_task_threshold(void *obj, void *val)
     return 0;
 }
 
+static int fill_task_dram_percent(void *obj, void *val)
+{
+    struct slide_params *params = (struct slide_params *)obj;
+    int value = parse_to_int(val);
+
+    if (value <= 0 || value > 100) {
+        etmemd_log(ETMEMD_LOG_WARN,
+                    "dram_percent %d is abnormal, the reasonable range is (0, 100],\
+                        cancle the dram_percent parameter of current task\n", value);
+        value = 0;
+    }
+
+    params->dram_percent = value;
+
+    return 0;
+}
+
 static struct config_item g_slide_task_config_items[] = {
     {"T", INT_VAL, fill_task_threshold, false},
+    {"dram_percent", INT_VAL, fill_task_dram_percent, true},
 };
 
 static int slide_fill_task(GKeyFile *config, struct task *tk)
-- 
2.27.0