From b580a60e4338d0542b7e7fbc9561277a19d44434 Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Mon, 30 May 2022 11:36:53 +0800 Subject: [PATCH 1/5] tests: add public var test The plugsched upgrade the kernel in the form of modules. Some data used by modules are defined in the kernel, so they exist in the .ko file as undefined symbols, which are then relocated when the scheduler is loaded. This test check if these symbols are 'UND'. Co-developed-by: Erwei Deng Signed-off-by: Xuchun Shang --- tests/bundles/ci | 1 + tests/test_public_var/assert | 53 ++++++++++++++++++++++++++++++++ tests/test_public_var/patch.diff | 0 3 files changed, 54 insertions(+) create mode 100644 tests/test_public_var/assert create mode 100644 tests/test_public_var/patch.diff diff --git a/tests/bundles/ci b/tests/bundles/ci index d1666d4..f8ceb74 100644 --- a/tests/bundles/ci +++ b/tests/bundles/ci @@ -1 +1,2 @@ quick_start +public_var \ No newline at end of file diff --git a/tests/test_public_var/assert b/tests/test_public_var/assert new file mode 100644 index 0000000..276ef14 --- /dev/null +++ b/tests/test_public_var/assert @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 + +import subprocess +import sh +import yaml +import os +import sys + +class TestPublicVar: + def setup_class(self): + print("Public vars test") + cur_sys = str(sh.uname('-r')).strip() + scheduler_rpm = sh.glob(os.path.join('/tmp/work', 'scheduler*.rpm')) + if len(scheduler_rpm) != 1: + print("Please check your scheduler rpm"); + self.teardown_class() + sys.exit(1) + scheduler_rpm = scheduler_rpm[0] + sh.rpm('-ivh', scheduler_rpm) + module = '/var/plugsched/' + cur_sys + '/scheduler.ko' + yaml_file = '/tmp/work/scheduler/working/sched_boundary.yaml' + with open(yaml_file, 'r') as f: + config = yaml.load(f, Loader=yaml.FullLoader) + self.public_vars = config['global_var']['extra_public'] + cmd = "objdump -t " + module + self.symtab = str(subprocess.check_output(cmd, shell=True)).split('\\n') + + def test_syms(self): + for var in self.public_vars: + for cur_line in self.symtab: + if cur_line == '' or cur_line.split()[-1] != var: + continue + if not '*UND*' in cur_line: + self.error_handler(var) + break + + def error_handler(self, var): + print("Public var: " + str(var) + "is not UND") + self.teardown_class() + print("Public vars test " + "\033[31mFAILED\033[0m") + sys.exit(1) + + def teardown_class(self): + tmp = subprocess.Popen("lsmod | grep scheduler", shell=True, stdout=subprocess.PIPE) + if tmp.stdout.read() != b'': + sh.rpm('-e', 'scheduler-xxx') + +if __name__ == '__main__': + test_unit = TestPublicVar() + test_unit.setup_class() + test_unit.test_syms() + test_unit.teardown_class() + print("Public vars test " + "\033[32mPASS\033[0m") \ No newline at end of file diff --git a/tests/test_public_var/patch.diff b/tests/test_public_var/patch.diff new file mode 100644 index 0000000..e69de29 -- Gitee From cb389a1512ec64679bea42f93227cc32e6528845 Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Mon, 30 May 2022 11:41:50 +0800 Subject: [PATCH 2/5] tests: add var uniformity test Perform inheritance tests on some public data (ensure consistency before and after module loading / unloading), such as sysctl and debugfs. We will compare whether the corresponding data is consistent before and after the module is loaded. Co-developed-by: Erwei Deng Signed-off-by: Xuchun Shang --- tests/bundles/ci | 3 +- tests/test_var_uniformity/assert | 98 ++++++++++++++++++++++++++++ tests/test_var_uniformity/patch.diff | 0 3 files changed, 100 insertions(+), 1 deletion(-) create mode 100644 tests/test_var_uniformity/assert create mode 100644 tests/test_var_uniformity/patch.diff diff --git a/tests/bundles/ci b/tests/bundles/ci index f8ceb74..3901fbe 100644 --- a/tests/bundles/ci +++ b/tests/bundles/ci @@ -1,2 +1,3 @@ quick_start -public_var \ No newline at end of file +public_var +var_uniformity \ No newline at end of file diff --git a/tests/test_var_uniformity/assert b/tests/test_var_uniformity/assert new file mode 100644 index 0000000..e7bd6cf --- /dev/null +++ b/tests/test_var_uniformity/assert @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +from typing import Dict +import sh +import os +import subprocess +import sys + +class TestVarUniformity: + def setup_class(self): + print("Var uniformity test") + self.global_name = [ + "/proc/sys/kernel/sched_child_runs_first", + "/proc/sys/kernel/sched_min_granularity_ns", + "/proc/sys/kernel/sched_latency_ns", + "/proc/sys/kernel/sched_wakeup_granularity_ns", + "/proc/sys/kernel/sched_tunable_scaling", + "/proc/sys/kernel/sched_migration_cost_ns", + "/proc/sys/kernel/sched_nr_migrate", + "/proc/sys/kernel/sched_schedstats", + "/proc/sys/kernel/numa_balancing_scan_delay_ms", + "/proc/sys/kernel/numa_balancing_scan_period_min_ms", + "/proc/sys/kernel/numa_balancing_scan_period_max_ms", + "/proc/sys/kernel/numa_balancing_scan_size_mb", + "/proc/sys/kernel/numa_balancing", + "/proc/sys/kernel/sched_rt_period_us", + "/proc/sys/kernel/sched_rt_runtime_us", + "/proc/sys/kernel/sched_rr_timeslice_ms", + "/proc/sys/kernel/sched_autogroup_enabled", + "/proc/sys/kernel/sched_cfs_bandwidth_slice_us", + "/sys/kernel/debug/sched_debug", + ] + + def before_change(self): + self.orig_data = {} + self.record_data(self.orig_data) + self.load_scheduler() + self.data_after_load = {} + self.record_data(self.data_after_load) + + def record_data(self, dict: Dict): + for item in self.global_name: + if not os.path.exists(item): + continue + dict[item] = str(sh.cat(item)).strip() + + def load_scheduler(self): + scheduler_rpm = sh.glob(os.path.join('/tmp/work', 'scheduler*.rpm')) + if len(scheduler_rpm) != 1: + print("Please check your scheduler rpm"); + self.teardown_class() + sys.exit(1) + scheduler_rpm = scheduler_rpm[0] + sh.rpm('-ivh', scheduler_rpm) + + def after_change_unload(self): + self.modify_data() + self.data_after_modified = {} + self.record_data(self.data_after_modified) + sh.rpm('-e', 'scheduler-xxx') + self.data_after_unload = {} + self.record_data(self.data_after_unload) + + def modify_data(self): + def reverse(ch): + if ch.isdigit(): + return '1' if ch == '0' else str(int(ch) - 1) + return 'N' if ch == 'Y' else 'Y' + + for k, v in self.orig_data.items(): + sh.echo(reverse(v), _out=k) + + def teardown_class(self): + tmp = subprocess.Popen("lsmod | grep scheduler", shell=True, stdout=subprocess.PIPE) + if tmp.stdout.read() != b'': + sh.rpm('-e', 'scheduler-xxx') + for k, v in self.orig_data.items(): + sh.echo(v, _out=k) + + def test_data_uniformity(self): + self.before_change() + if not self.orig_data == self.data_after_load: + self.error_handler() + self.after_change_unload() + if not self.data_after_modified == self.data_after_unload: + self.error_handler() + + def error_handler(self): + print("Var uniformity test " + "\033[31mFAILED\033[0m") + self.teardown_class() + sys.exit(1) + +if __name__ == '__main__': + unit_test = TestVarUniformity() + unit_test.setup_class() + unit_test.test_data_uniformity() + unit_test.teardown_class() + print("Var uniformity test " + "\033[32mPASS\033[0m") diff --git a/tests/test_var_uniformity/patch.diff b/tests/test_var_uniformity/patch.diff new file mode 100644 index 0000000..e69de29 -- Gitee From 07901c0234533d7e9e4e6920ae0d4119210be24d Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Mon, 30 May 2022 11:44:56 +0800 Subject: [PATCH 3/5] tests: add cpu_throttle test CFS bandwidth control is one of the representative features of CFS core. This test ensures the inheritance of CFS bandwidth control related data (consistency is guaranteed before and after module loading/unloading). The test steps are as follows: 1) Create a cpu cgroup, run busyloop (infinite loop) in it, set cpu.period and cpu.quota , check if the CPU% is correct. 2) Load the module and check the correctness of the CPU% of the busyloop process. 3) Modify the value of quota and check the correctness of CPU%. 4) Unload the module and check the correctness of CPU%. Co-developed-by: Erwei Deng Signed-off-by: Xuchun Shang --- tests/bundles/ci | 3 +- tests/test_cpu_throttle/assert | 89 ++++++++++++++++++++++++++++++ tests/test_cpu_throttle/patch.diff | 0 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/test_cpu_throttle/assert create mode 100644 tests/test_cpu_throttle/patch.diff diff --git a/tests/bundles/ci b/tests/bundles/ci index 3901fbe..0c90678 100644 --- a/tests/bundles/ci +++ b/tests/bundles/ci @@ -1,3 +1,4 @@ quick_start public_var -var_uniformity \ No newline at end of file +var_uniformity +cpu_throttle \ No newline at end of file diff --git a/tests/test_cpu_throttle/assert b/tests/test_cpu_throttle/assert new file mode 100644 index 0000000..50c28d2 --- /dev/null +++ b/tests/test_cpu_throttle/assert @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 + +import subprocess +import sh +import os +import sys + +class TestCPUThrottle: + def setup_class(self): + self.test_flag = True + print("CPU throttle test") + sh.mkdir('/sys/fs/cgroup/cpu/test') + + def init_cgroup(self): + cmd = "while :; do :; done" + self.child = subprocess.Popen(cmd, shell=True) + sh.echo(self.child.pid, _out='/sys/fs/cgroup/cpu/test/cgroup.procs') + + def set_cfs_quota(self, t_us): + sh.echo(t_us, _out='/sys/fs/cgroup/cpu/test/cpu.cfs_quota_us') + + def test_all(self): + self.set_cfs_quota('50000') + self.init_cgroup() + self.check_le_75() + self.check_after_load() + self.set_cfs_quota('100000') + self.check_gt_75() + self.check_after_unload() + + def check_le_75(self): + sh.yum('-y', 'install', 'sysstat') + cmd = "pidstat -h -u -p " + str(self.child.pid) + " 1 1 | awk '{print $7}' | tail -1" + cpu_util = float(subprocess.check_output(cmd, shell=True).split()[0]) + # assert cpu_util <= 75 + if cpu_util > 75: + self.error_handler(0, 75) + + def check_after_load(self): + scheduler_rpm = sh.glob(os.path.join('/tmp/work', 'scheduler*.rpm')) + if len(scheduler_rpm) != 1: + print("Please check your scheduler rpm"); + self.teardown_class() + sys.exit(1) + scheduler_rpm = scheduler_rpm[0] + sh.rpm('-ivh', scheduler_rpm) + cmd = "pidstat -h -u -p " + str(self.child.pid) + " 1 1 | awk '{print $7}' | tail -1" + cpu_util = float(subprocess.check_output(cmd, shell=True).split()[0]) + # assert cpu_util <= 75 + if cpu_util > 75: + self.error_handler(0, 75) + + def check_gt_75(self): + cmd = "pidstat -h -u -p " + str(self.child.pid) + " 1 1 | awk '{print $7}' | tail -1" + cpu_util = float(subprocess.check_output(cmd, shell=True).split()[0]) + # assert cpu_util >= 75 + if cpu_util < 75: + self.error_handler(1, 75) + + def check_after_unload(self): + sh.rpm('-e', 'scheduler-xxx') + cmd = "pidstat -h -u -p " + str(self.child.pid) + " 1 1 | awk '{print $7}' | tail -1" + cpu_util = float(subprocess.check_output(cmd, shell=True).split()[0]) + # assert cpu_util >= 75 + if cpu_util < 75: + self.error_handler(1, 75) + + def teardown_class(self): + self.child.kill() + self.child.wait() + sh.rmdir("/sys/fs/cgroup/cpu/test") + tmp = subprocess.Popen("lsmod | grep scheduler", shell=True, stdout=subprocess.PIPE) + if tmp.stdout.read() != b'': + sh.rpm('-e', 'scheduler-xxx') + + def error_handler(self, ty, bound): + err_msg = "CPU util should " + "less than" if ty == 0 else "greater than" + str(bound) + print(err_msg) + print("CPU throttle test " + "\033[31mFAILED\033[0m") + self.teardown_class() + sys.exit(1) + + +if __name__ == '__main__': + test_unit = TestCPUThrottle() + test_unit.setup_class() + test_unit.test_all() + test_unit.teardown_class() + print("CPU throttle test " + "\033[32mPASS\033[0m") diff --git a/tests/test_cpu_throttle/patch.diff b/tests/test_cpu_throttle/patch.diff new file mode 100644 index 0000000..e69de29 -- Gitee From 96fea2743c70c354b57b178e9f54375e58899929 Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Mon, 30 May 2022 11:47:35 +0800 Subject: [PATCH 4/5] tests: add CPU balance test Cherry-pick the upstream patch https://github.com/torvalds/linux/commit/2208cdaa56c957e20d8e16f28819aeb47851cb1e Modify the imbalance_pct from 125 to 117. This test checks if the parameter can be modified correctly. Signed-off-by: Xuchun Shang --- tests/bundles/ci | 3 ++- tests/test_cpu_bal/assert | 45 +++++++++++++++++++++++++++++++++++ tests/test_cpu_bal/patch.diff | 32 +++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 1 deletion(-) create mode 100644 tests/test_cpu_bal/assert create mode 100644 tests/test_cpu_bal/patch.diff diff --git a/tests/bundles/ci b/tests/bundles/ci index 0c90678..b3e52fd 100644 --- a/tests/bundles/ci +++ b/tests/bundles/ci @@ -1,4 +1,5 @@ quick_start public_var var_uniformity -cpu_throttle \ No newline at end of file +cpu_throttle +cpu_bal \ No newline at end of file diff --git a/tests/test_cpu_bal/assert b/tests/test_cpu_bal/assert new file mode 100644 index 0000000..cc0bc87 --- /dev/null +++ b/tests/test_cpu_bal/assert @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 + +import os +import sh +import sys +import subprocess +class CPUBalTest: + def setup_class(self): + print("CPU Balance test") + self.new_val = 117 + + def test_cpu_bal(self): + self.load_scheduler() + sh.echo('0', _out='/sys/devices/system/cpu/cpu0/online') + val = int(sh.cat('/proc/sys/kernel/sched_domain/cpu1/domain0/imbalance_pct').split()[0]) + if val != self.new_val: + self.error_handler() + + def load_scheduler(self): + scheduler_rpm = sh.glob(os.path.join('/tmp/work', 'scheduler*.rpm')) + if len(scheduler_rpm) != 1: + print("Please check your scheduler rpm"); + self.teardown_class() + sys.exit(1) + scheduler_rpm = scheduler_rpm[0] + sh.rpm('-ivh', scheduler_rpm) + + def error_handler(self): + print("The parameter imbalance_pct is not modified.") + print("CPU Balance test " + "\033[31mFAILED\033[0m") + self.teardown_class() + sys.exit(1) + + def teardown_class(self): + tmp = subprocess.Popen("lsmod | grep scheduler", shell=True, stdout=subprocess.PIPE) + if tmp.stdout.read() != b'': + sh.rpm('-e', 'scheduler-xxx') + +if __name__ == '__main__': + test_unit = CPUBalTest() + test_unit.setup_class() + test_unit.test_cpu_bal() + test_unit.teardown_class() + print("CPU Balance test " + "\033[32mPASS\033[0m") + diff --git a/tests/test_cpu_bal/patch.diff b/tests/test_cpu_bal/patch.diff new file mode 100644 index 0000000..dbd53e2 --- /dev/null +++ b/tests/test_cpu_bal/patch.diff @@ -0,0 +1,32 @@ +# https://github.com/torvalds/linux/commit/2208cdaa56c957e20d8e16f28819aeb47851cb1e +# sched/fair: Reduce minimal imbalance threshold +# The 25% default imbalance threshold for DIE and NUMA domain is large +# enough to generate significant unfairness between threads. A typical +# example is the case of 11 threads running on 2x4 CPUs. The imbalance of +# 20% between the 2 groups of 4 cores is just low enough to not trigger +# the load balance between the 2 groups. We will have always the same 6 +# threads on one group of 4 CPUs and the other 5 threads on the other +# group of CPUS. With a fair time sharing in each group, we ends up with +# +20% running time for the group of 5 threads. + +# Consider decreasing the imbalance threshold for overloaded case where we +# use the load to balance task and to ensure fair time sharing. + +# Signed-off-by: Vincent Guittot +# Signed-off-by: Peter Zijlstra (Intel) +# Reviewed-by: Phil Auld +# Acked-by: Hillf Danton + +diff --git a/scheduler/kernel/sched/mod/topology.c b/scheduler/kernel/sched/mod/topology.c +index 249bec7b0a4c..41df62884cea 100644 +--- a/scheduler/kernel/sched/mod/topology.c ++++ b/scheduler/kernel/sched/mod/topology.c +@@ -1097,7 +1097,7 @@ sd_init(struct sched_domain_topology_level *tl, + .min_interval = sd_weight, + .max_interval = 2*sd_weight, + .busy_factor = 32, +- .imbalance_pct = 125, ++ .imbalance_pct = 117, + + .cache_nice_tries = 0, + -- Gitee From 3ec1091cdf2888868491206dfe82bab8e0e7d0d9 Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Mon, 30 May 2022 11:50:25 +0800 Subject: [PATCH 5/5] tests: add sched syscall test After the scheduler module is loaded, the scheduler related syscall is upgraded to the version in the scheduler(plugsched) module. This test check the semantic correctness of the new sched syscall. The syscall releated scheduler included in this tests are 1) sched_setaffinity and sched_getaffinity (by cgroup/cpuset) 2) sched_setparam and sched_getparam (by chrt command) 3) sched_setscheduler and sched_getscheduler (by chrt command) Co-developed-by: Erwei Deng Signed-off-by: Xuchun Shang --- tests/bundles/ci | 3 +- tests/test_sched_syscall/assert | 71 +++++++++++++++++++++++++++++ tests/test_sched_syscall/patch.diff | 0 3 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 tests/test_sched_syscall/assert create mode 100644 tests/test_sched_syscall/patch.diff diff --git a/tests/bundles/ci b/tests/bundles/ci index b3e52fd..58f50d6 100644 --- a/tests/bundles/ci +++ b/tests/bundles/ci @@ -2,4 +2,5 @@ quick_start public_var var_uniformity cpu_throttle -cpu_bal \ No newline at end of file +cpu_bal +sched_syscall \ No newline at end of file diff --git a/tests/test_sched_syscall/assert b/tests/test_sched_syscall/assert new file mode 100644 index 0000000..1cb9043 --- /dev/null +++ b/tests/test_sched_syscall/assert @@ -0,0 +1,71 @@ +#!/usr/bin/env python3 + +import subprocess +import sys +import sh +import os + +class TestSchedSyscall: + def setup_class(self): + print("Sched syscall test") + cmd = "while :; do :; done" + self.child = subprocess.Popen(cmd, shell=True) + + def load_scheduler(self): + scheduler_rpm = sh.glob(os.path.join('/tmp/work', 'scheduler*.rpm')) + if len(scheduler_rpm) != 1: + print("Please check your scheduler rpm"); + self.teardown_class() + sys.exit(1) + scheduler_rpm = scheduler_rpm[0] + sh.rpm('-ivh', scheduler_rpm) + + def test_cpuset(self): + fa_mems = sh.cat("/sys/fs/cgroup/cpuset/cpuset.mems").split()[0] + fa_cpus = sh.cat("/sys/fs/cgroup/cpuset/cpuset.cpus").split()[0] + sh.mkdir("/sys/fs/cgroup/cpuset/test") + self.load_scheduler() + sh.echo(fa_mems, _out="/sys/fs/cgroup/cpuset/test/cpuset.mems") + sh.echo(fa_cpus, _out="/sys/fs/cgroup/cpuset/test/cpuset.cpus") + ch_mems = sh.cat("/sys/fs/cgroup/cpuset/test/cpuset.mems").split()[0] + ch_cpus = sh.cat("/sys/fs/cgroup/cpuset/test/cpuset.cpus").split()[0] + if fa_mems != ch_mems or fa_cpus != ch_cpus: + self.error_handler() + sh.rmdir("/sys/fs/cgroup/cpuset/test") + + def test_policy_and_prio(self): + cmd = "chrt -p -f 10 " + str(self.child.pid) + subprocess.Popen(cmd, shell=True) + res = sh.chrt('-p', self.child.pid).split('\n') + if res[0].split()[-1] != 'SCHED_FIFO' or res[1].split()[-1] != '10': + self.error_handler() + + def test_all(self): + self.test_cpuset() + self.test_policy_and_prio() + + def error_handler(self): + self.child.kill() + self.child.wait() + sh.rmdir("/sys/fs/cgroup/cpuset/test") + self.unload_scheduler() + print("Sched syscall test " + "\033[31mFAILED\033[0m") + sys.exit(1) + + def unload_scheduler(self): + tmp = subprocess.Popen("lsmod | grep scheduler", shell=True, stdout=subprocess.PIPE) + if tmp.stdout.read() != b'': + sh.rpm('-e', 'scheduler-xxx') + + def teardown_class(self): + self.child.kill() + self.child.wait() + self.unload_scheduler() + +if __name__ == '__main__': + test_unit = TestSchedSyscall() + test_unit.setup_class() + test_unit.test_all() + test_unit.teardown_class() + print("Sched syscall test " + "\033[32mPASS\033[0m") + diff --git a/tests/test_sched_syscall/patch.diff b/tests/test_sched_syscall/patch.diff new file mode 100644 index 0000000..e69de29 -- Gitee