From ebe2f4690f9ad9af845eac430e23daf66016ab98 Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Tue, 14 Jun 2022 16:24:28 +0800 Subject: [PATCH 1/2] src: Migrating from python2 to python3 Now the python version of plugsched is Python 2.x which is no longer supported. So migrating to Python 3.x is necessary. The differences of features used in plugsched between Python 2.x and Python 3.x are as follows: 1) map() returns an iterator, not a list in Python 3.x, So we need to use list to wrap its return values. 2) filter() is the same as map() in Python 3.x. 3) unicode is replaced with str in Python 3.x. 4) iteritems is replaced with items in Python 3.x. 5) sublist of function parameter are no longer supported in Python 3.x. And there is a python-related issue described in https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe The key point releated with plugsched is that the /usr/local path will not be included in python sys.path when dealing with rpmbuild. So we need to add the PYTHONPATH manually before rpmbuild start which implemented in add_python_path(cli.py). Signed-off-by: Xuchun Shang --- cli.py | 19 ++++++++++++++++--- sched_boundary/process.py | 11 ++++++----- sched_boundary/sched_boundary.py | 12 ++++++------ src/Makefile.plugsched | 4 ++-- src/sidecar.py | 4 ++-- tools/yaml-diff.py | 2 +- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/cli.py b/cli.py index 19c2592..029c3a3 100755 --- a/cli.py +++ b/cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause @@ -120,7 +120,7 @@ class Plugsched(object): break return i - candidates = map(os.path.basename, glob('%s/configs/%s*' % (self.plugsched_path, self.major))) + candidates = list(map(os.path.basename, glob('%s/configs/%s*' % (self.plugsched_path, self.major)))) if len(candidates) == 0: logging.fatal('''Can't find config directory, please add config for kernel %s''', self.KVER) @@ -208,9 +208,22 @@ class Plugsched(object): logging.info("Succeed!") + # when python3 working with rpmbuild, the /usr/local/python* path + # won't be in included in sys/path which results in some modules + # can't be find. So we need to add the PYTHONPATH manually. + # The detail about this can be find in + # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe + def add_python_path(self): + py_ver = sys.version[0:3] + python_path = '/usr/local/lib64/python' + py_ver + '/site-packages' + python_path += os.pathsep + python_path += '/usr/local/lib/python' + py_ver + '/site-packages' + os.environ["PYTHONPATH"] = python_path + def cmd_build(self): if not os.path.exists(self.work_dir): logging.fatal("plugsched: Can't find %s", self.work_dir) + self.add_python_path() logging.info("Preparing rpmbuild environment") rpmbuild_root = os.path.join(self.plugsched_path, 'rpmbuild') self.plugsched_sh.rm('rpmbuild', recursive=True, force=True) @@ -240,7 +253,7 @@ if __name__ == '__main__': rpmbuild_root = mkdtemp() sh.rpmbuild('--define', '%%_topdir %s' % rpmbuild_root, - '--define', '%%__python %s' % '/usr/bin/python2', + '--define', '%%__python %s' % '/usr/bin/python3', '-rp', '--nodeps', kernel_src_rpm) src = glob('kernel*/linux*', rpmbuild_root + '/BUILD/') diff --git a/sched_boundary/process.py b/sched_boundary/process.py index b6ec506..2c9f885 100644 --- a/sched_boundary/process.py +++ b/sched_boundary/process.py @@ -26,7 +26,7 @@ modpath = None Loader.add_constructor(resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, lambda loader, node: set(loader.construct_sequence(node))) Dumper.add_representer(set, lambda dumper, node: dumper.represent_list(node)) -Dumper.add_representer(unicode, +Dumper.add_representer(str, lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data)) def read_config(): @@ -122,15 +122,16 @@ def inflect(initial_insiders, edges): global __insiders __insiders = copy.deepcopy(initial_insiders) while True: - delete_insider = filter(None, map(inflect_one, edges)) + delete_insider = list(filter(None, list(map(inflect_one, edges)))) if not delete_insider: break __insiders -= set(delete_insider) return __insiders global_fn_dict = {} -def lookup_if_global((name, file)): +def lookup_if_global(name_and_file): # Returns None if function is a gcc built-in function + name, file = name_and_file file = global_fn_dict.get(name, None) if file == '?' else file return (name, file) if file else None @@ -142,7 +143,7 @@ if __name__ == '__main__': config = read_config() config['mod_files_basename'] = {os.path.basename(f): f for f in config['mod_files']} config['mod_header_files'] = [f for f in config['mod_files'] if f.endswith('.h')] - metas = map(read_meta, all_meta_files()) + metas = list(map(read_meta, all_meta_files())) func_class = { 'fn': set(), @@ -207,7 +208,7 @@ if __name__ == '__main__': if struct not in m['struct']: continue all_set |= set(m['struct'][struct]['all_fields']) - for field, users in m['struct'][struct]['public_fields'].iteritems(): + for field, users in m['struct'][struct]['public_fields'].items(): p_user = set(map(tuple, users)) & func_class['public_user'] if p_user: user_set |= p_user diff --git a/sched_boundary/sched_boundary.py b/sched_boundary/sched_boundary.py index fceae3b..6ed20d1 100644 --- a/sched_boundary/sched_boundary.py +++ b/sched_boundary/sched_boundary.py @@ -1,9 +1,9 @@ +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause from collections import defaultdict -from builtins import super -from itertools import izip, groupby as _groupby +from itertools import groupby as _groupby from yaml import load, dump, resolver, CLoader as Loader, CDumper as Dumper import json import re @@ -418,14 +418,14 @@ class SchedBoundaryCollect(SchedBoundary): def groupby(it, grouper, selector): sorted_list = sorted(it, key=grouper) - return dict((k, map(selector, v)) for k, v in _groupby(sorted_list, grouper)) + return dict((k, list(map(selector, v))) for k, v in _groupby(sorted_list, grouper)) - for struct, user_fields in public_fields.iteritems(): + for struct, user_fields in public_fields.items(): self.struct_properties[struct.name.name] = { "all_fields": [f.name for f in struct.fields if f.name], "public_fields": groupby(user_fields, - grouper=lambda (user, field): field.name, - selector=lambda (user, field): (user.name, os.path.relpath(user.location.file))) + grouper=lambda user_and_field: user_and_field[1].name, + selector=lambda user_and_field: (user_and_field[0].name, os.path.relpath(user_and_field[0].location.file))) } def collect_edges(self): diff --git a/src/Makefile.plugsched b/src/Makefile.plugsched index 4825438..22b3dbd 100644 --- a/src/Makefile.plugsched +++ b/src/Makefile.plugsched @@ -18,9 +18,9 @@ plugsched: scripts prepare sidecar $(MAKE) CFLAGS_MODULE=-fkeep-static-functions -C $(srctree) M=$(plugsched_modpath) modules sidecar: $(plugsched_modpath)/export_jump_sidecar.h - python2 $(plugsched_tmpdir)/sidecar.py $< ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) + python3 $(plugsched_tmpdir)/sidecar.py $< ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) collect: $(core-y) $(core-m) $(drivers-y) $(drivers-m) $(net-y) $(net-m) $(virt-y) analyze: - python2 $(plugsched_tmpdir)/process.py ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) + python3 $(plugsched_tmpdir)/process.py ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) extract: $(objs) diff --git a/src/sidecar.py b/src/sidecar.py index ba1d341..703f1f1 100644 --- a/src/sidecar.py +++ b/src/sidecar.py @@ -28,8 +28,8 @@ if __name__ == '__main__': modpath = sys.argv[4] with open(tmpdir + 'symbol_resolve/undefined_functions_sidecar.h', 'w') as f: - for fn, pos in sympos.iteritems(): + for fn, pos in sympos.items(): f.write('{"%s", %d},\n' % (fn, pos)) with open(modpath + 'tainted_functions_sidecar.h', 'w') as f: - for fn, pos in sympos.iteritems(): + for fn, pos in sympos.items(): f.write('TAINTED_FUNCTION(%s,%d)\n' % (fn, pos if pos else 1)) diff --git a/tools/yaml-diff.py b/tools/yaml-diff.py index a3c2df9..cdfe280 100755 --- a/tools/yaml-diff.py +++ b/tools/yaml-diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause -- Gitee From 1260249768b34dc43981b38bc1b4744cf7739e8a Mon Sep 17 00:00:00 2001 From: Xuchun Shang Date: Thu, 16 Jun 2022 11:45:22 +0800 Subject: [PATCH 2/2] dockerfile: update to support python3 The docker environment needs to be updated to support python3 version. Signed-off-by: Xuchun Shang --- Dockerfile.aarch64 | 30 ++++++++++++++++-------------- Dockerfile.x86_64 | 29 +++++++++++++++-------------- 2 files changed, 31 insertions(+), 28 deletions(-) diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 1dcffed..8c1bd1e 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -2,20 +2,22 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause From registry.cn-hangzhou.aliyuncs.com/alinux/alinux3-aa64 -RUN yum install python2 python2-devel gcc gcc-c++ wget libyaml-devel -y && \ - wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python2 get-pip.py -RUN pip install --upgrade setuptools && \ - pip install --global-option='--with-libyaml' pyyaml && \ - pip install six sh coloredlogs future fire jinja2 docopt && \ - yum install python2-lxml python2-pygments python2-six -y && \ - yum install systemd git make bison flex \ - gcc-plugin-devel \ - elfutils-libelf-devel openssl openssl-devel \ - elfutils-devel-static \ - glibc-static zlib-static \ - libstdc++-static \ - rpm-build rsync bc perl -y && \ +RUN yum install python3 python3-devel python3-lxml gcc gcc-c++ wget libyaml-devel -y && \ + wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py +RUN pip3 install --upgrade setuptools && \ + pip3 install --global-option='--with-libyaml' pyyaml && \ + pip3 install sh coloredlogs fire jinja2 docopt && \ + yum install make bison flex python3-lxml python3-six python3-pygments \ + gcc-plugin-devel \ + systemd git \ + elfutils-libelf-devel openssl openssl-devel \ + elfutils-devel-static \ + glibc-static zlib-static \ + libstdc++-static \ + platform-python-devel \ + rpm-build rsync bc perl -y && \ + yum install gcc-python-plugin --enablerepo=Plus -y && \ yum clean all RUN git clone https://gitee.com/src-anolis-sig/gcc-python-plugin.git && \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 26e42c1..ef51c02 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -2,20 +2,21 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause From openanolis/anolisos:8.4-x86_64 -RUN yum install python2 python2-devel gcc gcc-c++ wget libyaml-devel -y && \ - wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python2 get-pip.py -RUN pip install --upgrade setuptools && \ - pip install --global-option='--with-libyaml' pyyaml && \ - pip install six sh coloredlogs future fire jinja2 docopt && \ - yum install make bison flex \ - gcc-plugin-devel \ - elfutils-libelf-devel openssl openssl-devel \ - elfutils-devel-static \ - glibc-static zlib-static \ - libstdc++-static \ - platform-python-devel \ - rpm-build rsync bc perl -y && \ +RUN yum install python3 python3-devel python3-lxml gcc gcc-c++ wget libyaml-devel -y && \ + wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py +RUN pip3 install --upgrade setuptools && \ + pip3 install --global-option='--with-libyaml' pyyaml && \ + pip3 install sh coloredlogs fire jinja2 docopt && \ + yum install make bison flex python3-lxml python3-six python3-pygments \ + gcc-plugin-devel.x86_64 \ + systemd git \ + elfutils-libelf-devel.x86_64 openssl openssl-devel \ + elfutils-devel-static \ + glibc-static zlib-static \ + libstdc++-static \ + platform-python-devel \ + rpm-build rsync bc perl -y && \ yum install gcc-python-plugin --enablerepo=Plus -y && \ yum clean all -- Gitee