diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 1dcffed54354e5166615ea7f1e6353c0e8372050..8c1bd1e74b873ff579d0142589579256296c60b7 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -2,20 +2,22 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause From registry.cn-hangzhou.aliyuncs.com/alinux/alinux3-aa64 -RUN yum install python2 python2-devel gcc gcc-c++ wget libyaml-devel -y && \ - wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python2 get-pip.py -RUN pip install --upgrade setuptools && \ - pip install --global-option='--with-libyaml' pyyaml && \ - pip install six sh coloredlogs future fire jinja2 docopt && \ - yum install python2-lxml python2-pygments python2-six -y && \ - yum install systemd git make bison flex \ - gcc-plugin-devel \ - elfutils-libelf-devel openssl openssl-devel \ - elfutils-devel-static \ - glibc-static zlib-static \ - libstdc++-static \ - rpm-build rsync bc perl -y && \ +RUN yum install python3 python3-devel python3-lxml gcc gcc-c++ wget libyaml-devel -y && \ + wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py +RUN pip3 install --upgrade setuptools && \ + pip3 install --global-option='--with-libyaml' pyyaml && \ + pip3 install sh coloredlogs fire jinja2 docopt && \ + yum install make bison flex python3-lxml python3-six python3-pygments \ + gcc-plugin-devel \ + systemd git \ + elfutils-libelf-devel openssl openssl-devel \ + elfutils-devel-static \ + glibc-static zlib-static \ + libstdc++-static \ + platform-python-devel \ + rpm-build rsync bc perl -y && \ + yum install gcc-python-plugin --enablerepo=Plus -y && \ yum clean all RUN git clone https://gitee.com/src-anolis-sig/gcc-python-plugin.git && \ diff --git a/Dockerfile.x86_64 b/Dockerfile.x86_64 index 26e42c15cf252e38760bd34160656df269bf27a3..ef51c02510b0abb74a440e86a9381edc961b3640 100644 --- a/Dockerfile.x86_64 +++ b/Dockerfile.x86_64 @@ -2,20 +2,21 @@ # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause From openanolis/anolisos:8.4-x86_64 -RUN yum install python2 python2-devel gcc gcc-c++ wget libyaml-devel -y && \ - wget https://bootstrap.pypa.io/pip/2.7/get-pip.py && \ - python2 get-pip.py -RUN pip install --upgrade setuptools && \ - pip install --global-option='--with-libyaml' pyyaml && \ - pip install six sh coloredlogs future fire jinja2 docopt && \ - yum install make bison flex \ - gcc-plugin-devel \ - elfutils-libelf-devel openssl openssl-devel \ - elfutils-devel-static \ - glibc-static zlib-static \ - libstdc++-static \ - platform-python-devel \ - rpm-build rsync bc perl -y && \ +RUN yum install python3 python3-devel python3-lxml gcc gcc-c++ wget libyaml-devel -y && \ + wget https://bootstrap.pypa.io/pip/3.6/get-pip.py && \ + python3 get-pip.py +RUN pip3 install --upgrade setuptools && \ + pip3 install --global-option='--with-libyaml' pyyaml && \ + pip3 install sh coloredlogs fire jinja2 docopt && \ + yum install make bison flex python3-lxml python3-six python3-pygments \ + gcc-plugin-devel.x86_64 \ + systemd git \ + elfutils-libelf-devel.x86_64 openssl openssl-devel \ + elfutils-devel-static \ + glibc-static zlib-static \ + libstdc++-static \ + platform-python-devel \ + rpm-build rsync bc perl -y && \ yum install gcc-python-plugin --enablerepo=Plus -y && \ yum clean all diff --git a/cli.py b/cli.py index 19c2592e86b2bcd371ea069e0ff61856a653a0b9..029c3a3b81b5ed26fbe8a9cf9de808fc2b38d254 100755 --- a/cli.py +++ b/cli.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause @@ -120,7 +120,7 @@ class Plugsched(object): break return i - candidates = map(os.path.basename, glob('%s/configs/%s*' % (self.plugsched_path, self.major))) + candidates = list(map(os.path.basename, glob('%s/configs/%s*' % (self.plugsched_path, self.major)))) if len(candidates) == 0: logging.fatal('''Can't find config directory, please add config for kernel %s''', self.KVER) @@ -208,9 +208,22 @@ class Plugsched(object): logging.info("Succeed!") + # when python3 working with rpmbuild, the /usr/local/python* path + # won't be in included in sys/path which results in some modules + # can't be find. So we need to add the PYTHONPATH manually. + # The detail about this can be find in + # https://fedoraproject.org/wiki/Changes/Making_sudo_pip_safe + def add_python_path(self): + py_ver = sys.version[0:3] + python_path = '/usr/local/lib64/python' + py_ver + '/site-packages' + python_path += os.pathsep + python_path += '/usr/local/lib/python' + py_ver + '/site-packages' + os.environ["PYTHONPATH"] = python_path + def cmd_build(self): if not os.path.exists(self.work_dir): logging.fatal("plugsched: Can't find %s", self.work_dir) + self.add_python_path() logging.info("Preparing rpmbuild environment") rpmbuild_root = os.path.join(self.plugsched_path, 'rpmbuild') self.plugsched_sh.rm('rpmbuild', recursive=True, force=True) @@ -240,7 +253,7 @@ if __name__ == '__main__': rpmbuild_root = mkdtemp() sh.rpmbuild('--define', '%%_topdir %s' % rpmbuild_root, - '--define', '%%__python %s' % '/usr/bin/python2', + '--define', '%%__python %s' % '/usr/bin/python3', '-rp', '--nodeps', kernel_src_rpm) src = glob('kernel*/linux*', rpmbuild_root + '/BUILD/') diff --git a/sched_boundary/process.py b/sched_boundary/process.py index b6ec506cb9be8658f671550a2232400f04d8b2ee..2c9f88594cef7e5ad26f3c01e576fd781029cb39 100644 --- a/sched_boundary/process.py +++ b/sched_boundary/process.py @@ -26,7 +26,7 @@ modpath = None Loader.add_constructor(resolver.BaseResolver.DEFAULT_SEQUENCE_TAG, lambda loader, node: set(loader.construct_sequence(node))) Dumper.add_representer(set, lambda dumper, node: dumper.represent_list(node)) -Dumper.add_representer(unicode, +Dumper.add_representer(str, lambda dumper, data: dumper.represent_scalar(u'tag:yaml.org,2002:str', data)) def read_config(): @@ -122,15 +122,16 @@ def inflect(initial_insiders, edges): global __insiders __insiders = copy.deepcopy(initial_insiders) while True: - delete_insider = filter(None, map(inflect_one, edges)) + delete_insider = list(filter(None, list(map(inflect_one, edges)))) if not delete_insider: break __insiders -= set(delete_insider) return __insiders global_fn_dict = {} -def lookup_if_global((name, file)): +def lookup_if_global(name_and_file): # Returns None if function is a gcc built-in function + name, file = name_and_file file = global_fn_dict.get(name, None) if file == '?' else file return (name, file) if file else None @@ -142,7 +143,7 @@ if __name__ == '__main__': config = read_config() config['mod_files_basename'] = {os.path.basename(f): f for f in config['mod_files']} config['mod_header_files'] = [f for f in config['mod_files'] if f.endswith('.h')] - metas = map(read_meta, all_meta_files()) + metas = list(map(read_meta, all_meta_files())) func_class = { 'fn': set(), @@ -207,7 +208,7 @@ if __name__ == '__main__': if struct not in m['struct']: continue all_set |= set(m['struct'][struct]['all_fields']) - for field, users in m['struct'][struct]['public_fields'].iteritems(): + for field, users in m['struct'][struct]['public_fields'].items(): p_user = set(map(tuple, users)) & func_class['public_user'] if p_user: user_set |= p_user diff --git a/sched_boundary/sched_boundary.py b/sched_boundary/sched_boundary.py index fceae3b8645010d0dbc9a0e0789067d158b509fc..6ed20d1346e02453ca9fa8a3322601c7b45a46b0 100644 --- a/sched_boundary/sched_boundary.py +++ b/sched_boundary/sched_boundary.py @@ -1,9 +1,9 @@ +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause from collections import defaultdict -from builtins import super -from itertools import izip, groupby as _groupby +from itertools import groupby as _groupby from yaml import load, dump, resolver, CLoader as Loader, CDumper as Dumper import json import re @@ -418,14 +418,14 @@ class SchedBoundaryCollect(SchedBoundary): def groupby(it, grouper, selector): sorted_list = sorted(it, key=grouper) - return dict((k, map(selector, v)) for k, v in _groupby(sorted_list, grouper)) + return dict((k, list(map(selector, v))) for k, v in _groupby(sorted_list, grouper)) - for struct, user_fields in public_fields.iteritems(): + for struct, user_fields in public_fields.items(): self.struct_properties[struct.name.name] = { "all_fields": [f.name for f in struct.fields if f.name], "public_fields": groupby(user_fields, - grouper=lambda (user, field): field.name, - selector=lambda (user, field): (user.name, os.path.relpath(user.location.file))) + grouper=lambda user_and_field: user_and_field[1].name, + selector=lambda user_and_field: (user_and_field[0].name, os.path.relpath(user_and_field[0].location.file))) } def collect_edges(self): diff --git a/src/Makefile.plugsched b/src/Makefile.plugsched index 48254383981c132e15f05cbc3a703b77ee49a1e3..22b3dbd41ed2fd592bcc07c9fbe6d5bed2741258 100644 --- a/src/Makefile.plugsched +++ b/src/Makefile.plugsched @@ -18,9 +18,9 @@ plugsched: scripts prepare sidecar $(MAKE) CFLAGS_MODULE=-fkeep-static-functions -C $(srctree) M=$(plugsched_modpath) modules sidecar: $(plugsched_modpath)/export_jump_sidecar.h - python2 $(plugsched_tmpdir)/sidecar.py $< ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) + python3 $(plugsched_tmpdir)/sidecar.py $< ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) collect: $(core-y) $(core-m) $(drivers-y) $(drivers-m) $(net-y) $(net-m) $(virt-y) analyze: - python2 $(plugsched_tmpdir)/process.py ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) + python3 $(plugsched_tmpdir)/process.py ./vmlinux $(plugsched_tmpdir) $(plugsched_modpath) extract: $(objs) diff --git a/src/sidecar.py b/src/sidecar.py index ba1d34107d85ca7511d7ae5d6ed261e4c8374726..703f1f12936d39e1204d704e17b8d89eed505245 100644 --- a/src/sidecar.py +++ b/src/sidecar.py @@ -28,8 +28,8 @@ if __name__ == '__main__': modpath = sys.argv[4] with open(tmpdir + 'symbol_resolve/undefined_functions_sidecar.h', 'w') as f: - for fn, pos in sympos.iteritems(): + for fn, pos in sympos.items(): f.write('{"%s", %d},\n' % (fn, pos)) with open(modpath + 'tainted_functions_sidecar.h', 'w') as f: - for fn, pos in sympos.iteritems(): + for fn, pos in sympos.items(): f.write('TAINTED_FUNCTION(%s,%d)\n' % (fn, pos if pos else 1)) diff --git a/tools/yaml-diff.py b/tools/yaml-diff.py index a3c2df9e8fad3ee56eb53d5415e844b633071e9b..cdfe280fd955f402d23d98028282dca15860dbfe 100755 --- a/tools/yaml-diff.py +++ b/tools/yaml-diff.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 # Copyright 2019-2022 Alibaba Group Holding Limited. # SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause