From 9cf54b192fe016e1a5ea945ee8e12dcfcf64beb0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:04:06 +0200 Subject: [PATCH 01/43] lint `Tools/build/check_extension_modules.py` --- Tools/build/.ruff.toml | 5 +++++ Tools/build/check_extension_modules.py | 30 +++++++++++++++----------- 2 files changed, 23 insertions(+), 12 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 53bfe18f8de1b8..0c482066a5ce23 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -18,3 +18,8 @@ select = [ "W", # pycodestyle "YTT", # flake8-2020 ] + +[lint.per-file-ignores] +"check_extension_modules.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 66b2a262e11c57..20d9b5a012e42a 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,8 +17,9 @@ See --help for more information """ + +import _imp import argparse -import collections import enum import logging import os @@ -27,12 +28,15 @@ import sys import sysconfig import warnings -import _imp - +from collections.abc import Iterable from importlib._bootstrap import _load as bootstrap_load -from importlib.machinery import BuiltinImporter, ExtensionFileLoader, ModuleSpec +from importlib.machinery import ( + BuiltinImporter, + ExtensionFileLoader, + ModuleSpec, +) from importlib.util import spec_from_file_location, spec_from_loader -from typing import Iterable +from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -124,7 +128,9 @@ def __bool__(self): return self.value in {"builtin", "shared"} -ModuleInfo = collections.namedtuple("ModuleInfo", "name state") +class ModuleInfo(NamedTuple): + name: str + state: ModuleState class ModuleChecker: @@ -195,8 +201,10 @@ def print_three_column(modinfos: list[ModuleInfo]): # guarantee zip() doesn't drop anything while len(names) % 3: names.append("") - for l, m, r in zip(names[::3], names[1::3], names[2::3]): - print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) + for c1, c2, c3 in zip(names[::3], names[1::3], names[2::3]): + format_args = (longest, c1, longest, c2, longest, c3) + # the '%-*s' format specification is not support by str.format() + print("%-*s %-*s %-*s" % format_args) # noqa: UP031 if verbose and self.builtin_ok: print("The following *built-in* modules have been successfully built:") @@ -219,9 +227,7 @@ def print_three_column(modinfos: list[ModuleInfo]): print() if verbose and self.notavailable: - print( - f"The following modules are not available on platform '{self.platform}':" - ) + print(f"The following modules are not available on platform {self.platform!r}:") print_three_column(self.notavailable) print() @@ -420,7 +426,7 @@ def check_module_import(self, modinfo: ModuleInfo): except ImportError as e: logger.error("%s failed to import: %s", modinfo.name, e) raise - except Exception as e: + except Exception: if not hasattr(_imp, 'create_dynamic'): logger.warning("Dynamic extension '%s' ignored", modinfo.name) return From 7ee999a73060ad9666b4c7a2d6b0872801d8a0ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:05:18 +0200 Subject: [PATCH 02/43] lint `Tools/build/compute-changes.py` --- Tools/build/compute-changes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index b3be7df2dba6d0..c4d59dce10429e 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -16,7 +16,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Set + from collections.abc import Set as AbstractSet GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"] GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") @@ -102,7 +102,7 @@ def git_branches() -> tuple[str, str]: def get_changed_files( ref_a: str = GITHUB_DEFAULT_BRANCH, ref_b: str = "HEAD" -) -> Set[Path]: +) -> AbstractSet[Path]: """List the files changed between two Git refs, filtered by change type.""" args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--") print(*args) @@ -113,7 +113,7 @@ def get_changed_files( return frozenset(map(Path, filter(None, map(str.strip, changed_files)))) -def process_changed_files(changed_files: Set[Path]) -> Outputs: +def process_changed_files(changed_files: AbstractSet[Path]) -> Outputs: run_tests = False run_ci_fuzz = False run_docs = False From c3dd400cd78c9e665700ac26eff7216258ed99a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:15:14 +0200 Subject: [PATCH 03/43] lint `Tools/build/deepfreeze.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/deepfreeze.py | 33 +++++++++++++++++---------------- 2 files changed, 20 insertions(+), 16 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 0c482066a5ce23..ae0804417ac733 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -23,3 +23,6 @@ select = [ "check_extension_modules.py" = [ "E501", # Line too long ] +"deepfreeze.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 6ec7a97f70ebb3..d9612478824d65 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -5,6 +5,7 @@ On Windows, and in cross-compilation cases, it is executed by Python 3.10, and 3.11 features are not available. """ + import argparse import builtins import collections @@ -13,7 +14,7 @@ import re import time import types -from typing import Dict, FrozenSet, TextIO, Tuple +from typing import TextIO import umarshal @@ -57,8 +58,8 @@ def get_localsplus(code: types.CodeType): def get_localsplus_counts(code: types.CodeType, - names: Tuple[str, ...], - kinds: bytes) -> Tuple[int, int, int, int]: + names: tuple[str, ...], + kinds: bytes) -> tuple[int, int, int]: nlocals = 0 ncellvars = 0 nfreevars = 0 @@ -84,7 +85,7 @@ def get_localsplus_counts(code: types.CodeType, PyUnicode_4BYTE_KIND = 4 -def analyze_character_width(s: str) -> Tuple[int, bool]: +def analyze_character_width(s: str) -> tuple[int, bool]: maxchar = ' ' for c in s: maxchar = max(maxchar, c) @@ -109,7 +110,7 @@ class Printer: def __init__(self, file: TextIO) -> None: self.level = 0 self.file = file - self.cache: Dict[tuple[type, object, str], str] = {} + self.cache: dict[tuple[type, object, str], str] = {} self.hits, self.misses = 0, 0 self.finis: list[str] = [] self.inits: list[str] = [] @@ -235,7 +236,7 @@ def generate_unicode(self, name: str, s: str) -> str: utf8 = s.encode('utf-8') self.write(f'.utf8 = {make_string_literal(utf8)},') self.write(f'.utf8_length = {len(utf8)},') - with self.block(f"._data =", ","): + with self.block("._data =", ","): for i in range(0, len(s), 16): data = s[i:i+16] self.write(", ".join(map(str, map(ord, data))) + ",") @@ -292,7 +293,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_name = {co_name},") self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") - self.write(f"._co_cached = NULL,") + self.write("._co_cached = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") first_traceable = 0 for op in code.co_code[::2]: @@ -305,9 +306,9 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.inits.append(f"_PyStaticCode_Init({name_as_code})") return f"& {name}.ob_base.ob_base" - def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: + def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: if len(t) == 0: - return f"(PyObject *)& _Py_SINGLETON(tuple_empty)" + return "(PyObject *)& _Py_SINGLETON(tuple_empty)" items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -321,7 +322,7 @@ def generate_tuple(self, name: str, t: Tuple[object, ...]) -> str: with self.block("._object =", ","): self.object_var_head("PyTuple_Type", len(t)) if items: - with self.block(f".ob_item =", ","): + with self.block(".ob_item =", ","): for item in items: self.write(item + ",") return f"& {name}._object.ob_base.ob_base" @@ -379,7 +380,7 @@ def generate_complex(self, name: str, z: complex) -> str: self.write(f".cval = {{ {z.real}, {z.imag} }},") return f"&{name}.ob_base" - def generate_frozenset(self, name: str, fs: FrozenSet[object]) -> str: + def generate_frozenset(self, name: str, fs: frozenset[object]) -> str: try: fs = sorted(fs) except TypeError: @@ -402,7 +403,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, (types.CodeType, umarshal.Code)) : + if isinstance(obj, types.CodeType | umarshal.Code): val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) @@ -465,17 +466,17 @@ def generate(args: list[str], output: TextIO) -> None: printer = Printer(output) for arg in args: file, modname = arg.rsplit(':', 1) - with open(file, "r", encoding="utf8") as fd: + with open(file, encoding="utf8") as fd: source = fd.read() if is_frozen_header(source): code = decode_frozen_data(source) else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): + with printer.block("void\n_Py_Deepfreeze_Fini(void)"): for p in printer.finis: printer.write(p) - with printer.block(f"int\n_Py_Deepfreeze_Init(void)"): + with printer.block("int\n_Py_Deepfreeze_Init(void)"): for p in printer.inits: with printer.block(f"if ({p} < 0)"): printer.write("return -1;") @@ -513,7 +514,7 @@ def main() -> None: if args.file: if verbose: print(f"Reading targets from {args.file}") - with open(args.file, "rt", encoding="utf-8-sig") as fin: + with open(args.file, "rt", encoding="utf-8-sig") as fin: # noqa: UP015 rules = [x.strip() for x in fin] else: rules = args.args From 766a6dd31e4270d59dc929bd4edbf5ebc34e6ee4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:21:05 +0200 Subject: [PATCH 04/43] lint `Tools/build/freeze_modules.py` --- Tools/build/.ruff.toml | 4 ++++ Tools/build/freeze_modules.py | 13 +++++-------- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index ae0804417ac733..712aa9f1d0c1a5 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -26,3 +26,7 @@ select = [ "deepfreeze.py" = [ "E501", # Line too long ] +"freeze_modules.py" = [ + "E501", # Line too long + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` +] diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 8f74abdc83db47..e12815f54b3a0f 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -3,15 +3,14 @@ See the notes at the top of Python/frozen.c for more info. """ -from collections import namedtuple import hashlib import ntpath import os import posixpath +from collections import namedtuple from update_file import updating_file_with_tmpfile - ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) ROOT_DIR = os.path.abspath(ROOT_DIR) FROZEN_ONLY = os.path.join(ROOT_DIR, 'Tools', 'freeze', 'flag.py') @@ -482,7 +481,6 @@ def regen_frozen(modules): header = relpath_for_posix_display(src.frozenfile, parentdir) headerlines.append(f'#include "{header}"') - externlines = UniqueList() bootstraplines = [] stdliblines = [] testlines = [] @@ -509,11 +507,11 @@ def regen_frozen(modules): if mod.isalias: if not mod.orig: - entry = '{"%s", NULL},' % (mod.name,) + entry = '{"%s", NULL},' % (mod.name,) # noqa: UP031 elif mod.source.ispkg: - entry = '{"%s", "<%s"},' % (mod.name, mod.orig) + entry = '{"%s", "<%s"},' % (mod.name, mod.orig) # noqa: UP031 else: - entry = '{"%s", "%s"},' % (mod.name, mod.orig) + entry = '{"%s", "%s"},' % (mod.name, mod.orig) # noqa: UP031 aliaslines.append(indent + entry) for lines in (bootstraplines, stdliblines, testlines): @@ -625,7 +623,6 @@ def regen_makefile(modules): def regen_pcbuild(modules): projlines = [] filterlines = [] - corelines = [] for src in _iter_sources(modules): pyfile = relpath_for_windows_display(src.pyfile, ROOT_DIR) header = relpath_for_windows_display(src.frozenfile, ROOT_DIR) @@ -634,7 +631,7 @@ def regen_pcbuild(modules): projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(GeneratedFrozenModulesDir){header}') - projlines.append(f' ') + projlines.append(' ') filterlines.append(f' ') filterlines.append(' Python Files') From 2d4f8512060ee8b4700404ae08e7b921bb48a71a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:21:44 +0200 Subject: [PATCH 05/43] lint `Tools/build/generate-build-details.py` --- Tools/build/.ruff.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 712aa9f1d0c1a5..909466baf639bf 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -30,3 +30,6 @@ select = [ "E501", # Line too long "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` ] +"generate-build-details.py" = [ + "E501", # Line too long +] From d95fb61aa1c2394df718219bdb9281b8508bafb6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:23:11 +0200 Subject: [PATCH 06/43] lint `Tools/build/generate_global_objects.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/generate_global_objects.py | 21 +++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 909466baf639bf..c6b56b2581990e 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -33,3 +33,6 @@ select = [ "generate-build-details.py" = [ "E501", # Line too long ] +"generate_global_objects.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index 219fea771d53a6..e4c159366e4c6d 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -286,7 +286,8 @@ def generate_runtime_init(identifiers, strings): break else: raise NotImplementedError - assert nsmallposints and nsmallnegints + assert nsmallposints + assert nsmallnegints # Then target the runtime initializer. filename = os.path.join(INTERNAL, 'pycore_runtime_init_generated.h') @@ -365,19 +366,19 @@ def generate_static_strings_initializer(identifiers, strings): printer.write(START) printer.write("static inline void") with printer.block("_PyUnicode_InitStaticStrings(PyInterpreterState *interp)"): - printer.write(f'PyObject *string;') + printer.write('PyObject *string;') for i in sorted(identifiers): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') - printer.write(f'_PyUnicode_InternStatic(interp, &string);') - printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') + printer.write('_PyUnicode_InternStatic(interp, &string);') + printer.write('assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write('assert(PyUnicode_GET_LENGTH(string) != 1);') for value, name in sorted(strings.items()): printer.write(f'string = &_Py_STR({name});') - printer.write(f'_PyUnicode_InternStatic(interp, &string);') - printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') + printer.write('_PyUnicode_InternStatic(interp, &string);') + printer.write('assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write('assert(PyUnicode_GET_LENGTH(string) != 1);') printer.write(END) printer.write(after) @@ -433,8 +434,8 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' - + f'strings, use _Py_LATIN1_CHR instead: {string!r}') + f'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name elif name != strings[string]: From 6805f4add7a377e8243e2f4867102eb8bcec40ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:23:43 +0200 Subject: [PATCH 07/43] lint `Tools/build/generate_levenshtein_examples.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/generate_levenshtein_examples.py | 3 +-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index c6b56b2581990e..a9ea893b514122 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -36,3 +36,6 @@ select = [ "generate_global_objects.py" = [ "E501", # Line too long ] +"generate_levenshtein_examples.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/generate_levenshtein_examples.py b/Tools/build/generate_levenshtein_examples.py index 778eb458c541c0..30dcc7cf1a1479 100644 --- a/Tools/build/generate_levenshtein_examples.py +++ b/Tools/build/generate_levenshtein_examples.py @@ -1,12 +1,11 @@ """Generate 10,000 unique examples for the Levenshtein short-circuit tests.""" import argparse -from functools import lru_cache import json import os.path +from functools import lru_cache from random import choices, randrange - # This should be in sync with Lib/traceback.py. It's not importing those values # because this script is being executed by PYTHON_FOR_REGEN and not by the in-tree # build of Python. From ebd1007faeff1bfa117aef8f0e6fbe85c7c632cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:25:47 +0200 Subject: [PATCH 08/43] lint `Tools/build/generate_re_casefix.py` --- Tools/build/.ruff.toml | 4 ++++ Tools/build/generate_re_casefix.py | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index a9ea893b514122..f9b61d45984740 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -39,3 +39,7 @@ select = [ "generate_levenshtein_examples.py" = [ "E501", # Line too long ] +"generate_re_casefix.py" = [ + "E501", # Line too long + "UP031", # Use format specifiers instead of percent format +] diff --git a/Tools/build/generate_re_casefix.py b/Tools/build/generate_re_casefix.py index 6cebfbd025c58c..9345b8f4efe09d 100755 --- a/Tools/build/generate_re_casefix.py +++ b/Tools/build/generate_re_casefix.py @@ -9,7 +9,7 @@ def update_file(file, content): try: - with open(file, 'r', encoding='utf-8') as fobj: + with open(file, encoding='utf-8') as fobj: if fobj.read() == content: return False except (OSError, ValueError): @@ -50,7 +50,7 @@ def main(outfile='Lib/re/_casefix.py'): # List of codes of lowercased characters which have the same uppercase. equivalent_lower_codes = [sorted(t) for s in equivalent_chars - for t in [set(ord(c.lower()) for c in s)] + for t in [{ord(c.lower()) for c in s}] if len(t) > 1] bad_codes = [] From cadcb03c06a8e6e604b3c9694fab93aaf1981743 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:26:16 +0200 Subject: [PATCH 09/43] lint `Tools/build/generate_sbom.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/generate_sbom.py | 12 ++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index f9b61d45984740..060b74a5906f9e 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -43,3 +43,6 @@ select = [ "E501", # Line too long "UP031", # Use format specifiers instead of percent format ] +"generate_sbom.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 073c26ed13ce0a..5c2a3bc1364873 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -1,14 +1,14 @@ """Tool for generating Software Bill of Materials (SBOM) for Python's dependencies""" -import os -import re +import glob import hashlib import json -import glob -from pathlib import Path, PurePosixPath, PureWindowsPath +import os +import re import subprocess import sys -import urllib.request import typing +import urllib.request +from pathlib import Path, PurePosixPath, PureWindowsPath CPYTHON_ROOT_DIR = Path(__file__).parent.parent.parent @@ -250,7 +250,7 @@ def check_sbom_packages(sbom_data: dict[str, typing.Any]) -> None: license_concluded = package["licenseConcluded"] error_if( license_concluded != "NOASSERTION", - f"License identifier must be 'NOASSERTION'" + "License identifier must be 'NOASSERTION'" ) From 79659204320412c8f77ad9ee5020a71361293d29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:27:15 +0200 Subject: [PATCH 10/43] lint `Tools/build/generate_sre_constants.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/generate_sre_constants.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 060b74a5906f9e..250ae2806c3e93 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -46,3 +46,6 @@ select = [ "generate_sbom.py" = [ "E501", # Line too long ] +"generate_sre_constants.py" = [ + "UP031", # Use format specifiers instead of percent format +] diff --git a/Tools/build/generate_sre_constants.py b/Tools/build/generate_sre_constants.py index abea069c8bc0c5..f51339db333c4f 100755 --- a/Tools/build/generate_sre_constants.py +++ b/Tools/build/generate_sre_constants.py @@ -6,7 +6,7 @@ def update_file(file, content): try: - with open(file, 'r') as fobj: + with open(file) as fobj: if fobj.read() == content: return False except (OSError, ValueError): From 3bdf24a9f2ef58022d5e2d0bfa7178aa196f0431 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:27:36 +0200 Subject: [PATCH 11/43] lint `Tools/build/generate_stdlib_module_names.py` --- Tools/build/generate_stdlib_module_names.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/build/generate_stdlib_module_names.py b/Tools/build/generate_stdlib_module_names.py index f9fd29509f3225..9873890837fa8e 100644 --- a/Tools/build/generate_stdlib_module_names.py +++ b/Tools/build/generate_stdlib_module_names.py @@ -7,7 +7,6 @@ from check_extension_modules import ModuleChecker - SCRIPT_NAME = 'Tools/build/generate_stdlib_module_names.py' SRC_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__))) From 72e5eed7c12fc763df9d66c87260d008e63134e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 16:08:28 +0200 Subject: [PATCH 12/43] lint `Tools/build/generate_token.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/generate_token.py | 27 +++++++++++++-------------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 250ae2806c3e93..e52131154f1602 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -49,3 +49,6 @@ select = [ "generate_sre_constants.py" = [ "UP031", # Use format specifiers instead of percent format ] +"generate_token.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index a5f9828c466eda..bbbda6556d4afd 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -13,7 +13,6 @@ import re - SCRIPT_NAME = 'Tools/build/generate_token.py' AUTO_GENERATED_BY_SCRIPT = f'Auto-generated by {SCRIPT_NAME}' NT_OFFSET = 256 @@ -46,7 +45,7 @@ def load_tokens(path): def update_file(file, content): try: - with open(file, 'r') as fobj: + with open(file) as fobj: if fobj.read() == content: return False except (OSError, ValueError): @@ -108,14 +107,14 @@ def make_h(infile, outfile='Include/internal/pycore_token.h'): defines = [] for value, name in enumerate(tok_names[:ERRORTOKEN + 1]): - defines.append("#define %-15s %d\n" % (name, value)) + defines.append("#define %-15s %d\n" % (name, value)) # noqa: UP031 if update_file(outfile, token_h_template % ( ''.join(defines), len(tok_names), NT_OFFSET )): - print("%s regenerated from %s" % (outfile, infile)) + print(f"{outfile} regenerated from {infile}") token_c_template = f"""\ @@ -161,17 +160,17 @@ def generate_chars_to_token(mapping, n=1): write = result.append indent = ' ' * n write(indent) - write('switch (c%d) {\n' % (n,)) + write('switch (c%d) {\n' % (n,)) # noqa: UP031 for c in sorted(mapping): write(indent) value = mapping[c] if isinstance(value, dict): - write("case '%s':\n" % (c,)) + write("case '%s':\n" % (c,)) # noqa: UP031 write(generate_chars_to_token(value, n + 1)) write(indent) write(' break;\n') else: - write("case '%s': return %s;\n" % (c, value)) + write("case '%s': return %s;\n" % (c, value)) # noqa: UP031 write(indent) write('}\n') return ''.join(result) @@ -191,8 +190,8 @@ def make_c(infile, outfile='Parser/token.c'): names = [] for value, name in enumerate(tok_names): if value >= ERRORTOKEN: - name = '<%s>' % name - names.append(' "%s",\n' % name) + name = '<%s>' % name # noqa: UP031 + names.append(' "%s",\n' % name) # noqa: UP031 names.append(' "",\n') if update_file(outfile, token_c_template % ( @@ -201,7 +200,7 @@ def make_c(infile, outfile='Parser/token.c'): generate_chars_to_token(chars_to_token[2]), generate_chars_to_token(chars_to_token[3]) )): - print("%s regenerated from %s" % (outfile, infile)) + print(f"{outfile} regenerated from {infile}") token_inc_template = f"""\ @@ -253,7 +252,7 @@ def make_rst(infile, outfile='Doc/library/token-list.inc', exit('\n'.join(message_parts)) if update_file(outfile, token_inc_template % '\n'.join(names)): - print("%s regenerated from %s" % (outfile, infile)) + print(f"{outfile} regenerated from {infile}") token_py_template = f'''\ @@ -293,13 +292,13 @@ def make_py(infile, outfile='Lib/token.py'): constants = [] for value, name in enumerate(tok_names): - constants.append('%s = %d' % (name, value)) + constants.append('%s = %d' % (name, value)) # noqa: UP031 constants.insert(ERRORTOKEN, "# These aren't used by the C tokenizer but are needed for tokenize.py") token_types = [] for s, value in sorted(string_to_tok.items()): - token_types.append(' %r: %s,' % (s, tok_names[value])) + token_types.append(' %r: %s,' % (s, tok_names[value])) # noqa: UP031 if update_file(outfile, token_py_template % ( '\n'.join(constants), @@ -307,7 +306,7 @@ def make_py(infile, outfile='Lib/token.py'): NT_OFFSET, '\n'.join(token_types), )): - print("%s regenerated from %s" % (outfile, infile)) + print(f"{outfile} regenerated from {infile}") def main(op, infile='Grammar/Tokens', *args): From 23e1f437a8c226da15d991b43c7d8a9e9c34ed2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:28:50 +0200 Subject: [PATCH 13/43] lint `Tools/build/parse_html5_entities.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/parse_html5_entities.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index e52131154f1602..261a17b5e5f2f9 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -52,3 +52,6 @@ select = [ "generate_token.py" = [ "E501", # Line too long ] +"parse_html5_entities.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/parse_html5_entities.py b/Tools/build/parse_html5_entities.py index d2bf29091030a5..aca98497381a43 100755 --- a/Tools/build/parse_html5_entities.py +++ b/Tools/build/parse_html5_entities.py @@ -12,11 +12,11 @@ Written by Ezio Melotti and Iuliia Proskurnia. """ +import json import os import sys -import json -from urllib.request import urlopen from html.entities import html5 +from urllib.request import urlopen SCRIPT_NAME = 'Tools/build/parse_html5_entities.py' PAGE_URL = 'https://html.spec.whatwg.org/multipage/named-characters.html' @@ -40,20 +40,20 @@ def compare_dicts(old, new): """Compare the old and new dicts and print the differences.""" added = new.keys() - old.keys() if added: - print('{} entitie(s) have been added:'.format(len(added))) + print(f'{len(added)} entitie(s) have been added:') for name in sorted(added): - print(' {!r}: {!r}'.format(name, new[name])) + print(f' {name!r}: {new[name]!r}') removed = old.keys() - new.keys() if removed: - print('{} entitie(s) have been removed:'.format(len(removed))) + print(f'{len(removed)} entitie(s) have been removed:') for name in sorted(removed): - print(' {!r}: {!r}'.format(name, old[name])) + print(f' {name!r}: {old[name]!r}') changed = set() for name in (old.keys() & new.keys()): if old[name] != new[name]: changed.add((name, old[name], new[name])) if changed: - print('{} entitie(s) have been modified:'.format(len(changed))) + print(f'{len(changed)} entitie(s) have been modified:') for item in sorted(changed): print(' {!r}: {!r} -> {!r}'.format(*item)) @@ -111,5 +111,5 @@ def write_items(entities, file=sys.stdout): print('The current dictionary is updated.') else: compare_dicts(html5, new_html5) - print('Run "./python {0} --patch" to update Lib/html/entities.html ' - 'or "./python {0} --create" to see the generated ' 'dictionary.'.format(__file__)) + print(f'Run "./python {__file__} --patch" to update Lib/html/entities.html ' + f'or "./python {__file__} --create" to see the generated dictionary.') From 8b2d4dacfeba2dbd58e6488ad83257900632b115 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:36:33 +0200 Subject: [PATCH 14/43] lint `Tools/build/regen-configure.sh` --- Tools/build/regen-configure.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index c7683eb36763af..967e81b13f029e 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -6,18 +6,19 @@ set -e -x # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. IMAGE="ghcr.io/python/autoconf:2025.01.02.12581854023" +# shellcheck disable=SC2034 AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" -abs_srcdir=$(cd $(dirname $0)/../..; pwd) +abs_srcdir="$(cd "$(dirname "$0")/../.."; pwd)" if podman --version &>/dev/null; then RUNTIME="podman" elif docker --version &>/dev/null; then RUNTIME="docker" else - echo "$@ needs either Podman or Docker container runtime." >&2 + echo "$* needs either Podman or Docker container runtime." >&2 exit 1 fi From 87170021850b8d6a8b2de7bd584afde0b6d22d30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:38:20 +0200 Subject: [PATCH 15/43] lint `Tools/build/smelly.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/smelly.py | 11 +++++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 261a17b5e5f2f9..4f3867868eb1ca 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -55,3 +55,6 @@ select = [ "parse_html5_entities.py" = [ "E501", # Line too long ] +"smelly.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 7c534269c57a09..17e070c7b7fb8a 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -6,7 +6,6 @@ import sys import sysconfig - ALLOWED_PREFIXES = ('Py', '_Py') if sys.platform == 'darwin': ALLOWED_PREFIXES += ('__Py',) @@ -52,8 +51,8 @@ def get_exported_symbols(library, dynamic=False): if dynamic: args.append('--dynamic') args.append(library) - print("+ %s" % ' '.join(args)) - proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True) + print(f"+ {' '.join(args)}") + proc = subprocess.run(args, stdout=subprocess.PIPE, text=True) if proc.returncode: sys.stdout.write(proc.stdout) sys.exit(proc.returncode) @@ -80,7 +79,7 @@ def get_smelly_symbols(stdout, dynamic=False): symtype = parts[1].strip() symbol = parts[-1] - result = '%s (type: %s)' % (symbol, symtype) + result = f'{symbol} (type: {symtype})' if (symbol.startswith(ALLOWED_PREFIXES) or symbol in EXCEPTIONS or @@ -111,10 +110,10 @@ def check_library(library, dynamic=False): print() smelly_symbols.sort() for symbol in smelly_symbols: - print("Smelly symbol: %s" % symbol) + print(f"Smelly symbol: {symbol}") print() - print("ERROR: Found %s smelly symbols!" % len(smelly_symbols)) + print(f"ERROR: Found {len(smelly_symbols)} smelly symbols!") return len(smelly_symbols) From 42922db665a599249797896273dea7f92eb82aa7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:49:32 +0200 Subject: [PATCH 16/43] lint `Tools/build/stable_abi.py` --- Tools/build/.ruff.toml | 6 ++++ Tools/build/stable_abi.py | 70 ++++++++++++++++++--------------------- 2 files changed, 38 insertions(+), 38 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 4f3867868eb1ca..67389a79d7272a 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -19,6 +19,9 @@ select = [ "YTT", # flake8-2020 ] +[per-file-target-version] +"stable_abi.py" = "py311" # requires 'tomllib' + [lint.per-file-ignores] "check_extension_modules.py" = [ "E501", # Line too long @@ -58,3 +61,6 @@ select = [ "smelly.py" = [ "E501", # Line too long ] +"stable_abi.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 2874c436c1ebc9..07b76ed8e676dd 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -7,22 +7,22 @@ (relative to the manifest file, as they appear in the CPython codebase). """ -from functools import partial -from pathlib import Path -import dataclasses -import subprocess -import sysconfig import argparse -import textwrap -import tomllib +import csv +import dataclasses import difflib -import pprint -import sys +import io import os import os.path -import io +import pprint import re -import csv +import subprocess +import sys +import sysconfig +import textwrap +import tomllib +from functools import partial +from pathlib import Path SCRIPT_NAME = 'Tools/build/stable_abi.py' DEFAULT_MANIFEST_PATH = ( @@ -57,7 +57,7 @@ class Manifest: """Collection of `ABIItem`s forming the stable ABI/limited API.""" def __init__(self): - self.contents = dict() + self.contents = {} def add(self, item): if item.name in self.contents: @@ -337,7 +337,7 @@ def test_windows_feature_macros(self): write(")") for ifdef, names in optional_items.items(): write(f"if feature_macros[{ifdef!r}]:") - write(f" SYMBOL_NAMES += (") + write(" SYMBOL_NAMES += (") for name in names: write(f" {name!r},") write(" )") @@ -404,22 +404,20 @@ def do_unixy_check(manifest, args): # Get all macros first: we'll need feature macros like HAVE_FORK and # MS_WINDOWS for everything else present_macros = gcc_get_limited_api_macros(['Include/Python.h']) - feature_macros = set(m.name for m in manifest.select({'feature_macro'})) + feature_macros = {m.name for m in manifest.select({'feature_macro'})} feature_macros &= present_macros # Check that we have all needed macros - expected_macros = set( - item.name for item in manifest.select({'macro'}) - ) + expected_macros = {item.name for item in manifest.select({'macro'})} missing_macros = expected_macros - present_macros okay &= _report_unexpected_items( missing_macros, - 'Some macros from are not defined from "Include/Python.h"' - + 'with Py_LIMITED_API:') + 'Some macros from are not defined from "Include/Python.h" ' + 'with Py_LIMITED_API:') - expected_symbols = set(item.name for item in manifest.select( + expected_symbols = {item.name for item in manifest.select( {'function', 'data'}, include_abi_only=True, ifdef=feature_macros, - )) + )} # Check the static library (*.a) LIBRARY = sysconfig.get_config_var("LIBRARY") @@ -437,15 +435,15 @@ def do_unixy_check(manifest, args): manifest, LDLIBRARY, expected_symbols, dynamic=False) # Check definitions in the header files - expected_defs = set(item.name for item in manifest.select( + expected_defs = {item.name for item in manifest.select( {'function', 'data'}, include_abi_only=False, ifdef=feature_macros, - )) + )} found_defs = gcc_get_limited_api_definitions(['Include/Python.h']) missing_defs = expected_defs - found_defs okay &= _report_unexpected_items( missing_defs, 'Some expected declarations were not declared in ' - + '"Include/Python.h" with Py_LIMITED_API:') + '"Include/Python.h" with Py_LIMITED_API:') # Some Limited API macros are defined in terms of private symbols. # These are not part of Limited API (even though they're defined with @@ -455,7 +453,7 @@ def do_unixy_check(manifest, args): okay &= _report_unexpected_items( extra_defs, 'Some extra declarations were found in "Include/Python.h" ' - + 'with Py_LIMITED_API:') + 'with Py_LIMITED_API:') return okay @@ -477,7 +475,7 @@ def binutils_get_exported_symbols(library, dynamic=False): if dynamic: args.append("--dynamic") args.append(library) - proc = subprocess.run(args, stdout=subprocess.PIPE, universal_newlines=True) + proc = subprocess.run(args, stdout=subprocess.PIPE, text=True) if proc.returncode: sys.stdout.write(proc.stdout) sys.exit(proc.returncode) @@ -550,12 +548,8 @@ def gcc_get_limited_api_macros(headers): text=True, ) - return { - target - for target in re.findall( - r"#define (\w+)", preprocessor_output_with_macros - ) - } + matches = re.findall(r"#define (\w+)", preprocessor_output_with_macros) + return set(matches) def gcc_get_limited_api_definitions(headers): @@ -613,7 +607,7 @@ def check_private_names(manifest): if name.startswith('_') and not item.abi_only: raise ValueError( f'`{name}` is private (underscore-prefixed) and should be ' - + 'removed from the stable ABI list or marked `abi_only`') + f'removed from the stable ABI list or marked `abi_only`') def check_dump(manifest, filename): """Check that manifest.dump() corresponds to the data. @@ -624,7 +618,7 @@ def check_dump(manifest, filename): with filename.open('rb') as file: from_file = tomllib.load(file) if dumped != from_file: - print(f'Dump differs from loaded data!', file=sys.stderr) + print('Dump differs from loaded data!', file=sys.stderr) diff = difflib.unified_diff( pprint.pformat(dumped).splitlines(), pprint.pformat(from_file).splitlines(), @@ -653,8 +647,8 @@ def main(): ) parser.add_argument( "--generate-all", action='store_true', - help="as --generate, but generate all file(s) using default filenames." - + " (unlike --all, does not run any extra checks)", + help="as --generate, but generate all file(s) using default filenames. " + "(unlike --all, does not run any extra checks)", ) parser.add_argument( "-a", "--all", action='store_true', @@ -739,9 +733,9 @@ def main(): if not results: if args.generate: parser.error('No file specified. Use --generate-all to regenerate ' - + 'all files, or --help for usage.') + 'all files, or --help for usage.') parser.error('No check specified. Use --all to check all files, ' - + 'or --help for usage.') + 'or --help for usage.') failed_results = [name for name, result in results.items() if not result] From 6cf0357bf511f629e0f349ad9c13c32a7fda2054 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:51:08 +0200 Subject: [PATCH 17/43] lint `Tools/build/umarshal.py` --- Tools/build/.ruff.toml | 3 +++ Tools/build/umarshal.py | 18 +++++++++--------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 67389a79d7272a..cba64c2217d4a6 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -64,3 +64,6 @@ select = [ "stable_abi.py" = [ "E501", # Line too long ] +"umarshal.py" = [ + "E501", # Line too long +] diff --git a/Tools/build/umarshal.py b/Tools/build/umarshal.py index 8198a1780b8dad..679fa7caf9f931 100644 --- a/Tools/build/umarshal.py +++ b/Tools/build/umarshal.py @@ -1,8 +1,7 @@ # Implementation of marshal.loads() in pure Python import ast - -from typing import Any, Tuple +from typing import Any class Type: @@ -55,10 +54,10 @@ def __init__(self, **kwds: Any): def __repr__(self) -> str: return f"Code(**{self.__dict__})" - co_localsplusnames: Tuple[str] - co_localspluskinds: Tuple[int] + co_localsplusnames: tuple[str, ...] + co_localspluskinds: tuple[int, ...] - def get_localsplus_names(self, select_kind: int) -> Tuple[str, ...]: + def get_localsplus_names(self, select_kind: int) -> tuple[str, ...]: varnames: list[str] = [] for name, kind in zip(self.co_localsplusnames, self.co_localspluskinds): @@ -67,15 +66,15 @@ def get_localsplus_names(self, select_kind: int) -> Tuple[str, ...]: return tuple(varnames) @property - def co_varnames(self) -> Tuple[str, ...]: + def co_varnames(self) -> tuple[str, ...]: return self.get_localsplus_names(CO_FAST_LOCAL) @property - def co_cellvars(self) -> Tuple[str, ...]: + def co_cellvars(self) -> tuple[str, ...]: return self.get_localsplus_names(CO_FAST_CELL) @property - def co_freevars(self) -> Tuple[str, ...]: + def co_freevars(self) -> tuple[str, ...]: return self.get_localsplus_names(CO_FAST_FREE) @property @@ -309,7 +308,8 @@ def loads(data: bytes) -> Any: def main(): # Test - import marshal, pprint + import marshal + import pprint sample = {'foo': {(42, "bar", 3.14)}} data = marshal.dumps(sample) retval = loads(data) From 921cc0865c874d53e0aec643c7bd2e0838f8d936 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Thu, 10 Apr 2025 15:51:41 +0200 Subject: [PATCH 18/43] lint `Tools/build/verify_ensurepip_wheels.py` --- Tools/build/.ruff.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index cba64c2217d4a6..9d7bf38a8266f0 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -67,3 +67,6 @@ select = [ "umarshal.py" = [ "E501", # Line too long ] +"verify_ensurepip_wheels.py" = [ + "E501", # Line too long +] From 7401104c94a61849d2751797bdbd655303e3f090 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:24:58 +0200 Subject: [PATCH 19/43] force explicit Python version for known files --- Tools/build/.ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 9d7bf38a8266f0..fc7e52af38638a 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -20,6 +20,7 @@ select = [ ] [per-file-target-version] +"deepfreeze.py" = "py310" "stable_abi.py" = "py311" # requires 'tomllib' [lint.per-file-ignores] From effe67983e4e87bbab642badca313b74881050f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:26:19 +0200 Subject: [PATCH 20/43] minimize implementation diff --- Tools/build/.ruff.toml | 1 + Tools/build/check_extension_modules.py | 6 ++---- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index fc7e52af38638a..21355fe78d64a4 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -26,6 +26,7 @@ select = [ [lint.per-file-ignores] "check_extension_modules.py" = [ "E501", # Line too long + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` ] "deepfreeze.py" = [ "E501", # Line too long diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 20d9b5a012e42a..46729fcf58b78b 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -20,6 +20,7 @@ import _imp import argparse +import collections import enum import logging import os @@ -36,7 +37,6 @@ ModuleSpec, ) from importlib.util import spec_from_file_location, spec_from_loader -from typing import NamedTuple SRC_DIR = pathlib.Path(__file__).parent.parent.parent @@ -128,9 +128,7 @@ def __bool__(self): return self.value in {"builtin", "shared"} -class ModuleInfo(NamedTuple): - name: str - state: ModuleState +ModuleInfo = collections.namedtuple("ModuleInfo", "name state") class ModuleChecker: From 043805d85e1922f97c660a86fa0347540580b042 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:27:56 +0200 Subject: [PATCH 21/43] minimize implementation diff --- Tools/build/.ruff.toml | 3 ++- Tools/build/check_extension_modules.py | 7 ++++--- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 21355fe78d64a4..cd337a570bb37f 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -25,8 +25,9 @@ select = [ [lint.per-file-ignores] "check_extension_modules.py" = [ - "E501", # Line too long + "E501", # Line too long "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "UP031", # Use format specifiers instead of percent format ] "deepfreeze.py" = [ "E501", # Line too long diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 46729fcf58b78b..cd5cbd02d60905 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -200,9 +200,8 @@ def print_three_column(modinfos: list[ModuleInfo]): while len(names) % 3: names.append("") for c1, c2, c3 in zip(names[::3], names[1::3], names[2::3]): - format_args = (longest, c1, longest, c2, longest, c3) # the '%-*s' format specification is not support by str.format() - print("%-*s %-*s %-*s" % format_args) # noqa: UP031 + print("%-*s %-*s %-*s" % (longest, c1, longest, c2, longest, c3)) if verbose and self.builtin_ok: print("The following *built-in* modules have been successfully built:") @@ -225,7 +224,9 @@ def print_three_column(modinfos: list[ModuleInfo]): print() if verbose and self.notavailable: - print(f"The following modules are not available on platform {self.platform!r}:") + print( + f"The following modules are not available on platform '{self.platform}':" + ) print_three_column(self.notavailable) print() From 859bc912d1f14f55ba2094f04ca4d6f7cc0f106b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:29:54 +0200 Subject: [PATCH 22/43] minimize implementation diff --- Tools/build/check_extension_modules.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index cd5cbd02d60905..abe968b96b1e4f 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -199,9 +199,9 @@ def print_three_column(modinfos: list[ModuleInfo]): # guarantee zip() doesn't drop anything while len(names) % 3: names.append("") - for c1, c2, c3 in zip(names[::3], names[1::3], names[2::3]): + for l, m, r in zip(names[::3], names[1::3], names[2::3]): # noqa: E741 # the '%-*s' format specification is not support by str.format() - print("%-*s %-*s %-*s" % (longest, c1, longest, c2, longest, c3)) + print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) if verbose and self.builtin_ok: print("The following *built-in* modules have been successfully built:") From d0b1df478348d7c687ddbdac27bb840c7add6cd3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:32:10 +0200 Subject: [PATCH 23/43] minimize implementation diff --- Tools/build/.ruff.toml | 1 + Tools/build/freeze_modules.py | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index cd337a570bb37f..1726f09d145302 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -35,6 +35,7 @@ select = [ "freeze_modules.py" = [ "E501", # Line too long "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "UP031", # Use format specifiers instead of percent format ] "generate-build-details.py" = [ "E501", # Line too long diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index e12815f54b3a0f..2103bcf057592f 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -507,11 +507,11 @@ def regen_frozen(modules): if mod.isalias: if not mod.orig: - entry = '{"%s", NULL},' % (mod.name,) # noqa: UP031 + entry = '{"%s", NULL},' % (mod.name,) elif mod.source.ispkg: - entry = '{"%s", "<%s"},' % (mod.name, mod.orig) # noqa: UP031 + entry = '{"%s", "<%s"},' % (mod.name, mod.orig) else: - entry = '{"%s", "%s"},' % (mod.name, mod.orig) # noqa: UP031 + entry = '{"%s", "%s"},' % (mod.name, mod.orig) aliaslines.append(indent + entry) for lines in (bootstraplines, stdliblines, testlines): From af4a0ea6b94dbb1bce96ae8909bbd118186e5d4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:47:31 +0200 Subject: [PATCH 24/43] fixup lint? --- Tools/build/.ruff.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 1726f09d145302..b6c6853237cbcf 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -20,7 +20,6 @@ select = [ ] [per-file-target-version] -"deepfreeze.py" = "py310" "stable_abi.py" = "py311" # requires 'tomllib' [lint.per-file-ignores] From a7d4b02c51146ef0b80e1980c23b23c95acabf03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 14:49:27 +0200 Subject: [PATCH 25/43] Revert "fixup lint?" This reverts commit af4a0ea6b94dbb1bce96ae8909bbd118186e5d4c. --- Tools/build/.ruff.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index b6c6853237cbcf..1726f09d145302 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -20,6 +20,7 @@ select = [ ] [per-file-target-version] +"deepfreeze.py" = "py310" "stable_abi.py" = "py311" # requires 'tomllib' [lint.per-file-ignores] From 6cfee131a7a84417a0575487e9d2fa827ea2c702 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:01:03 +0200 Subject: [PATCH 26/43] bump ruff version --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb44c27704d455..ec53cf58bf78a6 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.9.1 + rev: v0.11.4 hooks: - id: ruff name: Run Ruff (lint) on Doc/ From b14db8b3c7f7ee3f187bb98ca78973a17a5eac4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:55:52 +0200 Subject: [PATCH 27/43] deepfreeze.py: address Hugo's review --- Tools/build/deepfreeze.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index d9612478824d65..9919641242f8ae 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -514,7 +514,7 @@ def main() -> None: if args.file: if verbose: print(f"Reading targets from {args.file}") - with open(args.file, "rt", encoding="utf-8-sig") as fin: # noqa: UP015 + with open(args.file, encoding="utf-8-sig") as fin: rules = [x.strip() for x in fin] else: rules = args.args From 3aaf9ccfe3bcf71c8943a0d8a04ef6157b5ca0fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 15:58:36 +0200 Subject: [PATCH 28/43] remove un-necessary comment --- Tools/build/check_extension_modules.py | 1 - 1 file changed, 1 deletion(-) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index abe968b96b1e4f..84dcd6ba57f12c 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -200,7 +200,6 @@ def print_three_column(modinfos: list[ModuleInfo]): while len(names) % 3: names.append("") for l, m, r in zip(names[::3], names[1::3], names[2::3]): # noqa: E741 - # the '%-*s' format specification is not support by str.format() print("%-*s %-*s %-*s" % (longest, l, longest, m, longest, r)) if verbose and self.builtin_ok: From dd8bfbb4d245e9623108c4ae1020cdd88211ae67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:15:14 +0200 Subject: [PATCH 29/43] update pre-commit config --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ec53cf58bf78a6..9a343bb9309bc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,9 +11,9 @@ repos: args: [--exit-non-zero-on-fix] files: ^Lib/test/ - id: ruff - name: Run Ruff (lint) on Tools/build/check_warnings.py + name: Run Ruff (lint) on Tools/build args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] - files: ^Tools/build/check_warnings.py + files: ^Tools/build/*.py - id: ruff name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] From 175b1eec1d31af3c749bec38af42237fe65e74e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Fri, 11 Apr 2025 16:18:50 +0200 Subject: [PATCH 30/43] fixup pattern --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9a343bb9309bc3..4823bc2b86cc84 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,7 @@ repos: - id: ruff name: Run Ruff (lint) on Tools/build args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] - files: ^Tools/build/*.py + files: ^Tools/build/ - id: ruff name: Run Ruff (lint) on Argument Clinic args: [--exit-non-zero-on-fix, --config=Tools/clinic/.ruff.toml] From ba937f94d4de9ae4dd62beac641b052c46526f04 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:03:10 +0100 Subject: [PATCH 31/43] Consolidate E501 ignores --- Tools/build/.ruff.toml | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 1726f09d145302..2673f7bde56d0a 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -25,52 +25,16 @@ select = [ [lint.per-file-ignores] "check_extension_modules.py" = [ - "E501", # Line too long "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` "UP031", # Use format specifiers instead of percent format ] -"deepfreeze.py" = [ - "E501", # Line too long -] "freeze_modules.py" = [ - "E501", # Line too long "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` "UP031", # Use format specifiers instead of percent format ] -"generate-build-details.py" = [ - "E501", # Line too long -] -"generate_global_objects.py" = [ - "E501", # Line too long -] -"generate_levenshtein_examples.py" = [ - "E501", # Line too long -] "generate_re_casefix.py" = [ - "E501", # Line too long "UP031", # Use format specifiers instead of percent format ] -"generate_sbom.py" = [ - "E501", # Line too long -] "generate_sre_constants.py" = [ "UP031", # Use format specifiers instead of percent format ] -"generate_token.py" = [ - "E501", # Line too long -] -"parse_html5_entities.py" = [ - "E501", # Line too long -] -"smelly.py" = [ - "E501", # Line too long -] -"stable_abi.py" = [ - "E501", # Line too long -] -"umarshal.py" = [ - "E501", # Line too long -] -"verify_ensurepip_wheels.py" = [ - "E501", # Line too long -] From be3afe3e95eb2eb19c4036a8a081d86726bd627c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:04:50 +0100 Subject: [PATCH 32/43] Consolidate E501 ignores --- Tools/build/.ruff.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 2673f7bde56d0a..509b7fe7383982 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -18,6 +18,9 @@ select = [ "W", # pycodestyle "YTT", # flake8-2020 ] +ignore = [ + "E501", # Line too long +] [per-file-target-version] "deepfreeze.py" = "py310" From d2d7aa828a85ff19b57e588bfe200db65f151744 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:05:09 +0100 Subject: [PATCH 33/43] Consolidate UP031 ignores --- Tools/build/.ruff.toml | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 509b7fe7383982..92273e96e4dbd7 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -18,26 +18,19 @@ select = [ "W", # pycodestyle "YTT", # flake8-2020 ] -ignore = [ - "E501", # Line too long -] [per-file-target-version] "deepfreeze.py" = "py310" "stable_abi.py" = "py311" # requires 'tomllib' [lint.per-file-ignores] +"{check_extension_modules,freeze_modules,generate_re_casefix,generate_sre_constants}.py" = [ + "UP031", # Use format specifiers instead of percent format +] + "check_extension_modules.py" = [ "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` - "UP031", # Use format specifiers instead of percent format ] "freeze_modules.py" = [ "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` - "UP031", # Use format specifiers instead of percent format -] -"generate_re_casefix.py" = [ - "UP031", # Use format specifiers instead of percent format -] -"generate_sre_constants.py" = [ - "UP031", # Use format specifiers instead of percent format ] From e71dbd484af0f376df4daca96b2b70bf7c95dae7 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:05:21 +0100 Subject: [PATCH 34/43] Consolidate UP031 ignores --- Tools/build/.ruff.toml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 92273e96e4dbd7..aa12dd81095254 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -18,6 +18,9 @@ select = [ "W", # pycodestyle "YTT", # flake8-2020 ] +ignore = [ + "E501", # Line too long +] [per-file-target-version] "deepfreeze.py" = "py310" From c7dd4c421693707586150779cb276de2c88d5f36 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:05:27 +0100 Subject: [PATCH 35/43] Revert selected changes --- Tools/build/.ruff.toml | 8 -------- Tools/build/compute-changes.py | 6 +++--- Tools/build/generate_global_objects.py | 2 +- Tools/build/regen-configure.sh | 5 ++--- Tools/build/stable_abi.py | 9 ++++----- 5 files changed, 10 insertions(+), 20 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index aa12dd81095254..3f18c296db9689 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -12,7 +12,6 @@ select = [ "LOG", # flake8-logging "PGH", # pygrep-hooks "PT", # flake8-pytest-style - "PYI", # flake8-pyi "RUF100", # Ban unused `# noqa` comments "UP", # pyupgrade "W", # pycodestyle @@ -30,10 +29,3 @@ ignore = [ "{check_extension_modules,freeze_modules,generate_re_casefix,generate_sre_constants}.py" = [ "UP031", # Use format specifiers instead of percent format ] - -"check_extension_modules.py" = [ - "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` -] -"freeze_modules.py" = [ - "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` -] diff --git a/Tools/build/compute-changes.py b/Tools/build/compute-changes.py index c4d59dce10429e..b3be7df2dba6d0 100644 --- a/Tools/build/compute-changes.py +++ b/Tools/build/compute-changes.py @@ -16,7 +16,7 @@ TYPE_CHECKING = False if TYPE_CHECKING: - from collections.abc import Set as AbstractSet + from collections.abc import Set GITHUB_DEFAULT_BRANCH = os.environ["GITHUB_DEFAULT_BRANCH"] GITHUB_CODEOWNERS_PATH = Path(".github/CODEOWNERS") @@ -102,7 +102,7 @@ def git_branches() -> tuple[str, str]: def get_changed_files( ref_a: str = GITHUB_DEFAULT_BRANCH, ref_b: str = "HEAD" -) -> AbstractSet[Path]: +) -> Set[Path]: """List the files changed between two Git refs, filtered by change type.""" args = ("git", "diff", "--name-only", f"{ref_a}...{ref_b}", "--") print(*args) @@ -113,7 +113,7 @@ def get_changed_files( return frozenset(map(Path, filter(None, map(str.strip, changed_files)))) -def process_changed_files(changed_files: AbstractSet[Path]) -> Outputs: +def process_changed_files(changed_files: Set[Path]) -> Outputs: run_tests = False run_ci_fuzz = False run_docs = False diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index e4c159366e4c6d..a6498ac416d9b0 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -434,7 +434,7 @@ def get_identifiers_and_strings() -> 'tuple[set[str], dict[str, str]]': # Give a nice message for common mistakes. # To cover tricky cases (like "\n") we also generate C asserts. raise ValueError( - f'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' + 'do not use &_Py_ID or &_Py_STR for one-character latin-1 ' f'strings, use _Py_LATIN1_CHR instead: {string!r}') if string not in strings: strings[string] = name diff --git a/Tools/build/regen-configure.sh b/Tools/build/regen-configure.sh index 967e81b13f029e..c7683eb36763af 100755 --- a/Tools/build/regen-configure.sh +++ b/Tools/build/regen-configure.sh @@ -6,19 +6,18 @@ set -e -x # sync with this script. Use the same container image than the job so the job # doesn't need to run autoreconf in a container. IMAGE="ghcr.io/python/autoconf:2025.01.02.12581854023" -# shellcheck disable=SC2034 AUTORECONF="autoreconf -ivf -Werror" WORK_DIR="/src" -abs_srcdir="$(cd "$(dirname "$0")/../.."; pwd)" +abs_srcdir=$(cd $(dirname $0)/../..; pwd) if podman --version &>/dev/null; then RUNTIME="podman" elif docker --version &>/dev/null; then RUNTIME="docker" else - echo "$* needs either Podman or Docker container runtime." >&2 + echo "$@ needs either Podman or Docker container runtime." >&2 exit 1 fi diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 07b76ed8e676dd..83761d50385ada 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -548,8 +548,7 @@ def gcc_get_limited_api_macros(headers): text=True, ) - matches = re.findall(r"#define (\w+)", preprocessor_output_with_macros) - return set(matches) + return set(re.findall(r"#define (\w+)", preprocessor_output_with_macros)) def gcc_get_limited_api_definitions(headers): @@ -607,7 +606,7 @@ def check_private_names(manifest): if name.startswith('_') and not item.abi_only: raise ValueError( f'`{name}` is private (underscore-prefixed) and should be ' - f'removed from the stable ABI list or marked `abi_only`') + 'removed from the stable ABI list or marked `abi_only`') def check_dump(manifest, filename): """Check that manifest.dump() corresponds to the data. @@ -647,8 +646,8 @@ def main(): ) parser.add_argument( "--generate-all", action='store_true', - help="as --generate, but generate all file(s) using default filenames. " - "(unlike --all, does not run any extra checks)", + help="as --generate, but generate all file(s) using default filenames." + " (unlike --all, does not run any extra checks)", ) parser.add_argument( "-a", "--all", action='store_true', From d4c570770c590091a6a0e7695b88c767bfbe7707 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:19:03 +0100 Subject: [PATCH 36/43] Revert whitespace --- Tools/build/check_extension_modules.py | 1 - Tools/build/deepfreeze.py | 1 - 2 files changed, 2 deletions(-) diff --git a/Tools/build/check_extension_modules.py b/Tools/build/check_extension_modules.py index 84dcd6ba57f12c..9815bcfe27d995 100644 --- a/Tools/build/check_extension_modules.py +++ b/Tools/build/check_extension_modules.py @@ -17,7 +17,6 @@ See --help for more information """ - import _imp import argparse import collections diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index 9919641242f8ae..f5e79561153f84 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -5,7 +5,6 @@ On Windows, and in cross-compilation cases, it is executed by Python 3.10, and 3.11 features are not available. """ - import argparse import builtins import collections From 407abeffc57df4d37ed4408ededdf476b1f2b130 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:22:22 +0100 Subject: [PATCH 37/43] Ignore F541 --- Tools/build/.ruff.toml | 1 + Tools/build/deepfreeze.py | 12 ++++++------ Tools/build/freeze_modules.py | 2 +- Tools/build/generate_global_objects.py | 14 +++++++------- Tools/build/stable_abi.py | 2 +- 5 files changed, 16 insertions(+), 15 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 3f18c296db9689..4828f9dfe7e8da 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -19,6 +19,7 @@ select = [ ] ignore = [ "E501", # Line too long + "F541", # f-string without any placeholders ] [per-file-target-version] diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index f5e79561153f84..a89112a12c4b48 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -235,7 +235,7 @@ def generate_unicode(self, name: str, s: str) -> str: utf8 = s.encode('utf-8') self.write(f'.utf8 = {make_string_literal(utf8)},') self.write(f'.utf8_length = {len(utf8)},') - with self.block("._data =", ","): + with self.block(f"._data =", ","): for i in range(0, len(s), 16): data = s[i:i+16] self.write(", ".join(map(str, map(ord, data))) + ",") @@ -292,7 +292,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: self.write(f".co_name = {co_name},") self.write(f".co_qualname = {co_qualname},") self.write(f".co_linetable = {co_linetable},") - self.write("._co_cached = NULL,") + self.write(f"._co_cached = NULL,") self.write(f".co_code_adaptive = {co_code_adaptive},") first_traceable = 0 for op in code.co_code[::2]: @@ -307,7 +307,7 @@ def generate_code(self, name: str, code: types.CodeType) -> str: def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: if len(t) == 0: - return "(PyObject *)& _Py_SINGLETON(tuple_empty)" + return f"(PyObject *)& _Py_SINGLETON(tuple_empty)" items = [self.generate(f"{name}_{i}", it) for i, it in enumerate(t)] self.write("static") with self.indent(): @@ -321,7 +321,7 @@ def generate_tuple(self, name: str, t: tuple[object, ...]) -> str: with self.block("._object =", ","): self.object_var_head("PyTuple_Type", len(t)) if items: - with self.block(".ob_item =", ","): + with self.block(f".ob_item =", ","): for item in items: self.write(item + ",") return f"& {name}._object.ob_base.ob_base" @@ -472,10 +472,10 @@ def generate(args: list[str], output: TextIO) -> None: else: code = compile(fd.read(), f"", "exec") printer.generate_file(modname, code) - with printer.block("void\n_Py_Deepfreeze_Fini(void)"): + with printer.block(f"void\n_Py_Deepfreeze_Fini(void)"): for p in printer.finis: printer.write(p) - with printer.block("int\n_Py_Deepfreeze_Init(void)"): + with printer.block(f"int\n_Py_Deepfreeze_Init(void)"): for p in printer.inits: with printer.block(f"if ({p} < 0)"): printer.write("return -1;") diff --git a/Tools/build/freeze_modules.py b/Tools/build/freeze_modules.py index 2103bcf057592f..3c43f7e3bbe8ca 100644 --- a/Tools/build/freeze_modules.py +++ b/Tools/build/freeze_modules.py @@ -631,7 +631,7 @@ def regen_pcbuild(modules): projlines.append(f' {src.frozenid}') projlines.append(f' $(IntDir){intfile}') projlines.append(f' $(GeneratedFrozenModulesDir){header}') - projlines.append(' ') + projlines.append(f' ') filterlines.append(f' ') filterlines.append(' Python Files') diff --git a/Tools/build/generate_global_objects.py b/Tools/build/generate_global_objects.py index a6498ac416d9b0..29bd995655e0b3 100644 --- a/Tools/build/generate_global_objects.py +++ b/Tools/build/generate_global_objects.py @@ -366,19 +366,19 @@ def generate_static_strings_initializer(identifiers, strings): printer.write(START) printer.write("static inline void") with printer.block("_PyUnicode_InitStaticStrings(PyInterpreterState *interp)"): - printer.write('PyObject *string;') + printer.write(f'PyObject *string;') for i in sorted(identifiers): # This use of _Py_ID() is ignored by iter_global_strings() # since iter_files() ignores .h files. printer.write(f'string = &_Py_ID({i});') - printer.write('_PyUnicode_InternStatic(interp, &string);') - printer.write('assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write('assert(PyUnicode_GET_LENGTH(string) != 1);') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') for value, name in sorted(strings.items()): printer.write(f'string = &_Py_STR({name});') - printer.write('_PyUnicode_InternStatic(interp, &string);') - printer.write('assert(_PyUnicode_CheckConsistency(string, 1));') - printer.write('assert(PyUnicode_GET_LENGTH(string) != 1);') + printer.write(f'_PyUnicode_InternStatic(interp, &string);') + printer.write(f'assert(_PyUnicode_CheckConsistency(string, 1));') + printer.write(f'assert(PyUnicode_GET_LENGTH(string) != 1);') printer.write(END) printer.write(after) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 83761d50385ada..2c0c21675d215f 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -337,7 +337,7 @@ def test_windows_feature_macros(self): write(")") for ifdef, names in optional_items.items(): write(f"if feature_macros[{ifdef!r}]:") - write(" SYMBOL_NAMES += (") + write(f" SYMBOL_NAMES += (") for name in names: write(f" {name!r},") write(" )") From 0986b77c909f88c7a8d4a42a978b6b5e1bfc7a33 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:32:24 +0100 Subject: [PATCH 38/43] Ignore UP038 (not recommended) --- Tools/build/.ruff.toml | 1 + Tools/build/deepfreeze.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 4828f9dfe7e8da..3dfd82a666f75e 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -20,6 +20,7 @@ select = [ ignore = [ "E501", # Line too long "F541", # f-string without any placeholders + "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` ] [per-file-target-version] diff --git a/Tools/build/deepfreeze.py b/Tools/build/deepfreeze.py index a89112a12c4b48..23f58447937976 100644 --- a/Tools/build/deepfreeze.py +++ b/Tools/build/deepfreeze.py @@ -402,7 +402,7 @@ def generate(self, name: str, obj: object) -> str: # print(f"Cache hit {key!r:.40}: {self.cache[key]!r:.40}") return self.cache[key] self.misses += 1 - if isinstance(obj, types.CodeType | umarshal.Code): + if isinstance(obj, (types.CodeType, umarshal.Code)) : val = self.generate_code(name, obj) elif isinstance(obj, tuple): val = self.generate_tuple(name, obj) From 673aca0bb5d5920c3c5e2aaa2c996363b220dcc2 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Fri, 11 Apr 2025 20:32:42 +0100 Subject: [PATCH 39/43] Prefer 'encoding' over 'text=True' --- Tools/build/smelly.py | 2 +- Tools/build/stable_abi.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Tools/build/smelly.py b/Tools/build/smelly.py index 17e070c7b7fb8a..9a360412a73a4d 100755 --- a/Tools/build/smelly.py +++ b/Tools/build/smelly.py @@ -52,7 +52,7 @@ def get_exported_symbols(library, dynamic=False): args.append('--dynamic') args.append(library) print(f"+ {' '.join(args)}") - proc = subprocess.run(args, stdout=subprocess.PIPE, text=True) + proc = subprocess.run(args, stdout=subprocess.PIPE, encoding='utf-8') if proc.returncode: sys.stdout.write(proc.stdout) sys.exit(proc.returncode) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index 2c0c21675d215f..e0142c06bb0bdc 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -475,7 +475,7 @@ def binutils_get_exported_symbols(library, dynamic=False): if dynamic: args.append("--dynamic") args.append(library) - proc = subprocess.run(args, stdout=subprocess.PIPE, text=True) + proc = subprocess.run(args, stdout=subprocess.PIPE, encoding='utf-8') if proc.returncode: sys.stdout.write(proc.stdout) sys.exit(proc.returncode) @@ -545,7 +545,7 @@ def gcc_get_limited_api_macros(headers): "-E", ] + [str(file) for file in headers], - text=True, + encoding='utf-8', ) return set(re.findall(r"#define (\w+)", preprocessor_output_with_macros)) @@ -583,7 +583,7 @@ def gcc_get_limited_api_definitions(headers): "-E", ] + [str(file) for file in headers], - text=True, + encoding='utf-8', stderr=subprocess.DEVNULL, ) stable_functions = set( From 4308c1f2ae5cc5cb568c4ffc295ab2b0151dd32e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 13 Apr 2025 09:38:12 +0200 Subject: [PATCH 40/43] align strings --- Tools/build/stable_abi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tools/build/stable_abi.py b/Tools/build/stable_abi.py index e0142c06bb0bdc..1ddd76cdd9bf64 100644 --- a/Tools/build/stable_abi.py +++ b/Tools/build/stable_abi.py @@ -647,7 +647,7 @@ def main(): parser.add_argument( "--generate-all", action='store_true', help="as --generate, but generate all file(s) using default filenames." - " (unlike --all, does not run any extra checks)", + " (unlike --all, does not run any extra checks)", ) parser.add_argument( "-a", "--all", action='store_true', From eae52159c4463d7b52322020fd608b16a8a2e14f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 13 Apr 2025 09:40:18 +0200 Subject: [PATCH 41/43] add new line --- Tools/build/generate_sbom.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Tools/build/generate_sbom.py b/Tools/build/generate_sbom.py index 5c2a3bc1364873..db01426e9722c3 100644 --- a/Tools/build/generate_sbom.py +++ b/Tools/build/generate_sbom.py @@ -1,4 +1,5 @@ """Tool for generating Software Bill of Materials (SBOM) for Python's dependencies""" + import glob import hashlib import json From 99a41ef8d5526509e2dd19a44d944ff0ca0bdde0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sun, 13 Apr 2025 18:38:46 +0200 Subject: [PATCH 42/43] ignore UP031 for `Tools/build/generate_token.py` --- Tools/build/.ruff.toml | 5 ++++- Tools/build/generate_token.py | 24 ++++++++++++------------ 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 3dfd82a666f75e..2878b8a45bb55b 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -28,6 +28,9 @@ ignore = [ "stable_abi.py" = "py311" # requires 'tomllib' [lint.per-file-ignores] -"{check_extension_modules,freeze_modules,generate_re_casefix,generate_sre_constants}.py" = [ +"{check_extension_modules,freeze_modules}.py" = [ + "UP031", # Use format specifiers instead of percent format +] +"generate_{re_casefix,sre_constants,token}.py" = [ "UP031", # Use format specifiers instead of percent format ] diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py index bbbda6556d4afd..7316333dcd452e 100755 --- a/Tools/build/generate_token.py +++ b/Tools/build/generate_token.py @@ -107,14 +107,14 @@ def make_h(infile, outfile='Include/internal/pycore_token.h'): defines = [] for value, name in enumerate(tok_names[:ERRORTOKEN + 1]): - defines.append("#define %-15s %d\n" % (name, value)) # noqa: UP031 + defines.append("#define %-15s %d\n" % (name, value)) if update_file(outfile, token_h_template % ( ''.join(defines), len(tok_names), NT_OFFSET )): - print(f"{outfile} regenerated from {infile}") + print("%s regenerated from %s" % (outfile, infile)) token_c_template = f"""\ @@ -160,17 +160,17 @@ def generate_chars_to_token(mapping, n=1): write = result.append indent = ' ' * n write(indent) - write('switch (c%d) {\n' % (n,)) # noqa: UP031 + write('switch (c%d) {\n' % (n,)) for c in sorted(mapping): write(indent) value = mapping[c] if isinstance(value, dict): - write("case '%s':\n" % (c,)) # noqa: UP031 + write("case '%s':\n" % (c,)) write(generate_chars_to_token(value, n + 1)) write(indent) write(' break;\n') else: - write("case '%s': return %s;\n" % (c, value)) # noqa: UP031 + write("case '%s': return %s;\n" % (c, value)) write(indent) write('}\n') return ''.join(result) @@ -190,8 +190,8 @@ def make_c(infile, outfile='Parser/token.c'): names = [] for value, name in enumerate(tok_names): if value >= ERRORTOKEN: - name = '<%s>' % name # noqa: UP031 - names.append(' "%s",\n' % name) # noqa: UP031 + name = '<%s>' % name + names.append(' "%s",\n' % name) names.append(' "",\n') if update_file(outfile, token_c_template % ( @@ -200,7 +200,7 @@ def make_c(infile, outfile='Parser/token.c'): generate_chars_to_token(chars_to_token[2]), generate_chars_to_token(chars_to_token[3]) )): - print(f"{outfile} regenerated from {infile}") + print("%s regenerated from %s" % (outfile, infile)) token_inc_template = f"""\ @@ -252,7 +252,7 @@ def make_rst(infile, outfile='Doc/library/token-list.inc', exit('\n'.join(message_parts)) if update_file(outfile, token_inc_template % '\n'.join(names)): - print(f"{outfile} regenerated from {infile}") + print("%s regenerated from %s" % (outfile, infile)) token_py_template = f'''\ @@ -292,13 +292,13 @@ def make_py(infile, outfile='Lib/token.py'): constants = [] for value, name in enumerate(tok_names): - constants.append('%s = %d' % (name, value)) # noqa: UP031 + constants.append('%s = %d' % (name, value)) constants.insert(ERRORTOKEN, "# These aren't used by the C tokenizer but are needed for tokenize.py") token_types = [] for s, value in sorted(string_to_tok.items()): - token_types.append(' %r: %s,' % (s, tok_names[value])) # noqa: UP031 + token_types.append(' %r: %s,' % (s, tok_names[value])) if update_file(outfile, token_py_template % ( '\n'.join(constants), @@ -306,7 +306,7 @@ def make_py(infile, outfile='Lib/token.py'): NT_OFFSET, '\n'.join(token_types), )): - print(f"{outfile} regenerated from {infile}") + print("%s regenerated from %s" % (outfile, infile)) def main(op, infile='Grammar/Tokens', *args): From 75a08cfcc651320df2eb67e1476293fdfb1f6624 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?B=C3=A9n=C3=A9dikt=20Tran?= <10796600+picnixz@users.noreply.github.com> Date: Sat, 19 Apr 2025 11:07:59 +0200 Subject: [PATCH 43/43] address review --- .pre-commit-config.yaml | 2 +- Tools/build/.ruff.toml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4823bc2b86cc84..c77030ca5fc658 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: args: [--exit-non-zero-on-fix] files: ^Lib/test/ - id: ruff - name: Run Ruff (lint) on Tools/build + name: Run Ruff (lint) on Tools/build/ args: [--exit-non-zero-on-fix, --config=Tools/build/.ruff.toml] files: ^Tools/build/ - id: ruff diff --git a/Tools/build/.ruff.toml b/Tools/build/.ruff.toml index 2878b8a45bb55b..e4f024333adf1a 100644 --- a/Tools/build/.ruff.toml +++ b/Tools/build/.ruff.toml @@ -12,6 +12,7 @@ select = [ "LOG", # flake8-logging "PGH", # pygrep-hooks "PT", # flake8-pytest-style + "PYI", # flake8-pyi "RUF100", # Ban unused `# noqa` comments "UP", # pyupgrade "W", # pycodestyle @@ -20,6 +21,8 @@ select = [ ignore = [ "E501", # Line too long "F541", # f-string without any placeholders + "PYI024", # Use `typing.NamedTuple` instead of `collections.namedtuple` + "PYI025", # Use `from collections.abc import Set as AbstractSet` "UP038", # Use `X | Y` in `isinstance` call instead of `(X, Y)` ]