diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..34cce10 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +name: CI + +on: + push: + branches: + - main + pull_request: + workflow_dispatch: + +jobs: + test: + strategy: + matrix: + python: + - "3.7" + - "3.8" + - "3.9" + - "3.10" + - "3.11" + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + cache: pip + + - name: test + run: make test INSTALL_EXTRA=test + +# lint: +# runs-on: ubuntu-latest +# steps: +# - uses: actions/checkout@v3 +# - uses: actions/setup-python@v4 +# with: +# python-version: "3.10" +# - name: lint +# run: make lint INSTALL_EXTRA=lint diff --git a/.github/workflows/listgen.yml b/.github/workflows/listgen.yml new file mode 100644 index 0000000..785c314 --- /dev/null +++ b/.github/workflows/listgen.yml @@ -0,0 +1,115 @@ +name: Generate stdlib lists + +on: + workflow_dispatch: + schedule: + - cron: "0 0 * * 2" + +jobs: + pre-list-legacy: + strategy: + matrix: + python: + - "3.7" + - "3.8" + - "3.9" + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + # NOTE: matrix.python is intentionally not used here. + python-version: "3.x" + - name: support deps + run: make dev INSTALL_EXTRA=support + - name: build pre-list + env: + LISTGEN_PYTHON_VERSION: "${{ matrix.python }}" + run: | + ./env/bin/python ./support/fetch-sphinx.py "${LISTGEN_PYTHON_VERSION}" > pre-list.txt + - name: upload pre-list + uses: actions/upload-artifact@v3 + with: + name: pre-list-${{ matrix.python }} + path: pre-list.txt + + expand-list-legacy: + needs: pre-list-legacy + strategy: + matrix: + python: + - "3.7" + - "3.8" + - "3.9" + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - uses: actions/download-artifact@v3 + with: + name: pre-list-${{ matrix.python }} + - name: walk modules + run: | + python -V + python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt" < pre-list.txt + rm pre-list.txt + sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt" + mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/ + - name: create PR + uses: peter-evans/create-pull-request@v5 + with: + commit-message: "[BOT] update list for ${{ matrix.python }}" + branch: update-stdlib-list-${{ matrix.python }} + base: main + branch-suffix: timestamp + title: "[BOT] update list for ${{ matrix.python }}" + body: | + This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change. + + Please review manually before merging. + assignees: "woodruffw" + reviewers: "woodruffw" + + expand-list: + strategy: + matrix: + python: + - "3.10" + - "3.11" + + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python }} + - name: walk modules + env: + LISTGEN_PYTHON_VERSION: ${{ matrix.python }} + run: | + python -V + python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt" + sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt" + mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/ + + - name: create PR + uses: peter-evans/create-pull-request@v5 + with: + commit-message: "[BOT] update list for ${{ matrix.python }}" + branch: update-stdlib-list-${{ matrix.python }} + base: main + branch-suffix: timestamp + title: "[BOT] update list for ${{ matrix.python }}" + body: | + This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change. + + Please review manually before merging. + assignees: "woodruffw" + reviewers: "woodruffw" diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 853cccf..0000000 --- a/.travis.yml +++ /dev/null @@ -1,31 +0,0 @@ -language: python - -sudo: false - -python: - - 2.7 - - 3.4 - - 3.5 - - 3.6 - - 3.7 - - 3.8 - -install: - - python setup.py sdist && version=$(python setup.py --version) && pushd dist && pip install stdlib-list-${version}.tar.gz && popd - -script: - - python -m tests - -deploy: - skip_cleanup: true - provider: pypi - user: ocefpaf - password: - secure: "Q4v+Im8wOvYkCfszigRwh26DKSBpgJ49AGffkP3gb592z4uVtVH5DWdXyUE/9zK8qVk4uA5NEBJc8+mHFCztUlYYrPXZCVzX+NyUWv3rmMsDf8tV5Y+9J6pdw0Wx6DvI6nqKD6s2hKu4pey2UW8LO9HQIi97Hlo5Hd5ivbakVX8=" - distributions: sdist bdist_wheel - upload_docs: no - on: - repo: jackmaney/python-stdlib-list - tags: true - all_branches: master - python: 3.7 diff --git a/Makefile b/Makefile index c26dea3..042a133 100644 --- a/Makefile +++ b/Makefile @@ -70,7 +70,7 @@ reformat: test tests: $(VENV)/pyvenv.cfg . $(VENV_BIN)/activate && \ pytest --cov=$(PY_MODULE) $(T) $(TEST_ARGS) && \ - python -m coverage report -m $(COV_ARGS) + python -m coverage report -m $(COV_ARGS) --fail-under 100 .PHONY: doc doc: $(VENV)/pyvenv.cfg diff --git a/README.md b/README.md index df7d0f3..4563eb3 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,11 @@ This package includes lists of all of the standard libraries for Python 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, and 3.9 along with the code for scraping the official Python docs to get said lists. +**IMPORTANT**: If you're on Python 3.10 or newer, you **probably don't need this library**. +See [`sys.stdlib_module_names`](https://docs.python.org/3/library/sys.html#sys.stdlib_module_names) +and [`sys.builtin_module_names`](https://docs.python.org/3/library/sys.html#sys.builtin_module_names) +for similar functionality. + ## Installation `stdlib-list` is available on PyPI: diff --git a/pyproject.toml b/pyproject.toml index 821e077..fb49c61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,9 @@ test = ["pytest", "pytest-cov", "coverage[toml]"] lint = ["black", "mypy", "ruff"] doc = ["sphinx", "sphinx_rtd_theme"] dev = ["build", "stdlib-list[test,lint,doc]"] +# CI only: used for list generation for Python versions < 3.10. +support = ["sphobjinv"] + [tool.black] line-length = 100 diff --git a/stdlib_list/__init__.py b/stdlib_list/__init__.py index 95e9050..ab610d9 100644 --- a/stdlib_list/__init__.py +++ b/stdlib_list/__init__.py @@ -8,3 +8,11 @@ short_versions, long_versions, ) + +__all__ = [ + "stdlib_list", + "in_stdlib", + "get_canonical_version", + "short_versions", + "long_versions", +] diff --git a/stdlib_list/base.py b/stdlib_list/base.py index 496fc16..75aa30b 100644 --- a/stdlib_list/base.py +++ b/stdlib_list/base.py @@ -6,7 +6,20 @@ from functools import lru_cache -long_versions = ["2.6.9", "2.7.9", "3.2.6", "3.3.6", "3.4.3", "3.5", "3.6", "3.7", "3.8", "3.9"] +long_versions = [ + "2.6.9", + "2.7.9", + "3.2.6", + "3.3.6", + "3.4.3", + "3.5", + "3.6", + "3.7", + "3.8", + "3.9", + "3.10", + "3.11", +] short_versions = [".".join(x.split(".")[:2]) for x in long_versions] @@ -23,11 +36,10 @@ def get_canonical_version(version): def stdlib_list(version=None): """ Given a ``version``, return a ``list`` of names of the Python Standard - Libraries for that version. These names are obtained from the Sphinx inventory - file (used in :py:mod:`sphinx.ext.intersphinx`). + Libraries for that version. :param str|None version: The version (as a string) whose list of libraries you want - (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``). + (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``). If not specified, the current version of Python will be used. @@ -72,7 +84,7 @@ def in_stdlib(module_name, version=None): :param str|None module_name: The module name (as a string) to query for. :param str|None version: The version (as a string) whose list of libraries you want - (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``). + (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``). If not specified, the current version of Python will be used. diff --git a/support/fetch-sphinx.py b/support/fetch-sphinx.py new file mode 100755 index 0000000..7398c18 --- /dev/null +++ b/support/fetch-sphinx.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python + +# fetch-sphinx.py: retrieve a particular Python version's stdlib list +# using its hosted Sphinx inventory. + +import sys +import sphobjinv as soi + +if __name__ == "__main__": + vers = sys.argv[1] + inv = soi.Inventory(url=f"https://docs.python.org/{vers}/objects.inv") + modules = list(sorted(obj.name for obj in inv.objects if obj.role == "module")) + print("\n".join(modules)) diff --git a/support/walk-modules.py b/support/walk-modules.py new file mode 100755 index 0000000..8713939 --- /dev/null +++ b/support/walk-modules.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python + +import inspect +import pkgutil +import sys + +SEEN_MODS = set() + + +def walk_pkgutil(mod_name, mod, io): + for pkg in pkgutil.walk_packages(mod.__path__, mod_name + "."): + if pkg.name in SEEN_MODS: + continue + else: + # We don't recurse here because `walk_packages` takes care of + # it for us. + SEEN_MODS.add(pkg.name) + print(pkg.name, file=io) + + +def walk_naive(mod_name, mod, io): + for attr in dir(mod): + attr_obj = getattr(mod, attr, None) + # Shouldn't happen, but who knows. + if attr_obj is None: + continue + + # If this member isn't a module, skip it. + if not inspect.ismodule(attr_obj): + continue + + # To filter "real" submodules from re-exports, we try + # and import the submodule by its qualified name. + # If the import fails, we know it's a re-exported module. + try: + submod_name = mod_name + "." + attr + __import__(submod_name) + walk(submod_name, io) + except ImportError: + # ...but sometimes we do want to include re-exports, since + # they might be things like "accelerator" modules that don't + # appear anywhere else. + # For example, `_bz2` might appear as a re-export. + try: + # Again, try and import to avoid module-looking object + # that don't actually appear on disk. Experimentally, + # there are a few of these (like "TK"). + __import__(attr) + walk(attr, io) + except ImportError: + continue + + +def walk(mod_name, io): + if mod_name in SEEN_MODS: + return + else: + SEEN_MODS.add(mod_name) + print(mod_name, file=io) + + # Try and import it. + try: + mod = __import__(mod_name) + + if hasattr(mod, "__path__"): + walk_pkgutil(mod_name, mod, io) + else: + walk_naive(mod_name, mod, io) + + except ImportError: + pass + + +if __name__ == "__main__": + output = sys.argv[1] + + with open(output, mode="w") as io: + for mod_name in sys.builtin_module_names: + walk(mod_name, io) + + if hasattr(sys, "stdlib_module_names"): + for mod_name in sys.stdlib_module_names: + walk(mod_name, io) + else: + for mod_name in sys.stdin: + walk(mod_name.rstrip("\n"), io) diff --git a/tests/__main__.py b/tests/__main__.py deleted file mode 100644 index 9fea50d..0000000 --- a/tests/__main__.py +++ /dev/null @@ -1,7 +0,0 @@ -import unittest - -from tests.test_basic import * -from tests.test_platform import * - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_base.py b/tests/test_base.py new file mode 100644 index 0000000..89fbc71 --- /dev/null +++ b/tests/test_base.py @@ -0,0 +1,29 @@ +import pkgutil +import pytest + +import stdlib_list + + +@pytest.mark.parametrize( + ("version", "canonicalized"), + [("2.7", "2.7"), ("3.10", "3.10")], +) +def test_get_canonical_version(version, canonicalized): + assert stdlib_list.get_canonical_version(version) == canonicalized + + +@pytest.mark.parametrize("version", ["nonsense", "1.2.3", "3.1000"]) +def test_get_canonical_version_raises(version): + with pytest.raises(ValueError, match=rf"No such version: {version}"): + stdlib_list.get_canonical_version(version) + + +@pytest.mark.parametrize("version", [*stdlib_list.short_versions, *stdlib_list.long_versions]) +def test_self_consistent(version): + list_path = f"lists/{stdlib_list.get_canonical_version(version)}.txt" + modules = pkgutil.get_data("stdlib_list", list_path).decode().splitlines() + + for mod_name in modules: + assert stdlib_list.in_stdlib(mod_name, version) + + assert modules == stdlib_list.stdlib_list(version) diff --git a/tests/test_basic.py b/tests/test_basic.py deleted file mode 100644 index 257f04b..0000000 --- a/tests/test_basic.py +++ /dev/null @@ -1,99 +0,0 @@ -import sys -import unittest - -import stdlib_list - -PY2 = sys.version_info[0] == 2 - - -class CurrentVersionBase(unittest.TestCase): - def setUp(self): - self.list = stdlib_list.stdlib_list(sys.version[:3]) - - -class TestCurrentVersion(CurrentVersionBase): - def test_string(self): - self.assertIn("string", self.list) - - def test_list_is_sorted(self): - self.assertEqual(sorted(self.list), self.list) - - def test_builtin_modules(self): - """Check all top level stdlib packages are recognised.""" - unknown_builtins = set() - for module_name in sys.builtin_module_names: - if module_name not in self.list: - unknown_builtins.add(module_name) - - self.assertFalse(sorted(unknown_builtins)) - - -class TestSysModules(CurrentVersionBase): - - # This relies on invocation in a clean python environment using unittest - # not using pytest. - - ignore_list = ["stdlib_list", "functools32", "tests", "_virtualenv_distutils"] - - def setUp(self): - super(TestSysModules, self).setUp() - self.maxDiff = None - - def test_preloaded_packages(self): - """Check all top level stdlib packages are recognised.""" - not_stdlib = set() - for module_name in sys.modules: - pkg, _, module = module_name.partition(".") - - # https://github.com/jackmaney/python-stdlib-list/issues/29 - if pkg.startswith("_sysconfigdata_"): - continue - - if pkg in self.ignore_list: - continue - - # Avoid duplicating errors covered by other tests - if pkg in sys.builtin_module_names: - continue - - if pkg not in self.list: - not_stdlib.add(pkg) - - self.assertFalse(sorted(not_stdlib)) - - def test_preloaded_modules(self): - """Check all stdlib modules are recognised.""" - not_stdlib = set() - for module_name in sys.modules: - pkg, _, module = module_name.partition(".") - - # https://github.com/jackmaney/python-stdlib-list/issues/29 - if pkg.startswith("_sysconfigdata_"): - continue - - if pkg in self.ignore_list: - continue - - # Avoid duplicating errors covered by other tests - if module_name in sys.builtin_module_names: - continue - - if PY2: - # Python 2.7 creates sub-modules for imports - if pkg in self.list and module in self.list: - continue - - # Python 2.7 deprecation solution for old names - if pkg == "email": - mod = sys.modules[module_name] - if mod.__class__.__name__ == "LazyImporter": - continue - - if module_name not in self.list: - not_stdlib.add(module_name) - - self.assertFalse(sorted(not_stdlib)) - - -if __name__ == "__main__": - unittest.main() diff --git a/tests/test_platform.py b/tests/test_platform.py deleted file mode 100644 index 226396a..0000000 --- a/tests/test_platform.py +++ /dev/null @@ -1,174 +0,0 @@ -import difflib -import os -import os.path -import sys -import unittest -from distutils.sysconfig import get_python_lib -from sysconfig import get_config_var - -import stdlib_list - -try: - sys.base_prefix - has_base_prefix = sys.base_prefix != sys.prefix -except AttributeError: - has_base_prefix = False - -shlib_ext = get_config_var("SHLIB_SUFFIX") or get_config_var("SO") - -tk_libs = get_config_var("TKPATH") - - -class UnifiedDiffAssertionError(AssertionError): - def __init__(self, expected, got, msg="Differences"): - super(UnifiedDiffAssertionError, self).__init__(self) - filename = "stdlib_list/lists/{}.txt".format(sys.version[:3]) - diff = difflib.unified_diff( - expected, got, lineterm="", fromfile="a/" + filename, tofile="b/" + filename - ) - self.description = "{name}\n{diff}".format(name=msg, diff="\n".join(diff)) - - def __str__(self): - return self.description - - -class CurrentPlatformBase(object): - - dir = None - ignore_test = False - - def setUp(self): - self.list = stdlib_list.stdlib_list(sys.version[:3]) - if self.dir: - self.assertTrue(os.path.isdir(self.dir)) - - def _collect_shared(self, name): - # stdlib extensions are not in subdirectories - if "/" in name: - return None - - return name.split(".", 1)[0] - - def _collect_file(self, name): - if name.endswith(shlib_ext): - return self._collect_shared(name) - - if not name.endswith(".py"): - return None - - # This excludes `_sysconfigdata_m_linux_x86_64-linux-gnu` - # https://github.com/jackmaney/python-stdlib-list/issues/29 - if "-" in name: - return None - - # Ignore this oddball stdlib test.test_frozen helper - if name == "__phello__.foo.py": - return None - - if name.endswith("/__init__.py"): - return name[:-12].replace("/", ".") - - # Strip .py and replace '/' - return name[:-3].replace("/", ".") - - def _collect_all(self, base): - base = base + "/" if not base.endswith("/") else base - base_len = len(base) - modules = [] - - for root, dirs, files in os.walk(base): - for filename in files: - relative_base = root[base_len:] - relative_path = os.path.join(relative_base, filename) - module_name = self._collect_file(relative_path) - if module_name: - modules.append(module_name) - - # In-place filtering of traversal, removing invalid module names - # and cache directories - for dir in dirs: - if "-" in dir: - dirs.remove(dir) - if "__pycache__" in dirs: - dirs.remove("__pycache__") - - if self.ignore_test and "test" in dirs: - dirs.remove("test") - - # openSUSE custom module added to stdlib directory - if "_import_failed" in dirs: - dirs.remove("_import_failed") - - return modules - - def assertNoDiff(self, base, new): - if base == new: - self.assertEqual(base, new) - else: - raise UnifiedDiffAssertionError(got=sorted(new), expected=sorted(base)) - - def test_dir(self): - needed = set(self.list) - items = self._collect_all(self.dir) - for item in items: - if item not in self.list: - needed.add(item) - - self.assertNoDiff(set(self.list), needed) - - -class TestPureLibDir(CurrentPlatformBase, unittest.TestCase): - def setUp(self): - self.dir = get_python_lib(standard_lib=True, plat_specific=False) - super(TestPureLibDir, self).setUp() - - -class TestPlatLibDir(CurrentPlatformBase, unittest.TestCase): - def setUp(self): - self.dir = get_python_lib(standard_lib=True, plat_specific=True) - super(TestPlatLibDir, self).setUp() - - -class TestSharedDir(CurrentPlatformBase, unittest.TestCase): - def setUp(self): - self.dir = get_config_var("DESTSHARED") - super(TestSharedDir, self).setUp() - - -if has_base_prefix: - - class TestBasePureLibDir(CurrentPlatformBase, unittest.TestCase): - def setUp(self): - base = sys.base_prefix - self.dir = get_python_lib( - standard_lib=True, plat_specific=False, prefix=base - ) - super(TestBasePureLibDir, self).setUp() - - class TestBasePlatLibDir(CurrentPlatformBase, unittest.TestCase): - def setUp(self): - base = sys.base_prefix - self.dir = get_python_lib( - standard_lib=True, plat_specific=True, prefix=base - ) - super(TestBasePlatLibDir, self).setUp() - - -if tk_libs: - tk_libs = tk_libs.strip(os.pathsep) - - class TestTkDir(CurrentPlatformBase, unittest.TestCase): - - # Python 2.7 tk-libs includes a `test` package, however it is - # added to sys.path by test.test_tk as a top level directory - # so test.widget_tests becomes module name `widget_tests` - ignore_test = True - - def setUp(self): - base = get_python_lib(standard_lib=True, plat_specific=False) - self.dir = os.path.join(base, tk_libs) - super(TestTkDir, self).setUp() - - -if __name__ == "__main__": - unittest.main()