Fix tests, run tests in CI

.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  test:
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+          cache: pip
+
+      - name: test
+        run: make test INSTALL_EXTRA=test
+
+#   lint:
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v3
+#       - uses: actions/setup-python@v4
+#         with:
+#           python-version: "3.10"
+#       - name: lint
+#         run: make lint INSTALL_EXTRA=lint

.github/workflows/listgen.yml

@@ -0,0 +1,115 @@
+name: Generate stdlib lists
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "0 0 * * 2"
+
+jobs:
+  pre-list-legacy:
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          # NOTE: matrix.python is intentionally not used here.
+          python-version: "3.x"
+      - name: support deps
+        run: make dev INSTALL_EXTRA=support
+      - name: build pre-list
+        env:
+          LISTGEN_PYTHON_VERSION: "${{ matrix.python }}"
+        run: |
+          ./env/bin/python ./support/fetch-sphinx.py "${LISTGEN_PYTHON_VERSION}" > pre-list.txt
+      - name: upload pre-list
+        uses: actions/upload-artifact@v3
+        with:
+          name: pre-list-${{ matrix.python }}
+          path: pre-list.txt
+
+  expand-list-legacy:
+    needs: pre-list-legacy
+    strategy:
+      matrix:
+        python:
+          - "3.7"
+          - "3.8"
+          - "3.9"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - uses: actions/download-artifact@v3
+        with:
+          name: pre-list-${{ matrix.python }}
+      - name: walk modules
+        run: |
+          python -V
+          python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt" < pre-list.txt
+          rm pre-list.txt
+          sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt"
+          mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/
+      - name: create PR
+        uses: peter-evans/create-pull-request@v5
+        with:
+          commit-message: "[BOT] update list for ${{ matrix.python }}"
+          branch: update-stdlib-list-${{ matrix.python }}
+          base: main
+          branch-suffix: timestamp
+          title: "[BOT] update list for ${{ matrix.python }}"
+          body: |
+            This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change.
+
+            Please review manually before merging.
+          assignees: "woodruffw"
+          reviewers: "woodruffw"
+
+  expand-list:
+    strategy:
+      matrix:
+        python:
+          - "3.10"
+          - "3.11"
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python }}
+      - name: walk modules
+        env:
+          LISTGEN_PYTHON_VERSION: ${{ matrix.python }}
+        run: |
+          python -V
+          python ./support/walk-modules.py "${LISTGEN_PYTHON_VERSION}.txt"
+          sort -o "${LISTGEN_PYTHON_VERSION}.txt" "${LISTGEN_PYTHON_VERSION}.txt"
+          mv "${LISTGEN_PYTHON_VERSION}.txt" ./stdlib_list/lists/
+
+      - name: create PR
+        uses: peter-evans/create-pull-request@v5
+        with:
+          commit-message: "[BOT] update list for ${{ matrix.python }}"
+          branch: update-stdlib-list-${{ matrix.python }}
+          base: main
+          branch-suffix: timestamp
+          title: "[BOT] update list for ${{ matrix.python }}"
+          body: |
+            This is an automated pull request, updating `${{ matrix.python }}.txt` after a detected change.
+
+            Please review manually before merging.
+          assignees: "woodruffw"
+          reviewers: "woodruffw"

Makefile

@@ -70,7 +70,7 @@ reformat:
 test tests: $(VENV)/pyvenv.cfg
 	. $(VENV_BIN)/activate && \
 		pytest --cov=$(PY_MODULE) $(T) $(TEST_ARGS) && \
-		python -m coverage report -m $(COV_ARGS)
+		python -m coverage report -m $(COV_ARGS) --fail-under 100
 
 .PHONY: doc
 doc: $(VENV)/pyvenv.cfg

README.md

@@ -4,6 +4,11 @@ This package includes lists of all of the standard libraries for Python 2.6,
 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, and 3.9 along with the code for
 scraping the official Python docs to get said lists.
 
+**IMPORTANT**: If you're on Python 3.10 or newer, you **probably don't need this library**.
+See [`sys.stdlib_module_names`](https://docs.python.org/3/library/sys.html#sys.stdlib_module_names)
+and [`sys.builtin_module_names`](https://docs.python.org/3/library/sys.html#sys.builtin_module_names)
+for similar functionality.
+
 ## Installation
 
 `stdlib-list` is available on PyPI:

pyproject.toml

@@ -32,6 +32,9 @@ test = ["pytest", "pytest-cov", "coverage[toml]"]
 lint = ["black", "mypy", "ruff"]
 doc = ["sphinx", "sphinx_rtd_theme"]
 dev = ["build", "stdlib-list[test,lint,doc]"]
+# CI only: used for list generation for Python versions < 3.10.
+support = ["sphobjinv"]
+
 
 [tool.black]
 line-length = 100

stdlib_list/__init__.py

@@ -8,3 +8,11 @@
     short_versions,
     long_versions,
 )
+
+__all__ = [
+    "stdlib_list",
+    "in_stdlib",
+    "get_canonical_version",
+    "short_versions",
+    "long_versions",
+]

stdlib_list/base.py

@@ -6,7 +6,20 @@
 
 from functools import lru_cache
 
-long_versions = ["2.6.9", "2.7.9", "3.2.6", "3.3.6", "3.4.3", "3.5", "3.6", "3.7", "3.8", "3.9"]
+long_versions = [
+    "2.6.9",
+    "2.7.9",
+    "3.2.6",
+    "3.3.6",
+    "3.4.3",
+    "3.5",
+    "3.6",
+    "3.7",
+    "3.8",
+    "3.9",
+    "3.10",
+    "3.11",
+]
 
 short_versions = [".".join(x.split(".")[:2]) for x in long_versions]
 
@@ -23,11 +36,10 @@ def get_canonical_version(version):
 def stdlib_list(version=None):
     """
     Given a ``version``, return a ``list`` of names of the Python Standard
-    Libraries for that version. These names are obtained from the Sphinx inventory
-    file (used in :py:mod:`sphinx.ext.intersphinx`).
+    Libraries for that version.
 
     :param str|None version: The version (as a string) whose list of libraries you want
-        (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``).
+        (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``).
 
         If not specified, the current version of Python will be used.
 
@@ -72,7 +84,7 @@ def in_stdlib(module_name, version=None):
 
     :param str|None module_name: The module name (as a string) to query for.
     :param str|None version: The version (as a string) whose list of libraries you want
-        (one of ``"2.6"``, ``"2.7"``, ``"3.2"``, ``"3.3"``, ``"3.4"``, or ``"3.5"``).
+        (formatted as ``X.Y``, e.g. ``"2.7"`` or ``"3.10"``).
 
         If not specified, the current version of Python will be used.
 

support/fetch-sphinx.py

@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+# fetch-sphinx.py: retrieve a particular Python version's stdlib list
+# using its hosted Sphinx inventory.
+
+import sys
+import sphobjinv as soi
+
+if __name__ == "__main__":
+    vers = sys.argv[1]
+    inv = soi.Inventory(url=f"https://docs.python.org/{vers}/objects.inv")
+    modules = list(sorted(obj.name for obj in inv.objects if obj.role == "module"))
+    print("\n".join(modules))

support/walk-modules.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python
+
+import inspect
+import pkgutil
+import sys
+
+SEEN_MODS = set()
+
+
+def walk_pkgutil(mod_name, mod, io):
+    for pkg in pkgutil.walk_packages(mod.__path__, mod_name + "."):
+        if pkg.name in SEEN_MODS:
+            continue
+        else:
+            # We don't recurse here because `walk_packages` takes care of
+            # it for us.
+            SEEN_MODS.add(pkg.name)
+            print(pkg.name, file=io)
+
+
+def walk_naive(mod_name, mod, io):
+    for attr in dir(mod):
+        attr_obj = getattr(mod, attr, None)
+        # Shouldn't happen, but who knows.
+        if attr_obj is None:
+            continue
+
+        # If this member isn't a module, skip it.
+        if not inspect.ismodule(attr_obj):
+            continue
+
+        # To filter "real" submodules from re-exports, we try
+        # and import the submodule by its qualified name.
+        # If the import fails, we know it's a re-exported module.
+        try:
+            submod_name = mod_name + "." + attr
+            __import__(submod_name)
+            walk(submod_name, io)
+        except ImportError:
+            # ...but sometimes we do want to include re-exports, since
+            # they might be things like "accelerator" modules that don't
+            # appear anywhere else.
+            # For example, `_bz2` might appear as a re-export.
+            try:
+                # Again, try and import to avoid module-looking object
+                # that don't actually appear on disk. Experimentally,
+                # there are a few of these (like "TK").
+                __import__(attr)
+                walk(attr, io)
+            except ImportError:
+                continue
+
+
+def walk(mod_name, io):
+    if mod_name in SEEN_MODS:
+        return
+    else:
+        SEEN_MODS.add(mod_name)
+        print(mod_name, file=io)
+
+    # Try and import it.
+    try:
+        mod = __import__(mod_name)
+
+        if hasattr(mod, "__path__"):
+            walk_pkgutil(mod_name, mod, io)
+        else:
+            walk_naive(mod_name, mod, io)
+
+    except ImportError:
+        pass
+
+
+if __name__ == "__main__":
+    output = sys.argv[1]
+
+    with open(output, mode="w") as io:
+        for mod_name in sys.builtin_module_names:
+            walk(mod_name, io)
+
+        if hasattr(sys, "stdlib_module_names"):
+            for mod_name in sys.stdlib_module_names:
+                walk(mod_name, io)
+        else:
+            for mod_name in sys.stdin:
+                walk(mod_name.rstrip("\n"), io)