From 6ced81761612fe33e339a09362042cb65ac388cb Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:33:23 -0400 Subject: [PATCH 1/6] Add no-GIL interpreter support Add `pytest-run-parallel` as dependency, test no-GIL interpreters in CI, and mark Cython module as safe for freethreaded interpreters. --- .github/workflows/test.yml | 6 ++--- .github/workflows/wheel.yml | 3 ++- msgpack/_cmsgpack.pyx | 2 +- msgpack/_packer.pyx | 2 +- msgpack/_unpacker.pyx | 2 +- requirements.txt | 1 + test/test_multithreading.py | 50 +++++++++++++++++++++++++++++++++++++ 7 files changed, 59 insertions(+), 7 deletions(-) create mode 100644 test/test_multithreading.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 23d221c8..b84611dd 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14-dev", "3.13", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14", "3.14t", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }} @@ -41,12 +41,12 @@ jobs: - name: Test (C extension) shell: bash run: | - pytest -v test + PYTHON_GIL=0 pytest -v test - name: Test (pure Python fallback) shell: bash run: | - MSGPACK_PUREPYTHON=1 pytest -v test + PYTHON_GIL=0 MSGPACK_PUREPYTHON=1 pytest -v test - name: build packages shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index 686d7dd0..aa60b3cb 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,8 +32,9 @@ jobs: uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" - CIBW_TEST_COMMAND: "pytest {package}/test" + CIBW_TEST_COMMAND: "PYTHON_GIL=0 pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" + CIBW_ENABLE: cpython-freerelease - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' diff --git a/msgpack/_cmsgpack.pyx b/msgpack/_cmsgpack.pyx index 1faaac3a..41340ee0 100644 --- a/msgpack/_cmsgpack.pyx +++ b/msgpack/_cmsgpack.pyx @@ -1,5 +1,5 @@ # coding: utf-8 -#cython: embedsignature=True, c_string_encoding=ascii, language_level=3 +#cython: embedsignature=True, c_string_encoding=ascii, language_level=3, freethreading_compatible=True from cpython.datetime cimport import_datetime, datetime_new import_datetime() diff --git a/msgpack/_packer.pyx b/msgpack/_packer.pyx index 402b6946..94d6600b 100644 --- a/msgpack/_packer.pyx +++ b/msgpack/_packer.pyx @@ -1,5 +1,5 @@ # coding: utf-8 - +# cython: freethreading_compatible = True from cpython cimport * from cpython.bytearray cimport PyByteArray_Check, PyByteArray_CheckExact from cpython.datetime cimport ( diff --git a/msgpack/_unpacker.pyx b/msgpack/_unpacker.pyx index 34ff3304..717666d6 100644 --- a/msgpack/_unpacker.pyx +++ b/msgpack/_unpacker.pyx @@ -1,5 +1,5 @@ # coding: utf-8 - +# cython: freethreading_compatible = True from cpython cimport * cdef extern from "Python.h": ctypedef struct PyObject diff --git a/requirements.txt b/requirements.txt index 78a2f38f..5ad4c0f1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ Cython~=3.1.1 +pytest-run-parallel[psutil] diff --git a/test/test_multithreading.py b/test/test_multithreading.py new file mode 100644 index 00000000..911420e1 --- /dev/null +++ b/test/test_multithreading.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +from concurrent.futures import ThreadPoolExecutor +from msgpack import Packer +import threading + + +def run_threaded( + func, + num_threads=8, + pass_count=False, + pass_barrier=False, + outer_iterations=1, + prepare_args=None, +): + """Runs a function many times in parallel""" + for _ in range(outer_iterations): + with ThreadPoolExecutor(max_workers=num_threads) as tpe: + if prepare_args is None: + args = [] + else: + args = prepare_args() + if pass_barrier: + barrier = threading.Barrier(num_threads) + args.append(barrier) + if pass_count: + all_args = [(func, i, *args) for i in range(num_threads)] + else: + all_args = [(func, *args) for i in range(num_threads)] + try: + futures = [] + for arg in all_args: + futures.append(tpe.submit(*arg)) + finally: + if len(futures) < num_threads and pass_barrier: + barrier.abort() + for f in futures: + f.result() + + +def test_multithread_packing(): + output = [] + test_data = "abcd" * 10_000_000 + packer = Packer() + + def closure(b): + data = packer.pack(test_data) + output.append(data) + b.wait() + + run_threaded(closure, num_threads=10, pass_barrier=True, pass_count=False) From 75cdd03f28ec72c049ff41657e5871cac6ac8ac5 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:40:14 -0400 Subject: [PATCH 2/6] Exclude PYTHON_GIL=0 for now --- .github/workflows/test.yml | 2 +- .github/workflows/wheel.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b84611dd..37dfb2af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -41,7 +41,7 @@ jobs: - name: Test (C extension) shell: bash run: | - PYTHON_GIL=0 pytest -v test + pytest -v test - name: Test (pure Python fallback) shell: bash diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index aa60b3cb..febd05e9 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -32,7 +32,7 @@ jobs: uses: pypa/cibuildwheel@v2.23.3 env: CIBW_TEST_REQUIRES: "pytest" - CIBW_TEST_COMMAND: "PYTHON_GIL=0 pytest {package}/test" + CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" CIBW_ENABLE: cpython-freerelease From cafe4f0985f3712882d26ca45bddf8c49f6adca7 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Sun, 20 Jul 2025 11:41:57 -0400 Subject: [PATCH 3/6] Lint --- test/test_buffer.py | 4 ++-- test/test_multithreading.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/test/test_buffer.py b/test/test_buffer.py index 2c5a14c5..ca097222 100644 --- a/test/test_buffer.py +++ b/test/test_buffer.py @@ -17,7 +17,7 @@ def test_unpack_bytearray(): obj = unpackb(buf, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_unpack_memoryview(): @@ -26,7 +26,7 @@ def test_unpack_memoryview(): obj = unpackb(view, use_list=1) assert [b"foo", b"bar"] == obj expected_type = bytes - assert all(type(s) == expected_type for s in obj) + assert all(type(s) is expected_type for s in obj) def test_packer_getbuffer(): diff --git a/test/test_multithreading.py b/test/test_multithreading.py index 911420e1..6694fdc6 100644 --- a/test/test_multithreading.py +++ b/test/test_multithreading.py @@ -1,7 +1,8 @@ #!/usr/bin/env python3 +import threading from concurrent.futures import ThreadPoolExecutor + from msgpack import Packer -import threading def run_threaded( From 3cc2a384d5c525be7845de9391af0896be2de9bf Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Fri, 25 Jul 2025 14:50:49 +0000 Subject: [PATCH 4/6] Exclude PYTHON_GIL for now --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 37dfb2af..e9c25a26 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -46,7 +46,7 @@ jobs: - name: Test (pure Python fallback) shell: bash run: | - PYTHON_GIL=0 MSGPACK_PUREPYTHON=1 pytest -v test + MSGPACK_PUREPYTHON=1 pytest -v test - name: build packages shell: bash From 61aa23fa0b6d1a496bdb3c9094932f77852a88de Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 6 Aug 2025 21:44:32 +0000 Subject: [PATCH 5/6] Update .github/workflows/wheel.yml Co-authored-by: Lysandros Nikolaou --- .github/workflows/wheel.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/wheel.yml b/.github/workflows/wheel.yml index febd05e9..c8b8b520 100644 --- a/.github/workflows/wheel.yml +++ b/.github/workflows/wheel.yml @@ -34,7 +34,7 @@ jobs: CIBW_TEST_REQUIRES: "pytest" CIBW_TEST_COMMAND: "pytest {package}/test" CIBW_SKIP: "pp* cp38-macosx_*" - CIBW_ENABLE: cpython-freerelease + CIBW_ENABLE: cpython-freethreading - name: Build sdist if: runner.os == 'Linux' && runner.arch == 'X64' From d965025f2e0dee80908f0f09ad8e2fba97730e53 Mon Sep 17 00:00:00 2001 From: Charlie Lin Date: Wed, 6 Aug 2025 21:50:26 +0000 Subject: [PATCH 6/6] Fix extremely silly typo --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 50babbf2..989cb42a 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: os: ["ubuntu-latest", "windows-latest", "macos-latest"] - py: ["3.14=dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] + py: ["3.14-dev", "3.14t-dev", "3.13", "3.13t", "3.12", "3.11", "3.10", "3.9", "3.8"] runs-on: ${{ matrix.os }} name: Run test with Python ${{ matrix.py }} on ${{ matrix.os }}