From 2273bcee229ce898f34f2d21f55ba4afbf306a6e Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 00:32:11 +0100 Subject: [PATCH 1/9] Convert zstd __new__ methods to Argument Clinic --- Modules/_zstd/clinic/compressor.c.h | 27 +++++++++++++++++++++++++-- Modules/_zstd/clinic/decompressor.c.h | 27 +++++++++++++++++++++++++-- Modules/_zstd/clinic/zstddict.c.h | 27 +++++++++++++++++++++++++-- Modules/_zstd/compressor.c | 11 ++++++++--- Modules/_zstd/decompressor.c | 11 ++++++++--- Modules/_zstd/zstddict.c | 11 ++++++++--- 6 files changed, 99 insertions(+), 15 deletions(-) diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index d7909cdf89fcd1..9c31c4959e0854 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -6,7 +6,30 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +static PyObject * +_zstd_ZstdCompressor_new_impl(PyTypeObject *type); + +static PyObject * +_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &zstd_compressor_type_spec; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("ZstdCompressor", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("ZstdCompressor", kwargs)) { + goto exit; + } + return_value = _zstd_ZstdCompressor_new_impl(type); + +exit: + return return_value; +} PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, "ZstdCompressor(level=None, options=None, zstd_dict=None)\n" @@ -252,4 +275,4 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=ef69eab155be39f6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e893afb075d1b7c6 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 9359c637203f8f..2d95355c9c690a 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -8,7 +8,30 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +static PyObject * +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type); + +static PyObject * +_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &zstd_decompressor_type_spec; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("ZstdDecompressor", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("ZstdDecompressor", kwargs)) { + goto exit; + } + return_value = _zstd_ZstdDecompressor_new_impl(type); + +exit: + return return_value; +} PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, "ZstdDecompressor(zstd_dict=None, options=None)\n" @@ -227,4 +250,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=ae703f0465a2906d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bb79556ce103345f input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 4e0f7b64172a74..dc5bbbfe892372 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -7,7 +7,30 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() -#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() +#include "pycore_modsupport.h" // _PyArg_NoKeywords() + +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type); + +static PyObject * +_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) +{ + PyObject *return_value = NULL; + PyTypeObject *base_tp = &zstd_dict_type_spec; + + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoPositional("ZstdDict", args)) { + goto exit; + } + if ((type == base_tp || type->tp_init == base_tp->tp_init) && + !_PyArg_NoKeywords("ZstdDict", kwargs)) { + goto exit; + } + return_value = _zstd_ZstdDict_new_impl(type); + +exit: + return return_value; +} PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, "ZstdDict(dict_content, is_raw=False)\n" @@ -204,4 +227,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=59257c053f74eda7 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=02d46b3330c7f72a input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 355a27d2734a1b..ba9fcfd6dbc50e 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -318,11 +318,16 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) return 0; } +/*[clinic input] +@classmethod +_zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdCompressor_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=d1f5f14b617789e4 input=9a6b2592cd79b822]*/ { - ZstdCompressor *self; - self = PyObject_GC_New(ZstdCompressor, type); + ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index dc3e90a22d8dbb..915ead75f53ad3 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -530,11 +530,16 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length } +/*[clinic input] +@classmethod +_zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwds) +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=4987162efa80a1ea input=b7a70c8bea6b451f]*/ { - ZstdDecompressor *self; - self = PyObject_GC_New(ZstdDecompressor, type); + ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 47bc8a84ca24a7..ac0303c0a85606 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -25,11 +25,16 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" #define ZstdDict_CAST(op) ((ZstdDict *)op) +/*[clinic input] +@classmethod +_zstd.ZstdDict.__new__ as _zstd_ZstdDict_new +[clinic start generated code]*/ + static PyObject * -_zstd_ZstdDict_new(PyTypeObject *type, PyObject *Py_UNUSED(args), PyObject *Py_UNUSED(kwargs)) +_zstd_ZstdDict_new_impl(PyTypeObject *type) +/*[clinic end generated code: output=6ae0c050f7046b51 input=de39a2a410c79d2b]*/ { - ZstdDict *self; - self = PyObject_GC_New(ZstdDict, type); + ZstdDict* self = PyObject_GC_New(ZstdDict, type); if (self == NULL) { goto error; } From 2ac18850182217f7dcb35457158f9db1cc3ae378 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 00:53:39 +0100 Subject: [PATCH 2/9] Merge _zstd_ZstdDict___init__ into _zstd_ZstdDict_new --- Modules/_zstd/clinic/zstddict.c.h | 43 +++-------- Modules/_zstd/zstddict.c | 114 ++++++++++++------------------ Modules/_zstd/zstddict.h | 3 - 3 files changed, 57 insertions(+), 103 deletions(-) diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index dc5bbbfe892372..ea15b07f9f80df 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -7,32 +7,9 @@ preserve # include "pycore_runtime.h" // _Py_ID() #endif #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() -#include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -static PyObject * -_zstd_ZstdDict_new_impl(PyTypeObject *type); - -static PyObject * -_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - PyTypeObject *base_tp = &zstd_dict_type_spec; - - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoPositional("ZstdDict", args)) { - goto exit; - } - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("ZstdDict", kwargs)) { - goto exit; - } - return_value = _zstd_ZstdDict_new_impl(type); - -exit: - return return_value; -} - -PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, +PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, "ZstdDict(dict_content, is_raw=False)\n" "--\n" "\n" @@ -50,14 +27,14 @@ PyDoc_STRVAR(_zstd_ZstdDict___init____doc__, "It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" "ZstdDecompressor objects."); -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw); +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, + int is_raw); -static int -_zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 2 @@ -106,7 +83,7 @@ _zstd_ZstdDict___init__(PyObject *self, PyObject *args, PyObject *kwargs) goto exit; } skip_optional_pos: - return_value = _zstd_ZstdDict___init___impl((ZstdDict *)self, dict_content, is_raw); + return_value = _zstd_ZstdDict_new_impl(type, dict_content, is_raw); exit: return return_value; @@ -227,4 +204,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=02d46b3330c7f72a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=af0e16983f15c065 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index ac0303c0a85606..8b8c85e5ac90a5 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -28,58 +28,6 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" /*[clinic input] @classmethod _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new -[clinic start generated code]*/ - -static PyObject * -_zstd_ZstdDict_new_impl(PyTypeObject *type) -/*[clinic end generated code: output=6ae0c050f7046b51 input=de39a2a410c79d2b]*/ -{ - ZstdDict* self = PyObject_GC_New(ZstdDict, type); - if (self == NULL) { - goto error; - } - - self->dict_content = NULL; - self->initialized = 0; - self->d_dict = NULL; - - /* ZSTD_CDict dict */ - self->c_dicts = PyDict_New(); - if (self->c_dicts == NULL) { - goto error; - } - - return (PyObject*)self; - -error: - if (self != NULL) { - PyObject_GC_Del(self); - } - return NULL; -} - -static void -ZstdDict_dealloc(PyObject *ob) -{ - ZstdDict *self = ZstdDict_CAST(ob); - - PyObject_GC_UnTrack(self); - - /* Free ZSTD_DDict instance */ - ZSTD_freeDDict(self->d_dict); - - /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ - Py_CLEAR(self->dict_content); - Py_CLEAR(self->c_dicts); - - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_Del(ob); - Py_DECREF(tp); -} - -/*[clinic input] -_zstd.ZstdDict.__init__ - dict_content: object A bytes-like object, dictionary's content. is_raw: bool = False @@ -95,24 +43,31 @@ It's thread-safe, and can be shared by multiple ZstdCompressor / ZstdDecompressor objects. [clinic start generated code]*/ -static int -_zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, - int is_raw) -/*[clinic end generated code: output=c5f5a0d8377d037c input=e6750f62a513b3ee]*/ +static PyObject * +_zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, + int is_raw) +/*[clinic end generated code: output=3ebff839cb3be6d7 input=a0356fa7336dea1c]*/ { - /* Only called once */ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; + ZstdDict* self = PyObject_GC_New(ZstdDict, type); + if (self == NULL) { + goto error; + } + + self->dict_content = NULL; + self->d_dict = NULL; + + /* ZSTD_CDict dict */ + self->c_dicts = PyDict_New(); + if (self->c_dicts == NULL) { + goto error; } - self->initialized = 1; /* Check dict_content's type */ self->dict_content = PyBytes_FromObject(dict_content); if (self->dict_content == NULL) { PyErr_SetString(PyExc_TypeError, "dict_content argument should be bytes-like object."); - return -1; + goto error; } /* Both ordinary dictionary and "raw content" dictionary should @@ -120,7 +75,7 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, if (Py_SIZE(self->dict_content) < 8) { PyErr_SetString(PyExc_ValueError, "Zstd dictionary content should at least 8 bytes."); - return -1; + goto error; } /* Get dict_id, 0 means "raw content" dictionary. */ @@ -136,12 +91,38 @@ _zstd_ZstdDict___init___impl(ZstdDict *self, PyObject *dict_content, "dict_content argument is a \"raw content\" zstd " "dictionary, set is_raw parameter to True."; PyErr_SetString(PyExc_ValueError, msg); - return -1; + goto error; } // Can only track self once self->dict_content is included PyObject_GC_Track(self); - return 0; + + return (PyObject*)self; + +error: + if (self != NULL) { + PyObject_GC_Del(self); + } + return NULL; +} + +static void +ZstdDict_dealloc(PyObject *ob) +{ + ZstdDict *self = ZstdDict_CAST(ob); + + PyObject_GC_UnTrack(self); + + /* Free ZSTD_DDict instance */ + ZSTD_freeDDict(self->d_dict); + + /* Release dict_content after Free ZSTD_CDict/ZSTD_DDict instances */ + Py_CLEAR(self->dict_content); + Py_CLEAR(self->c_dicts); + + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_Del(ob); + Py_DECREF(tp); } PyDoc_STRVAR(ZstdDict_dictid_doc, @@ -271,9 +252,8 @@ static PyType_Slot zstddict_slots[] = { {Py_tp_getset, ZstdDict_getset}, {Py_tp_new, _zstd_ZstdDict_new}, {Py_tp_dealloc, ZstdDict_dealloc}, - {Py_tp_init, _zstd_ZstdDict___init__}, {Py_tp_str, ZstdDict_str}, - {Py_tp_doc, (char*)_zstd_ZstdDict___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDict_new__doc__}, {Py_sq_length, ZstdDict_length}, {Py_tp_traverse, ZstdDict_traverse}, {Py_tp_clear, ZstdDict_clear}, diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index e82498f5dd1901..c27a17936f89d2 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -23,9 +23,6 @@ typedef struct { PyObject *dict_content; /* Dictionary id */ uint32_t dict_id; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdDict; #endif // !ZSTD_DICT_H From 0ca7db8e17a2eb5cd79bba54a06e77857ea15a8c Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 00:59:16 +0100 Subject: [PATCH 3/9] Merge _zstd_ZstdDecompressor___init__ into _zstd_ZstdDecompressor_new --- Modules/_zstd/clinic/decompressor.c.h | 43 +++---------- Modules/_zstd/decompressor.c | 92 ++++++++++----------------- 2 files changed, 45 insertions(+), 90 deletions(-) diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index 2d95355c9c690a..ebd26c5920f01c 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -8,32 +8,9 @@ preserve #endif #include "pycore_abstract.h" // _PyNumber_Index() #include "pycore_critical_section.h"// Py_BEGIN_CRITICAL_SECTION() -#include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -static PyObject * -_zstd_ZstdDecompressor_new_impl(PyTypeObject *type); - -static PyObject * -_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - PyTypeObject *base_tp = &zstd_decompressor_type_spec; - - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoPositional("ZstdDecompressor", args)) { - goto exit; - } - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("ZstdDecompressor", kwargs)) { - goto exit; - } - return_value = _zstd_ZstdDecompressor_new_impl(type); - -exit: - return return_value; -} - -PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, "ZstdDecompressor(zstd_dict=None, options=None)\n" "--\n" "\n" @@ -47,14 +24,14 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor___init____doc__, "Thread-safe at method level. For one-shot decompression, use the decompress()\n" "function instead."); -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options); +static PyObject * +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options); -static int -_zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdDecompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 2 @@ -105,7 +82,7 @@ _zstd_ZstdDecompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs } options = fastargs[1]; skip_optional_pos: - return_value = _zstd_ZstdDecompressor___init___impl((ZstdDecompressor *)self, zstd_dict, options); + return_value = _zstd_ZstdDecompressor_new_impl(type, zstd_dict, options); exit: return return_value; @@ -250,4 +227,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=bb79556ce103345f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=219d26afb3452705 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 915ead75f53ad3..202c1d9b22ae72 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -48,9 +48,6 @@ typedef struct { /* For ZstdDecompressor, 0 or 1. 1 means the end of the first frame has been reached. */ bool eof; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdDecompressor; #define ZstdDecompressor_CAST(op) ((ZstdDecompressor *)op) @@ -533,18 +530,28 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length /*[clinic input] @classmethod _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new + + zstd_dict: object = None + A ZstdDict object, a pre-trained zstd dictionary. + options: object = None + A dict object that contains advanced decompression parameters. + +Create a decompressor object for decompressing data incrementally. + +Thread-safe at method level. For one-shot decompression, use the decompress() +function instead. [clinic start generated code]*/ static PyObject * -_zstd_ZstdDecompressor_new_impl(PyTypeObject *type) -/*[clinic end generated code: output=4987162efa80a1ea input=b7a70c8bea6b451f]*/ +_zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, + PyObject *options) +/*[clinic end generated code: output=590ca65c1102ff4a input=e73db62a54e25e4b]*/ { ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } - self->initialized = 0; self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; @@ -567,6 +574,27 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type) goto error; } + /* Load dictionary to decompression context */ + if (zstd_dict != Py_None) { + if (_zstd_load_d_dict(self, zstd_dict) < 0) { + goto error; + } + + /* Py_INCREF the dict */ + Py_INCREF(zstd_dict); + self->dict = zstd_dict; + } + + /* Set option to decompression context */ + if (options != Py_None) { + if (_zstd_set_d_parameters(self, options) < 0) { + goto error; + } + } + + // We can only start tracking self with the GC once self->dict is set. + PyObject_GC_Track(self); + return (PyObject*)self; error: @@ -600,55 +628,6 @@ ZstdDecompressor_dealloc(PyObject *ob) Py_DECREF(tp); } -/*[clinic input] -_zstd.ZstdDecompressor.__init__ - - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - options: object = None - A dict object that contains advanced decompression parameters. - -Create a decompressor object for decompressing data incrementally. - -Thread-safe at method level. For one-shot decompression, use the decompress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdDecompressor___init___impl(ZstdDecompressor *self, - PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=703af2f1ec226642 input=8fd72999acc1a146]*/ -{ - /* Only called once */ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; - } - self->initialized = 1; - - /* Load dictionary to decompression context */ - if (zstd_dict != Py_None) { - if (_zstd_load_d_dict(self, zstd_dict) < 0) { - return -1; - } - - /* Py_INCREF the dict */ - Py_INCREF(zstd_dict); - self->dict = zstd_dict; - } - - /* Set option to decompression context */ - if (options != Py_None) { - if (_zstd_set_d_parameters(self, options) < 0) { - return -1; - } - } - - // We can only start tracking self with the GC once self->dict is set. - PyObject_GC_Track(self); - return 0; -} - /*[clinic input] @critical_section @getter @@ -774,11 +753,10 @@ ZstdDecompressor_clear(PyObject *ob) static PyType_Slot ZstdDecompressor_slots[] = { {Py_tp_new, _zstd_ZstdDecompressor_new}, {Py_tp_dealloc, ZstdDecompressor_dealloc}, - {Py_tp_init, _zstd_ZstdDecompressor___init__}, {Py_tp_methods, ZstdDecompressor_methods}, {Py_tp_members, ZstdDecompressor_members}, {Py_tp_getset, ZstdDecompressor_getset}, - {Py_tp_doc, (char*)_zstd_ZstdDecompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdDecompressor_new__doc__}, {Py_tp_traverse, ZstdDecompressor_traverse}, {Py_tp_clear, ZstdDecompressor_clear}, {0, 0} From 98ee61ae4f59af9ea526305d81f56b7d2957de6f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 01:01:29 +0100 Subject: [PATCH 4/9] Merge _zstd_ZstdCompressor___init__ into _zstd_ZstdCompressor_new --- Modules/_zstd/clinic/compressor.c.h | 43 +++-------- Modules/_zstd/compressor.c | 113 +++++++++++----------------- 2 files changed, 56 insertions(+), 100 deletions(-) diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index 9c31c4959e0854..27734db2547908 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -6,32 +6,9 @@ preserve # include "pycore_gc.h" // PyGC_Head # include "pycore_runtime.h" // _Py_ID() #endif -#include "pycore_modsupport.h" // _PyArg_NoKeywords() +#include "pycore_modsupport.h" // _PyArg_UnpackKeywords() -static PyObject * -_zstd_ZstdCompressor_new_impl(PyTypeObject *type); - -static PyObject * -_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) -{ - PyObject *return_value = NULL; - PyTypeObject *base_tp = &zstd_compressor_type_spec; - - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoPositional("ZstdCompressor", args)) { - goto exit; - } - if ((type == base_tp || type->tp_init == base_tp->tp_init) && - !_PyArg_NoKeywords("ZstdCompressor", kwargs)) { - goto exit; - } - return_value = _zstd_ZstdCompressor_new_impl(type); - -exit: - return return_value; -} - -PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, +PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, "ZstdCompressor(level=None, options=None, zstd_dict=None)\n" "--\n" "\n" @@ -47,14 +24,14 @@ PyDoc_STRVAR(_zstd_ZstdCompressor___init____doc__, "Thread-safe at method level. For one-shot compression, use the compress()\n" "function instead."); -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict); +static PyObject * +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict); -static int -_zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) +static PyObject * +_zstd_ZstdCompressor_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) { - int return_value = -1; + PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) #define NUM_KEYWORDS 3 @@ -112,7 +89,7 @@ _zstd_ZstdCompressor___init__(PyObject *self, PyObject *args, PyObject *kwargs) } zstd_dict = fastargs[2]; skip_optional_pos: - return_value = _zstd_ZstdCompressor___init___impl((ZstdCompressor *)self, level, options, zstd_dict); + return_value = _zstd_ZstdCompressor_new_impl(type, level, options, zstd_dict); exit: return return_value; @@ -275,4 +252,4 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=e893afb075d1b7c6 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f30de86bbd38a79a input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index ba9fcfd6dbc50e..36e44db76e7bac 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -42,9 +42,6 @@ typedef struct { /* Compression level */ int compression_level; - - /* __init__ has been called, 0 or 1. */ - bool initialized; } ZstdCompressor; #define ZstdCompressor_CAST(op) ((ZstdCompressor *)op) @@ -321,18 +318,30 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) /*[clinic input] @classmethod _zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new + + level: object = None + The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. + options: object = None + A dict object that contains advanced compression parameters. + zstd_dict: object = None + A ZstdDict object, a pre-trained zstd dictionary. + +Create a compressor object for compressing data incrementally. + +Thread-safe at method level. For one-shot compression, use the compress() +function instead. [clinic start generated code]*/ static PyObject * -_zstd_ZstdCompressor_new_impl(PyTypeObject *type) -/*[clinic end generated code: output=d1f5f14b617789e4 input=9a6b2592cd79b822]*/ +_zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, + PyObject *options, PyObject *zstd_dict) +/*[clinic end generated code: output=cdef61eafecac3d7 input=a9e9d73f246d6588]*/ { ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } - self->initialized = 0; self->dict = NULL; self->use_multithread = 0; @@ -351,82 +360,28 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type) /* Last mode */ self->last_mode = ZSTD_e_end; - return (PyObject*)self; - -error: - if (self != NULL) { - PyObject_GC_Del(self); - } - return NULL; -} - -static void -ZstdCompressor_dealloc(PyObject *ob) -{ - ZstdCompressor *self = ZstdCompressor_CAST(ob); - - PyObject_GC_UnTrack(self); - - /* Free compression context */ - ZSTD_freeCCtx(self->cctx); - - /* Py_XDECREF the dict after free the compression context */ - Py_CLEAR(self->dict); - - PyTypeObject *tp = Py_TYPE(self); - PyObject_GC_Del(ob); - Py_DECREF(tp); -} - -/*[clinic input] -_zstd.ZstdCompressor.__init__ - - level: object = None - The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. - options: object = None - A dict object that contains advanced compression parameters. - zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. - -Create a compressor object for compressing data incrementally. - -Thread-safe at method level. For one-shot compression, use the compress() -function instead. -[clinic start generated code]*/ - -static int -_zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, - PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=215e6c4342732f96 input=9f79b0d8d34c8ef0]*/ -{ - if (self->initialized) { - PyErr_SetString(PyExc_RuntimeError, "reinitialization not supported"); - return -1; - } - self->initialized = 1; - if (level != Py_None && options != Py_None) { PyErr_SetString(PyExc_RuntimeError, "Only one of level or options should be used."); - return -1; + goto error; } /* Set compressLevel/options to compression context */ if (level != Py_None) { if (_zstd_set_c_parameters(self, level, "level", "int") < 0) { - return -1; + goto error; } } if (options != Py_None) { if (_zstd_set_c_parameters(self, options, "options", "dict") < 0) { - return -1; + goto error; } } /* Load dictionary to compression context */ if (zstd_dict != Py_None) { if (_zstd_load_c_dict(self, zstd_dict) < 0) { - return -1; + goto error; } /* Py_INCREF the dict */ @@ -436,7 +391,32 @@ _zstd_ZstdCompressor___init___impl(ZstdCompressor *self, PyObject *level, // We can only start tracking self with the GC once self->dict is set. PyObject_GC_Track(self); - return 0; + + return (PyObject*)self; + +error: + if (self != NULL) { + PyObject_GC_Del(self); + } + return NULL; +} + +static void +ZstdCompressor_dealloc(PyObject *ob) +{ + ZstdCompressor *self = ZstdCompressor_CAST(ob); + + PyObject_GC_UnTrack(self); + + /* Free compression context */ + ZSTD_freeCCtx(self->cctx); + + /* Py_XDECREF the dict after free the compression context */ + Py_CLEAR(self->dict); + + PyTypeObject *tp = Py_TYPE(self); + PyObject_GC_Del(ob); + Py_DECREF(tp); } static PyObject * @@ -722,10 +702,9 @@ ZstdCompressor_clear(PyObject *ob) static PyType_Slot zstdcompressor_slots[] = { {Py_tp_new, _zstd_ZstdCompressor_new}, {Py_tp_dealloc, ZstdCompressor_dealloc}, - {Py_tp_init, _zstd_ZstdCompressor___init__}, {Py_tp_methods, ZstdCompressor_methods}, {Py_tp_members, ZstdCompressor_members}, - {Py_tp_doc, (char*)_zstd_ZstdCompressor___init____doc__}, + {Py_tp_doc, (void *)_zstd_ZstdCompressor_new__doc__}, {Py_tp_traverse, ZstdCompressor_traverse}, {Py_tp_clear, ZstdCompressor_clear}, {0, 0} From 15fc9cee2d76a73bb4fc1f5f2ae4b1698e10235f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 01:24:04 +0100 Subject: [PATCH 5/9] post-merge tweaks --- Lib/test/test_zstd.py | 12 ++++++++---- Modules/_zstd/clinic/compressor.c.h | 4 ++-- Modules/_zstd/clinic/zstddict.c.h | 26 ++++++++++++-------------- Modules/_zstd/compressor.c | 14 +++++--------- Modules/_zstd/decompressor.c | 11 ++++------- Modules/_zstd/zstddict.c | 19 ++++++++----------- 6 files changed, 39 insertions(+), 47 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index 713294c4c27685..d255317e7412d3 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -507,7 +507,7 @@ def test_decompress_epilogue_flags(self): self.assertFalse(d.needs_input) def test_decompressor_arg(self): - zd = ZstdDict(b'12345678', True) + zd = ZstdDict(b'12345678', is_raw=True) with self.assertRaises(TypeError): d = ZstdDecompressor(zstd_dict={}) @@ -1021,6 +1021,10 @@ def test_decompressor_skippable(self): class ZstdDictTestCase(unittest.TestCase): def test_is_raw(self): + # must be passed as a keyword argument + with self.assertRaises(TypeError): + ZstdDict(bytes(8), True) + # content < 8 b = b'1234567' with self.assertRaises(ValueError): @@ -1096,7 +1100,7 @@ def test_train_dict(self): TRAINED_DICT = train_dict(SAMPLES, DICT_SIZE1) - ZstdDict(TRAINED_DICT.dict_content, False) + ZstdDict(TRAINED_DICT.dict_content, is_raw=False) self.assertNotEqual(TRAINED_DICT.dict_id, 0) self.assertGreater(len(TRAINED_DICT.dict_content), 0) @@ -1250,7 +1254,7 @@ def _nbytes(dat): def test_as_prefix(self): # V1 V1 = THIS_FILE_BYTES - zd = ZstdDict(V1, True) + zd = ZstdDict(V1, is_raw=True) # V2 mid = len(V1) // 2 @@ -1266,7 +1270,7 @@ def test_as_prefix(self): self.assertEqual(decompress(dat, zd.as_prefix), V2) # use wrong prefix - zd2 = ZstdDict(SAMPLES[0], True) + zd2 = ZstdDict(SAMPLES[0], is_raw=True) try: decompressed = decompress(dat, zd2.as_prefix) except ZstdError: # expected diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index 27734db2547908..e38ad378427ba1 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -15,7 +15,7 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, "Create a compressor object for compressing data incrementally.\n" "\n" " level\n" -" The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT.\n" +" The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT.\n" " options\n" " A dict object that contains advanced compression parameters.\n" " zstd_dict\n" @@ -252,4 +252,4 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=f30de86bbd38a79a input=a9049054013a1b77]*/ +/*[clinic end generated code: output=c33f03bf68e679e5 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index ea15b07f9f80df..222c6965117f4f 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -10,19 +10,17 @@ preserve #include "pycore_modsupport.h" // _PyArg_UnpackKeywords() PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, -"ZstdDict(dict_content, is_raw=False)\n" +"ZstdDict(dict_content, /, *, is_raw=False)\n" "--\n" "\n" "Represents a zstd dictionary, which can be used for compression/decompression.\n" "\n" " dict_content\n" -" A bytes-like object, dictionary\'s content.\n" +" The content of a Zstandard dictionary as a bytes-like object.\n" " is_raw\n" -" This parameter is for advanced user. True means dict_content\n" -" argument is a \"raw content\" dictionary, free of any format\n" -" restriction. False means dict_content argument is an ordinary\n" -" zstd dictionary, was created by zstd functions, follow a\n" -" specified format.\n" +" If true, perform no checks on *dict_content*, useful for some\n" +" advanced cases. Otherwise, check that the content represents\n" +" a Zstandard dictionary created by the zstd functions.\n" "\n" "It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" "ZstdDecompressor objects."); @@ -37,7 +35,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) PyObject *return_value = NULL; #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - #define NUM_KEYWORDS 2 + #define NUM_KEYWORDS 1 static struct { PyGC_Head _this_is_not_used; PyObject_VAR_HEAD @@ -46,7 +44,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) } _kwtuple = { .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) .ob_hash = -1, - .ob_item = { &_Py_ID(dict_content), &_Py_ID(is_raw), }, + .ob_item = { &_Py_ID(is_raw), }, }; #undef NUM_KEYWORDS #define KWTUPLE (&_kwtuple.ob_base.ob_base) @@ -55,7 +53,7 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) # define KWTUPLE NULL #endif // !Py_BUILD_CORE - static const char * const _keywords[] = {"dict_content", "is_raw", NULL}; + static const char * const _keywords[] = {"", "is_raw", NULL}; static _PyArg_Parser _parser = { .keywords = _keywords, .fname = "ZstdDict", @@ -70,19 +68,19 @@ _zstd_ZstdDict_new(PyTypeObject *type, PyObject *args, PyObject *kwargs) int is_raw = 0; fastargs = _PyArg_UnpackKeywords(_PyTuple_CAST(args)->ob_item, nargs, kwargs, NULL, &_parser, - /*minpos*/ 1, /*maxpos*/ 2, /*minkw*/ 0, /*varpos*/ 0, argsbuf); + /*minpos*/ 1, /*maxpos*/ 1, /*minkw*/ 0, /*varpos*/ 0, argsbuf); if (!fastargs) { goto exit; } dict_content = fastargs[0]; if (!noptargs) { - goto skip_optional_pos; + goto skip_optional_kwonly; } is_raw = PyObject_IsTrue(fastargs[1]); if (is_raw < 0) { goto exit; } -skip_optional_pos: +skip_optional_kwonly: return_value = _zstd_ZstdDict_new_impl(type, dict_content, is_raw); exit: @@ -204,4 +202,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=af0e16983f15c065 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f301594e7b37f709 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 36e44db76e7bac..4b8bc4a11d8c22 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -318,9 +318,8 @@ _zstd_load_c_dict(ZstdCompressor *self, PyObject *dict) /*[clinic input] @classmethod _zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new - level: object = None - The compression level to use, defaults to ZSTD_CLEVEL_DEFAULT. + The compression level to use. Defaults to COMPRESSION_LEVEL_DEFAULT. options: object = None A dict object that contains advanced compression parameters. zstd_dict: object = None @@ -335,17 +334,15 @@ function instead. static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=cdef61eafecac3d7 input=a9e9d73f246d6588]*/ +/*[clinic end generated code: output=cdef61eafecac3d7 input=b9ea61ecafbb1b1e]*/ { ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { goto error; } - self->dict = NULL; self->use_multithread = 0; - /* Compression context */ self->cctx = ZSTD_createCCtx(); if (self->cctx == NULL) { @@ -378,18 +375,17 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, } } - /* Load dictionary to compression context */ + /* Load zstd dictionary to compression context */ + self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_c_dict(self, zstd_dict) < 0) { goto error; } - - /* Py_INCREF the dict */ Py_INCREF(zstd_dict); self->dict = zstd_dict; } - // We can only start tracking self with the GC once self->dict is set. + // We can only start GC tracking once self->dict is set. PyObject_GC_Track(self); return (PyObject*)self; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 202c1d9b22ae72..ce07baac27763c 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -530,7 +530,6 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length /*[clinic input] @classmethod _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new - zstd_dict: object = None A ZstdDict object, a pre-trained zstd dictionary. options: object = None @@ -545,14 +544,13 @@ function instead. static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=590ca65c1102ff4a input=e73db62a54e25e4b]*/ +/*[clinic end generated code: output=590ca65c1102ff4a input=0f161edb33d83216]*/ { ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { goto error; } - self->dict = NULL; self->input_buffer = NULL; self->input_buffer_size = 0; self->in_begin = -1; @@ -574,13 +572,12 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, goto error; } - /* Load dictionary to decompression context */ + /* Load zstd dictionary to decompression context */ + self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_d_dict(self, zstd_dict) < 0) { goto error; } - - /* Py_INCREF the dict */ Py_INCREF(zstd_dict); self->dict = zstd_dict; } @@ -592,7 +589,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, } } - // We can only start tracking self with the GC once self->dict is set. + // We can only start GC tracking once self->dict is set. PyObject_GC_Track(self); return (PyObject*)self; diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 8b8c85e5ac90a5..35ea2a0a27dd16 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -29,13 +29,13 @@ class _zstd.ZstdDict "ZstdDict *" "&zstd_dict_type_spec" @classmethod _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new dict_content: object - A bytes-like object, dictionary's content. + The content of a Zstandard dictionary as a bytes-like object. + / + * is_raw: bool = False - This parameter is for advanced user. True means dict_content - argument is a "raw content" dictionary, free of any format - restriction. False means dict_content argument is an ordinary - zstd dictionary, was created by zstd functions, follow a - specified format. + If true, perform no checks on *dict_content*, useful for some + advanced cases. Otherwise, check that the content represents + a Zstandard dictionary created by the zstd functions. Represents a zstd dictionary, which can be used for compression/decompression. @@ -46,7 +46,7 @@ ZstdDecompressor objects. static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, int is_raw) -/*[clinic end generated code: output=3ebff839cb3be6d7 input=a0356fa7336dea1c]*/ +/*[clinic end generated code: output=3ebff839cb3be6d7 input=e9e22fdc68fa04cc]*/ { ZstdDict* self = PyObject_GC_New(ZstdDict, type); if (self == NULL) { @@ -86,10 +86,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, if (!is_raw && self->dict_id == 0) { char *msg = "The dict_content argument is not a valid zstd " "dictionary. The first 4 bytes of a valid zstd dictionary " - "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n" - "If you are an advanced user, and can be sure that " - "dict_content argument is a \"raw content\" zstd " - "dictionary, set is_raw parameter to True."; + "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n"; PyErr_SetString(PyExc_ValueError, msg); goto error; } From 4cc2d4e2111cab9b1855fea854fda79a388edaf6 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 03:38:33 +0100 Subject: [PATCH 6/9] Consistently use Zstandard vs zstd --- Lib/test/test_zstd.py | 4 ++-- Modules/_zstd/_zstdmodule.c | 35 +++++++++++++--------------- Modules/_zstd/_zstdmodule.h | 5 +--- Modules/_zstd/buffer.h | 5 +--- Modules/_zstd/clinic/_zstdmodule.c.h | 6 ++--- Modules/_zstd/compressor.c | 9 +++---- Modules/_zstd/decompressor.c | 11 ++++----- Modules/_zstd/zstddict.c | 22 +++++++---------- Modules/_zstd/zstddict.h | 5 +--- 9 files changed, 39 insertions(+), 63 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index d255317e7412d3..fe6b2231624e61 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -1072,9 +1072,9 @@ def test_invalid_dict(self): # corrupted zd = ZstdDict(dict_content, is_raw=False) - with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_CDict.*?content\.$'): ZstdCompressor(zstd_dict=zd.as_digested_dict) - with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?corrupted'): + with self.assertRaisesRegex(ZstdError, r'ZSTD_DDict.*?content\.$'): ZstdDecompressor(zd) # wrong type diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index c3852fe89732bc..24796604187ec1 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef Py_BUILD_CORE_BUILTIN # define Py_BUILD_CORE_MODULE 1 @@ -34,17 +31,17 @@ set_zstd_error(const _zstd_state* const state, switch (type) { case ERR_DECOMPRESS: - msg = "Unable to decompress zstd data: %s"; + msg = "Unable to decompress Zstandard data: %s"; break; case ERR_COMPRESS: - msg = "Unable to compress zstd data: %s"; + msg = "Unable to compress Zstandard data: %s"; break; case ERR_LOAD_D_DICT: - msg = "Unable to load zstd dictionary or prefix for decompression: %s"; + msg = "Unable to load Zstandard dictionary or prefix for decompression: %s"; break; case ERR_LOAD_C_DICT: - msg = "Unable to load zstd dictionary or prefix for compression: %s"; + msg = "Unable to load Zstandard dictionary or prefix for compression: %s"; break; case ERR_GET_C_BOUNDS: @@ -58,10 +55,10 @@ set_zstd_error(const _zstd_state* const state, break; case ERR_TRAIN_DICT: - msg = "Unable to train zstd dictionary: %s"; + msg = "Unable to train the Zstandard dictionary: %s"; break; case ERR_FINALIZE_DICT: - msg = "Unable to finalize zstd dictionary: %s"; + msg = "Unable to finalize the Zstandard dictionary: %s"; break; default: @@ -187,13 +184,13 @@ _zstd.train_dict The size of the dictionary. / -Internal function, train a zstd dictionary on sample data. +Internal function, train a Zstandard dictionary on sample data. [clinic start generated code]*/ static PyObject * _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=8e87fe43935e8f77 input=70fcd8937f2528b6]*/ +/*[clinic end generated code: output=8e87fe43935e8f77 input=829e31fbbf3454b0]*/ { // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict // are pretty similar. We should see if we can refactor them to share that code. @@ -295,7 +292,7 @@ _zstd.finalize_dict Optimize for a specific zstd compression level, 0 means default. / -Internal function, finalize a zstd dictionary. +Internal function, finalize a Zstandard dictionary. [clinic start generated code]*/ static PyObject * @@ -303,7 +300,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size, int compression_level) -/*[clinic end generated code: output=f91821ba5ae85bda input=130d1508adb55ba1]*/ +/*[clinic end generated code: output=f91821ba5ae85bda input=11af97dcc7608059]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; @@ -457,9 +454,9 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) if (ZSTD_isError(frame_size)) { _zstd_state* const mod_state = get_zstd_state(module); PyErr_Format(mod_state->ZstdError, - "Error when finding the compressed size of a zstd frame. " - "Make sure the frame_buffer argument starts from the " - "beginning of a frame, and its length not less than this " + "Error when finding the compressed size of a Zstandard frame. " + "Ensure the frame_buffer argument starts from the " + "beginning of a frame, and its length is not less than this " "complete frame. Zstd error message: %s.", ZSTD_getErrorName(frame_size)); return NULL; @@ -494,9 +491,9 @@ _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) _zstd_state* const mod_state = get_zstd_state(module); PyErr_SetString(mod_state->ZstdError, "Error when getting information from the header of " - "a zstd frame. Make sure the frame_buffer argument " + "a Zstandard frame. Ensure the frame_buffer argument " "starts from the beginning of a frame, and its length " - "not less than the frame header (6~18 bytes)."); + "is not less than the frame header (6~18 bytes)."); return NULL; } diff --git a/Modules/_zstd/_zstdmodule.h b/Modules/_zstd/_zstdmodule.h index 00e0d2177f31f6..b36486442c6567 100644 --- a/Modules/_zstd/_zstdmodule.h +++ b/Modules/_zstd/_zstdmodule.h @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* Declarations shared between different parts of the _zstd module*/ diff --git a/Modules/_zstd/buffer.h b/Modules/_zstd/buffer.h index c902eef4f8e037..bff3a81d8aa11c 100644 --- a/Modules/_zstd/buffer.h +++ b/Modules/_zstd/buffer.h @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef ZSTD_BUFFER_H #define ZSTD_BUFFER_H diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index fc9f49813df6f1..13c4a53302c64c 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -13,7 +13,7 @@ PyDoc_STRVAR(_zstd_train_dict__doc__, "train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" -"Internal function, train a zstd dictionary on sample data.\n" +"Internal function, train a Zstandard dictionary on sample data.\n" "\n" " samples_bytes\n" " Concatenation of samples.\n" @@ -73,7 +73,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__, " dict_size, compression_level, /)\n" "--\n" "\n" -"Internal function, finalize a zstd dictionary.\n" +"Internal function, finalize a Zstandard dictionary.\n" "\n" " custom_dict_bytes\n" " Custom dictionary content.\n" @@ -428,4 +428,4 @@ _zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=8445b658dcdcbb9c input=a9049054013a1b77]*/ +/*[clinic end generated code: output=553eaf1ea05d38ef input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 4b8bc4a11d8c22..4a2c47d06c813a 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdCompressor class definitions */ @@ -186,8 +183,8 @@ _get_CDict(ZstdDict *self, int compressionLevel) _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_CDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_CDict instance from " + "Zstandard dictionary content."); } goto error; } diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index ce07baac27763c..8ed61e6b96fca8 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdDecompressor class definition */ @@ -78,8 +75,8 @@ _get_DDict(ZstdDict *self) _zstd_state* const mod_state = PyType_GetModuleState(Py_TYPE(self)); if (mod_state != NULL) { PyErr_SetString(mod_state->ZstdError, - "Failed to create ZSTD_DDict instance from zstd " - "dictionary content. Maybe the content is corrupted."); + "Failed to create a ZSTD_DDict instance from " + "Zstandard dictionary content."); } } } @@ -370,7 +367,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length /* Check .eof flag */ if (self->eof) { - PyErr_SetString(PyExc_EOFError, "Already at the end of a zstd frame."); + PyErr_SetString(PyExc_EOFError, "Already at the end of a Zstandard frame."); assert(ret == NULL); return NULL; } diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index 35ea2a0a27dd16..d837d30943cd9b 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ /* ZstdDict class definitions */ @@ -84,9 +81,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, /* Check validity for ordinary dictionary */ if (!is_raw && self->dict_id == 0) { - char *msg = "The dict_content argument is not a valid zstd " - "dictionary. The first 4 bytes of a valid zstd dictionary " - "should be a magic number: b'\\x37\\xA4\\x30\\xEC'.\n"; + char *msg = "Invalid Zstandard dictionary and is_raw not set.\n"; PyErr_SetString(PyExc_ValueError, msg); goto error; } @@ -123,15 +118,14 @@ ZstdDict_dealloc(PyObject *ob) } PyDoc_STRVAR(ZstdDict_dictid_doc, -"ID of zstd dictionary, a 32-bit unsigned int value.\n\n" -"Non-zero means ordinary dictionary, was created by zstd functions, follow\n" -"a specified format.\n\n" -"0 means a \"raw content\" dictionary, free of any format restriction, used\n" -"for advanced user."); +"The ID of Zstandard dictionary, an integer between 0 and 2**32.\n\n" +"A non-zero value represents an ordinary Zstandard dictionary, " +"conforming to the standardised format.\n\n" +"The special value '0' means a 'raw content' dictionary," +"without any restrictions on format or content."); PyDoc_STRVAR(ZstdDict_dictcontent_doc, -"The content of zstd dictionary, a bytes object, it's the same as dict_content\n" -"argument in ZstdDict.__init__() method. It can be used with other programs."); +"The content of a Zstandard dictionary, as a bytes object."); static PyObject * ZstdDict_str(PyObject *ob) diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index c27a17936f89d2..5e29a54202a6ab 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -1,7 +1,4 @@ -/* -Low level interface to Meta's zstd library for use in the compression.zstd -Python module. -*/ +/* Low level interface to the Zstandard algorthm & the zstd library. */ #ifndef ZSTD_DICT_H #define ZSTD_DICT_H From e8aac009203427a5b46a979e5b1944b89f2c6773 Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 11:25:23 +0100 Subject: [PATCH 7/9] regen-all --- Include/internal/pycore_global_objects_fini_generated.h | 1 - Include/internal/pycore_global_strings.h | 1 - Include/internal/pycore_runtime_init_generated.h | 1 - Include/internal/pycore_unicodeobject_generated.h | 4 ---- 4 files changed, 7 deletions(-) diff --git a/Include/internal/pycore_global_objects_fini_generated.h b/Include/internal/pycore_global_objects_fini_generated.h index 54931e8504e8ca..d896e870630418 100644 --- a/Include/internal/pycore_global_objects_fini_generated.h +++ b/Include/internal/pycore_global_objects_fini_generated.h @@ -902,7 +902,6 @@ _PyStaticObjects_CheckRefcnt(PyInterpreterState *interp) { _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(deterministic)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(device)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict)); - _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dict_content)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(dictcomp)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(difference_update)); _PyStaticObject_CheckRefcnt((PyObject *)&_Py_ID(digest)); diff --git a/Include/internal/pycore_global_strings.h b/Include/internal/pycore_global_strings.h index a665195c8996c7..a06d7495bab8e7 100644 --- a/Include/internal/pycore_global_strings.h +++ b/Include/internal/pycore_global_strings.h @@ -393,7 +393,6 @@ struct _Py_global_strings { STRUCT_FOR_ID(deterministic) STRUCT_FOR_ID(device) STRUCT_FOR_ID(dict) - STRUCT_FOR_ID(dict_content) STRUCT_FOR_ID(dictcomp) STRUCT_FOR_ID(difference_update) STRUCT_FOR_ID(digest) diff --git a/Include/internal/pycore_runtime_init_generated.h b/Include/internal/pycore_runtime_init_generated.h index 01289f6118d34f..83301d8aef7697 100644 --- a/Include/internal/pycore_runtime_init_generated.h +++ b/Include/internal/pycore_runtime_init_generated.h @@ -900,7 +900,6 @@ extern "C" { INIT_ID(deterministic), \ INIT_ID(device), \ INIT_ID(dict), \ - INIT_ID(dict_content), \ INIT_ID(dictcomp), \ INIT_ID(difference_update), \ INIT_ID(digest), \ diff --git a/Include/internal/pycore_unicodeobject_generated.h b/Include/internal/pycore_unicodeobject_generated.h index 8ec1ac1e56d1ff..c0f5f2b17f6609 100644 --- a/Include/internal/pycore_unicodeobject_generated.h +++ b/Include/internal/pycore_unicodeobject_generated.h @@ -1360,10 +1360,6 @@ _PyUnicode_InitStaticStrings(PyInterpreterState *interp) { _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); assert(PyUnicode_GET_LENGTH(string) != 1); - string = &_Py_ID(dict_content); - _PyUnicode_InternStatic(interp, &string); - assert(_PyUnicode_CheckConsistency(string, 1)); - assert(PyUnicode_GET_LENGTH(string) != 1); string = &_Py_ID(dictcomp); _PyUnicode_InternStatic(interp, &string); assert(_PyUnicode_CheckConsistency(string, 1)); From fe03b15c26b8a5c1b67e1f6c3c8122167a369cad Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Sun, 11 May 2025 14:41:37 +0100 Subject: [PATCH 8/9] remove now-unneeded include directives --- Modules/_zstd/compressor.c | 1 - Modules/_zstd/zstddict.h | 1 - 2 files changed, 2 deletions(-) diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 4a2c47d06c813a..702d90370d2980 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -18,7 +18,6 @@ class _zstd.ZstdCompressor "ZstdCompressor *" "&zstd_compressor_type_spec" #include "buffer.h" #include "zstddict.h" -#include // bool #include // offsetof() #include // ZSTD_*() diff --git a/Modules/_zstd/zstddict.h b/Modules/_zstd/zstddict.h index 5e29a54202a6ab..e8a55a3670b869 100644 --- a/Modules/_zstd/zstddict.h +++ b/Modules/_zstd/zstddict.h @@ -3,7 +3,6 @@ #ifndef ZSTD_DICT_H #define ZSTD_DICT_H -#include // bool #include // ZSTD_DDict typedef struct { From bb6c416052e164d82e47491156221941d9fa8b1f Mon Sep 17 00:00:00 2001 From: Adam Turner <9087854+aa-turner@users.noreply.github.com> Date: Mon, 12 May 2025 09:19:24 +0100 Subject: [PATCH 9/9] tweak comments --- Lib/test/test_zstd.py | 8 +++--- Modules/_zstd/_zstdmodule.c | 38 +++++++++++++-------------- Modules/_zstd/clinic/_zstdmodule.c.h | 22 +++++++--------- Modules/_zstd/clinic/compressor.c.h | 10 +++---- Modules/_zstd/clinic/decompressor.c.h | 6 ++--- Modules/_zstd/clinic/zstddict.c.h | 10 +++---- Modules/_zstd/compressor.c | 20 +++++++------- Modules/_zstd/decompressor.c | 16 +++++------ Modules/_zstd/zstddict.c | 14 +++++----- 9 files changed, 70 insertions(+), 74 deletions(-) diff --git a/Lib/test/test_zstd.py b/Lib/test/test_zstd.py index fe6b2231624e61..541db4441b035c 100644 --- a/Lib/test/test_zstd.py +++ b/Lib/test/test_zstd.py @@ -288,8 +288,8 @@ def test_unknown_compression_parameter(self): KEY = 100001234 option = {CompressionParameter.compression_level: 10, KEY: 200000000} - pattern = r'Zstd compression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY + pattern = (r'Invalid zstd compression parameter.*?' + fr'"unknown parameter \(key {KEY}\)"') with self.assertRaisesRegex(ZstdError, pattern): ZstdCompressor(options=option) @@ -420,8 +420,8 @@ def test_unknown_decompression_parameter(self): KEY = 100001234 options = {DecompressionParameter.window_log_max: DecompressionParameter.window_log_max.bounds()[1], KEY: 200000000} - pattern = r'Zstd decompression parameter.*?"unknown parameter \(key %d\)"' \ - % KEY + pattern = (r'Invalid zstd decompression parameter.*?' + fr'"unknown parameter \(key {KEY}\)"') with self.assertRaisesRegex(ZstdError, pattern): ZstdDecompressor(options=options) diff --git a/Modules/_zstd/_zstdmodule.c b/Modules/_zstd/_zstdmodule.c index 24796604187ec1..0294828aa106ea 100644 --- a/Modules/_zstd/_zstdmodule.c +++ b/Modules/_zstd/_zstdmodule.c @@ -149,7 +149,7 @@ set_parameter_error(const _zstd_state* const state, int is_compress, } if (ZSTD_isError(bounds.error)) { PyErr_Format(state->ZstdError, - "Zstd %s parameter \"%s\" is invalid.", + "Invalid zstd %s parameter \"%s\".", type, name); return; } @@ -184,13 +184,13 @@ _zstd.train_dict The size of the dictionary. / -Internal function, train a Zstandard dictionary on sample data. +Train a Zstandard dictionary on sample data. [clinic start generated code]*/ static PyObject * _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size) -/*[clinic end generated code: output=8e87fe43935e8f77 input=829e31fbbf3454b0]*/ +/*[clinic end generated code: output=8e87fe43935e8f77 input=d20dedb21c72cb62]*/ { // TODO(emmatyping): The preamble and suffix to this function and _finalize_dict // are pretty similar. We should see if we can refactor them to share that code. @@ -255,7 +255,7 @@ _zstd_train_dict_impl(PyObject *module, PyBytesObject *samples_bytes, chunk_sizes, (uint32_t)chunks_number); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { _zstd_state* const mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_TRAIN_DICT, zstd_ret); @@ -289,10 +289,10 @@ _zstd.finalize_dict dict_size: Py_ssize_t The size of the dictionary. compression_level: int - Optimize for a specific zstd compression level, 0 means default. + Optimize for a specific Zstandard compression level, 0 means default. / -Internal function, finalize a Zstandard dictionary. +Finalize a Zstandard dictionary. [clinic start generated code]*/ static PyObject * @@ -300,7 +300,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, PyBytesObject *samples_bytes, PyObject *samples_sizes, Py_ssize_t dict_size, int compression_level) -/*[clinic end generated code: output=f91821ba5ae85bda input=11af97dcc7608059]*/ +/*[clinic end generated code: output=f91821ba5ae85bda input=3c7e2480aa08fb56]*/ { Py_ssize_t chunks_number; size_t *chunk_sizes = NULL; @@ -357,7 +357,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, /* Parameters */ - /* Optimize for a specific zstd compression level, 0 means default. */ + /* Optimize for a specific Zstandard compression level, 0 means default. */ params.compressionLevel = compression_level; /* Write log to stderr, 0 = none. */ params.notificationLevel = 0; @@ -373,7 +373,7 @@ _zstd_finalize_dict_impl(PyObject *module, PyBytesObject *custom_dict_bytes, (uint32_t)chunks_number, params); Py_END_ALLOW_THREADS - /* Check zstd dict error */ + /* Check Zstandard dict error */ if (ZDICT_isError(zstd_ret)) { _zstd_state* const mod_state = get_zstd_state(module); set_zstd_error(mod_state, ERR_FINALIZE_DICT, zstd_ret); @@ -404,12 +404,12 @@ _zstd.get_param_bounds is_compress: bool True for CompressionParameter, False for DecompressionParameter. -Internal function, get CompressionParameter/DecompressionParameter bounds. +Get CompressionParameter/DecompressionParameter bounds. [clinic start generated code]*/ static PyObject * _zstd_get_param_bounds_impl(PyObject *module, int parameter, int is_compress) -/*[clinic end generated code: output=4acf5a876f0620ca input=84e669591e487008]*/ +/*[clinic end generated code: output=4acf5a876f0620ca input=45742ef0a3531b65]*/ { ZSTD_bounds bound; if (is_compress) { @@ -439,14 +439,12 @@ _zstd.get_frame_size A bytes-like object, it should start from the beginning of a frame, and contains at least one complete frame. -Get the size of a zstd frame, including frame header and 4-byte checksum if it has one. - -It will iterate all blocks' headers within a frame, to accumulate the frame size. +Get the size of a Zstandard frame, including the header and optional checksum. [clinic start generated code]*/ static PyObject * _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=a7384c2f8780f442 input=7d3ad24311893bf3]*/ +/*[clinic end generated code: output=a7384c2f8780f442 input=3b9f73f8c8129d38]*/ { size_t frame_size; @@ -469,14 +467,14 @@ _zstd_get_frame_size_impl(PyObject *module, Py_buffer *frame_buffer) _zstd.get_frame_info frame_buffer: Py_buffer - A bytes-like object, containing the header of a zstd frame. + A bytes-like object, containing the header of a Zstandard frame. -Internal function, get zstd frame infomation from a frame header. +Get Zstandard frame infomation from a frame header. [clinic start generated code]*/ static PyObject * _zstd_get_frame_info_impl(PyObject *module, Py_buffer *frame_buffer) -/*[clinic end generated code: output=56e033cf48001929 input=1816f14656b6aa22]*/ +/*[clinic end generated code: output=56e033cf48001929 input=94b240583ae22ca5]*/ { uint64_t decompressed_size; uint32_t dict_id; @@ -515,13 +513,13 @@ _zstd.set_parameter_types d_parameter_type: object(subclass_of='&PyType_Type') DecompressionParameter IntEnum type object -Internal function, set CompressionParameter/DecompressionParameter types for validity check. +Set CompressionParameter and DecompressionParameter types for validity check. [clinic start generated code]*/ static PyObject * _zstd_set_parameter_types_impl(PyObject *module, PyObject *c_parameter_type, PyObject *d_parameter_type) -/*[clinic end generated code: output=f3313b1294f19502 input=30402523871b8280]*/ +/*[clinic end generated code: output=f3313b1294f19502 input=75d7a953580fae5f]*/ { _zstd_state* const mod_state = get_zstd_state(module); diff --git a/Modules/_zstd/clinic/_zstdmodule.c.h b/Modules/_zstd/clinic/_zstdmodule.c.h index 13c4a53302c64c..766e1cfa776767 100644 --- a/Modules/_zstd/clinic/_zstdmodule.c.h +++ b/Modules/_zstd/clinic/_zstdmodule.c.h @@ -13,7 +13,7 @@ PyDoc_STRVAR(_zstd_train_dict__doc__, "train_dict($module, samples_bytes, samples_sizes, dict_size, /)\n" "--\n" "\n" -"Internal function, train a Zstandard dictionary on sample data.\n" +"Train a Zstandard dictionary on sample data.\n" "\n" " samples_bytes\n" " Concatenation of samples.\n" @@ -73,7 +73,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__, " dict_size, compression_level, /)\n" "--\n" "\n" -"Internal function, finalize a Zstandard dictionary.\n" +"Finalize a Zstandard dictionary.\n" "\n" " custom_dict_bytes\n" " Custom dictionary content.\n" @@ -84,7 +84,7 @@ PyDoc_STRVAR(_zstd_finalize_dict__doc__, " dict_size\n" " The size of the dictionary.\n" " compression_level\n" -" Optimize for a specific zstd compression level, 0 means default."); +" Optimize for a specific Zstandard compression level, 0 means default."); #define _ZSTD_FINALIZE_DICT_METHODDEF \ {"finalize_dict", _PyCFunction_CAST(_zstd_finalize_dict), METH_FASTCALL, _zstd_finalize_dict__doc__}, @@ -149,7 +149,7 @@ PyDoc_STRVAR(_zstd_get_param_bounds__doc__, "get_param_bounds($module, /, parameter, is_compress)\n" "--\n" "\n" -"Internal function, get CompressionParameter/DecompressionParameter bounds.\n" +"Get CompressionParameter/DecompressionParameter bounds.\n" "\n" " parameter\n" " The parameter to get bounds.\n" @@ -220,13 +220,11 @@ PyDoc_STRVAR(_zstd_get_frame_size__doc__, "get_frame_size($module, /, frame_buffer)\n" "--\n" "\n" -"Get the size of a zstd frame, including frame header and 4-byte checksum if it has one.\n" +"Get the size of a Zstandard frame, including the header and optional checksum.\n" "\n" " frame_buffer\n" " A bytes-like object, it should start from the beginning of a frame,\n" -" and contains at least one complete frame.\n" -"\n" -"It will iterate all blocks\' headers within a frame, to accumulate the frame size."); +" and contains at least one complete frame."); #define _ZSTD_GET_FRAME_SIZE_METHODDEF \ {"get_frame_size", _PyCFunction_CAST(_zstd_get_frame_size), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_size__doc__}, @@ -291,10 +289,10 @@ PyDoc_STRVAR(_zstd_get_frame_info__doc__, "get_frame_info($module, /, frame_buffer)\n" "--\n" "\n" -"Internal function, get zstd frame infomation from a frame header.\n" +"Get Zstandard frame infomation from a frame header.\n" "\n" " frame_buffer\n" -" A bytes-like object, containing the header of a zstd frame."); +" A bytes-like object, containing the header of a Zstandard frame."); #define _ZSTD_GET_FRAME_INFO_METHODDEF \ {"get_frame_info", _PyCFunction_CAST(_zstd_get_frame_info), METH_FASTCALL|METH_KEYWORDS, _zstd_get_frame_info__doc__}, @@ -359,7 +357,7 @@ PyDoc_STRVAR(_zstd_set_parameter_types__doc__, "set_parameter_types($module, /, c_parameter_type, d_parameter_type)\n" "--\n" "\n" -"Internal function, set CompressionParameter/DecompressionParameter types for validity check.\n" +"Set CompressionParameter and DecompressionParameter types for validity check.\n" "\n" " c_parameter_type\n" " CompressionParameter IntEnum type object\n" @@ -428,4 +426,4 @@ _zstd_set_parameter_types(PyObject *module, PyObject *const *args, Py_ssize_t na exit: return return_value; } -/*[clinic end generated code: output=553eaf1ea05d38ef input=a9049054013a1b77]*/ +/*[clinic end generated code: output=437b084f149e68e5 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/compressor.c.h b/Modules/_zstd/clinic/compressor.c.h index e38ad378427ba1..f69161b590e5b7 100644 --- a/Modules/_zstd/clinic/compressor.c.h +++ b/Modules/_zstd/clinic/compressor.c.h @@ -19,7 +19,7 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_new__doc__, " options\n" " A dict object that contains advanced compression parameters.\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" "\n" "Thread-safe at method level. For one-shot compression, use the compress()\n" "function instead."); @@ -189,9 +189,9 @@ PyDoc_STRVAR(_zstd_ZstdCompressor_flush__doc__, " Can be these 2 values ZstdCompressor.FLUSH_FRAME,\n" " ZstdCompressor.FLUSH_BLOCK\n" "\n" -"Flush any remaining data left in internal buffers. Since zstd data consists\n" -"of one or more independent frames, the compressor object can still be used\n" -"after this method is called."); +"Flush any remaining data left in internal buffers. Since Zstandard data\n" +"consists of one or more independent frames, the compressor object can still\n" +"be used after this method is called."); #define _ZSTD_ZSTDCOMPRESSOR_FLUSH_METHODDEF \ {"flush", _PyCFunction_CAST(_zstd_ZstdCompressor_flush), METH_FASTCALL|METH_KEYWORDS, _zstd_ZstdCompressor_flush__doc__}, @@ -252,4 +252,4 @@ _zstd_ZstdCompressor_flush(PyObject *self, PyObject *const *args, Py_ssize_t nar exit: return return_value; } -/*[clinic end generated code: output=c33f03bf68e679e5 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=ee2d1dc298de790c input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/decompressor.c.h b/Modules/_zstd/clinic/decompressor.c.h index ebd26c5920f01c..4ecb19e9bde6ed 100644 --- a/Modules/_zstd/clinic/decompressor.c.h +++ b/Modules/_zstd/clinic/decompressor.c.h @@ -17,7 +17,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_new__doc__, "Create a decompressor object for decompressing data incrementally.\n" "\n" " zstd_dict\n" -" A ZstdDict object, a pre-trained zstd dictionary.\n" +" A ZstdDict object, a pre-trained Zstandard dictionary.\n" " options\n" " A dict object that contains advanced decompression parameters.\n" "\n" @@ -130,7 +130,7 @@ PyDoc_STRVAR(_zstd_ZstdDecompressor_decompress__doc__, "Decompress *data*, returning uncompressed bytes if possible, or b\'\' otherwise.\n" "\n" " data\n" -" A bytes-like object, zstd data to be decompressed.\n" +" A bytes-like object, Zstandard data to be decompressed.\n" " max_length\n" " Maximum size of returned data. When it is negative, the size of\n" " output buffer is unlimited. When it is nonnegative, returns at\n" @@ -227,4 +227,4 @@ _zstd_ZstdDecompressor_decompress(PyObject *self, PyObject *const *args, Py_ssiz return return_value; } -/*[clinic end generated code: output=219d26afb3452705 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7a4d278f9244e684 input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/clinic/zstddict.c.h b/Modules/_zstd/clinic/zstddict.c.h index 222c6965117f4f..34e0e4b3ecfe72 100644 --- a/Modules/_zstd/clinic/zstddict.c.h +++ b/Modules/_zstd/clinic/zstddict.c.h @@ -13,17 +13,17 @@ PyDoc_STRVAR(_zstd_ZstdDict_new__doc__, "ZstdDict(dict_content, /, *, is_raw=False)\n" "--\n" "\n" -"Represents a zstd dictionary, which can be used for compression/decompression.\n" +"Represents a Zstandard dictionary.\n" "\n" " dict_content\n" " The content of a Zstandard dictionary as a bytes-like object.\n" " is_raw\n" " If true, perform no checks on *dict_content*, useful for some\n" " advanced cases. Otherwise, check that the content represents\n" -" a Zstandard dictionary created by the zstd functions.\n" +" a Zstandard dictionary created by the zstd library or CLI.\n" "\n" -"It\'s thread-safe, and can be shared by multiple ZstdCompressor /\n" -"ZstdDecompressor objects."); +"The dictionary can be used for compression or decompression, and can be shared\n" +"by multiple ZstdCompressor or ZstdDecompressor objects."); static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, @@ -202,4 +202,4 @@ _zstd_ZstdDict_as_prefix_get(PyObject *self, void *Py_UNUSED(context)) return return_value; } -/*[clinic end generated code: output=f301594e7b37f709 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=bfb31c1187477afd input=a9049054013a1b77]*/ diff --git a/Modules/_zstd/compressor.c b/Modules/_zstd/compressor.c index 702d90370d2980..f794acbc19cfc7 100644 --- a/Modules/_zstd/compressor.c +++ b/Modules/_zstd/compressor.c @@ -115,7 +115,7 @@ _zstd_set_c_parameters(ZstdCompressor *self, PyObject *level_or_options, self->compression_level = value_v; } else if (key_v == ZSTD_c_nbWorkers) { - /* From zstd library doc: + /* From the zstd library docs: 1. When nbWorkers >= 1, triggers asynchronous mode when used with ZSTD_compressStream2(). 2, Default value is `0`, aka "single-threaded mode" : no @@ -319,7 +319,7 @@ _zstd.ZstdCompressor.__new__ as _zstd_ZstdCompressor_new options: object = None A dict object that contains advanced compression parameters. zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. + A ZstdDict object, a pre-trained Zstandard dictionary. Create a compressor object for compressing data incrementally. @@ -330,7 +330,7 @@ function instead. static PyObject * _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, PyObject *options, PyObject *zstd_dict) -/*[clinic end generated code: output=cdef61eafecac3d7 input=b9ea61ecafbb1b1e]*/ +/*[clinic end generated code: output=cdef61eafecac3d7 input=92de0211ae20ffdc]*/ { ZstdCompressor* self = PyObject_GC_New(ZstdCompressor, type); if (self == NULL) { @@ -371,7 +371,7 @@ _zstd_ZstdCompressor_new_impl(PyTypeObject *type, PyObject *level, } } - /* Load zstd dictionary to compression context */ + /* Load Zstandard dictionary to compression context */ self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_c_dict(self, zstd_dict) < 0) { @@ -446,7 +446,7 @@ compress_impl(ZstdCompressor *self, Py_buffer *data, } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS zstd_ret = ZSTD_compressStream2(self->cctx, &out, &in, end_directive); @@ -510,7 +510,7 @@ compress_mt_continue_impl(ZstdCompressor *self, Py_buffer *data) goto error; } - /* zstd stream compress */ + /* Zstandard stream compress */ while (1) { Py_BEGIN_ALLOW_THREADS do { @@ -619,14 +619,14 @@ _zstd.ZstdCompressor.flush Finish the compression process. -Flush any remaining data left in internal buffers. Since zstd data consists -of one or more independent frames, the compressor object can still be used -after this method is called. +Flush any remaining data left in internal buffers. Since Zstandard data +consists of one or more independent frames, the compressor object can still +be used after this method is called. [clinic start generated code]*/ static PyObject * _zstd_ZstdCompressor_flush_impl(ZstdCompressor *self, int mode) -/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=a766870301932b85]*/ +/*[clinic end generated code: output=b7cf2c8d64dcf2e3 input=0ab19627f323cdbc]*/ { PyObject *ret; diff --git a/Modules/_zstd/decompressor.c b/Modules/_zstd/decompressor.c index 8ed61e6b96fca8..852b796a872eef 100644 --- a/Modules/_zstd/decompressor.c +++ b/Modules/_zstd/decompressor.c @@ -259,8 +259,8 @@ _zstd_load_d_dict(ZstdDecompressor *self, PyObject *dict) finish ZSTD_decompressStream()'s size_t return value: - - 0 when a frame is completely decoded and fully flushed, zstd's internal - buffer has no data. + - 0 when a frame is completely decoded and fully flushed, + zstd's internal buffer has no data. - An error code, which can be tested using ZSTD_isError(). - Or any other value > 0, which means there is still some decoding or flushing to do to complete current frame. @@ -305,7 +305,7 @@ decompress_impl(ZstdDecompressor *self, ZSTD_inBuffer *in, } /* Need to check out before in. Maybe zstd's internal buffer still has - a few bytes can be output, grow the buffer and continue. */ + a few bytes that can be output, grow the buffer and continue. */ if (out.pos == out.size) { /* Output buffer exhausted */ @@ -528,7 +528,7 @@ stream_decompress(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length @classmethod _zstd.ZstdDecompressor.__new__ as _zstd_ZstdDecompressor_new zstd_dict: object = None - A ZstdDict object, a pre-trained zstd dictionary. + A ZstdDict object, a pre-trained Zstandard dictionary. options: object = None A dict object that contains advanced decompression parameters. @@ -541,7 +541,7 @@ function instead. static PyObject * _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, PyObject *options) -/*[clinic end generated code: output=590ca65c1102ff4a input=0f161edb33d83216]*/ +/*[clinic end generated code: output=590ca65c1102ff4a input=213daa57e3ea4062]*/ { ZstdDecompressor* self = PyObject_GC_New(ZstdDecompressor, type); if (self == NULL) { @@ -569,7 +569,7 @@ _zstd_ZstdDecompressor_new_impl(PyTypeObject *type, PyObject *zstd_dict, goto error; } - /* Load zstd dictionary to decompression context */ + /* Load Zstandard dictionary to decompression context */ self->dict = NULL; if (zstd_dict != Py_None) { if (_zstd_load_d_dict(self, zstd_dict) < 0) { @@ -663,7 +663,7 @@ _zstd_ZstdDecompressor_unused_data_get_impl(ZstdDecompressor *self) _zstd.ZstdDecompressor.decompress data: Py_buffer - A bytes-like object, zstd data to be decompressed. + A bytes-like object, Zstandard data to be decompressed. max_length: Py_ssize_t = -1 Maximum size of returned data. When it is negative, the size of output buffer is unlimited. When it is nonnegative, returns at @@ -689,7 +689,7 @@ static PyObject * _zstd_ZstdDecompressor_decompress_impl(ZstdDecompressor *self, Py_buffer *data, Py_ssize_t max_length) -/*[clinic end generated code: output=a4302b3c940dbec6 input=830e455bc9a50b6e]*/ +/*[clinic end generated code: output=a4302b3c940dbec6 input=6463dfdf98091caa]*/ { PyObject *ret; /* Thread-safe code */ diff --git a/Modules/_zstd/zstddict.c b/Modules/_zstd/zstddict.c index d837d30943cd9b..264946e8da2a99 100644 --- a/Modules/_zstd/zstddict.c +++ b/Modules/_zstd/zstddict.c @@ -32,18 +32,18 @@ _zstd.ZstdDict.__new__ as _zstd_ZstdDict_new is_raw: bool = False If true, perform no checks on *dict_content*, useful for some advanced cases. Otherwise, check that the content represents - a Zstandard dictionary created by the zstd functions. + a Zstandard dictionary created by the zstd library or CLI. -Represents a zstd dictionary, which can be used for compression/decompression. +Represents a Zstandard dictionary. -It's thread-safe, and can be shared by multiple ZstdCompressor / -ZstdDecompressor objects. +The dictionary can be used for compression or decompression, and can be shared +by multiple ZstdCompressor or ZstdDecompressor objects. [clinic start generated code]*/ static PyObject * _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, int is_raw) -/*[clinic end generated code: output=3ebff839cb3be6d7 input=e9e22fdc68fa04cc]*/ +/*[clinic end generated code: output=3ebff839cb3be6d7 input=6b5de413869ae878]*/ { ZstdDict* self = PyObject_GC_New(ZstdDict, type); if (self == NULL) { @@ -71,7 +71,7 @@ _zstd_ZstdDict_new_impl(PyTypeObject *type, PyObject *dict_content, at least 8 bytes */ if (Py_SIZE(self->dict_content) < 8) { PyErr_SetString(PyExc_ValueError, - "Zstd dictionary content should at least 8 bytes."); + "Zstandard dictionary content should at least 8 bytes."); goto error; } @@ -118,7 +118,7 @@ ZstdDict_dealloc(PyObject *ob) } PyDoc_STRVAR(ZstdDict_dictid_doc, -"The ID of Zstandard dictionary, an integer between 0 and 2**32.\n\n" +"the Zstandard dictionary, an int between 0 and 2**32.\n\n" "A non-zero value represents an ordinary Zstandard dictionary, " "conforming to the standardised format.\n\n" "The special value '0' means a 'raw content' dictionary,"