From 2ac3f826917eec15ac9146df643ba86e47df94df Mon Sep 17 00:00:00 2001 From: dvora-h Date: Wed, 23 Feb 2022 12:20:12 +0200 Subject: [PATCH 1/9] cluster support for functions --- redis/cluster.py | 13 +++++++++++++ redis/commands/cluster.py | 2 ++ tests/test_function.py | 26 +++++++++++++++++++++++++- 3 files changed, 40 insertions(+), 1 deletion(-) diff --git a/redis/cluster.py b/redis/cluster.py index b8d6b1997f..831e638110 100644 --- a/redis/cluster.py +++ b/redis/cluster.py @@ -289,12 +289,21 @@ class RedisCluster(RedisClusterCommands): [ "FLUSHALL", "FLUSHDB", + "FUNCTION DELETE", + "FUNCTION FLUSH", + "FUNCTION LIST", + "FUNCTION LOAD", + "FUNCTION RESTORE", "SCRIPT EXISTS", "SCRIPT FLUSH", "SCRIPT LOAD", ], PRIMARIES, ), + list_keys_to_dict( + ["FUNCTION DUMP"], + RANDOM, + ), list_keys_to_dict( [ "CLUSTER COUNTKEYSINSLOT", @@ -843,6 +852,10 @@ def determine_slot(self, *args): else: keys = self._get_command_keys(*args) if keys is None or len(keys) == 0: + # FCALL can call a function with 0 keys, that means the function + # can be run on any node so we can just return a random slot + if command in ("FCALL", "FCALL_RO"): + return random.randrange(0, REDIS_CLUSTER_HASH_SLOTS) raise RedisClusterException( "No way to dispatch this command to Redis Cluster. " "Missing key.\nYou can execute the command by specifying " diff --git a/redis/commands/cluster.py b/redis/commands/cluster.py index 8bdcbbadf6..33209767dd 100644 --- a/redis/commands/cluster.py +++ b/redis/commands/cluster.py @@ -4,6 +4,7 @@ from .core import ( ACLCommands, DataAccessCommands, + FunctionCommands, ManagementCommands, PubSubCommands, ScriptCommands, @@ -212,6 +213,7 @@ class RedisClusterCommands( PubSubCommands, ClusterDataAccessCommands, ScriptCommands, + FunctionCommands, ): """ A class for all Redis Cluster commands diff --git a/tests/test_function.py b/tests/test_function.py index 921ba30444..35a989cbd4 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -10,7 +10,6 @@ return redis.call('GET', keys[1]) end)" -@pytest.mark.onlynoncluster # @skip_if_server_version_lt("7.0.0") turn on after redis 7 release class TestFunction: @pytest.fixture(autouse=True) @@ -44,6 +43,7 @@ def test_function_flush(self, unstable_r): with pytest.raises(ResponseError): unstable_r.function_flush("ABC") + @pytest.mark.onlynoncluster def test_function_list(self, unstable_r): unstable_r.function_load("Lua", "mylib", function) res = [ @@ -62,6 +62,30 @@ def test_function_list(self, unstable_r): assert unstable_r.function_list(library="*lib") == res assert unstable_r.function_list(withcode=True)[0][9] == function + @pytest.mark.onlynoncluster + def test_function_list_on_cluster(self, unstable_r): + unstable_r.function_load("Lua", "mylib", function) + function_list = [ + [ + "library_name", + "mylib", + "engine", + "LUA", + "description", + None, + "functions", + [["name", "myfunc", "description", None]], + ], + ] + primaries = unstable_r.get_primaries() + res = {} + for node in primaries: + res[node.name] = function_list + assert unstable_r.function_list() == res + assert unstable_r.function_list(library="*lib") == res + node = primaries[0].name + assert unstable_r.function_list(withcode=True)[node][0][9] == function + def test_fcall(self, unstable_r): unstable_r.function_load("Lua", "mylib", set_function) unstable_r.function_load("Lua", "mylib2", get_function) From e2ba620f306c8452a8922eb60a4be03c11ec805a Mon Sep 17 00:00:00 2001 From: dvora-h Date: Wed, 23 Feb 2022 12:38:56 +0200 Subject: [PATCH 2/9] fix test_list_on_cluster mark --- tests/test_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_function.py b/tests/test_function.py index 35a989cbd4..1fb62f89f8 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -62,7 +62,7 @@ def test_function_list(self, unstable_r): assert unstable_r.function_list(library="*lib") == res assert unstable_r.function_list(withcode=True)[0][9] == function - @pytest.mark.onlynoncluster + @pytest.mark.onlyoncluster def test_function_list_on_cluster(self, unstable_r): unstable_r.function_load("Lua", "mylib", function) function_list = [ From b3c98a3721cd57193e747d7c1333b49785dad8d3 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Wed, 23 Feb 2022 12:52:27 +0200 Subject: [PATCH 3/9] fix mark --- tests/test_function.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_function.py b/tests/test_function.py index 1fb62f89f8..7a83d45344 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -62,7 +62,7 @@ def test_function_list(self, unstable_r): assert unstable_r.function_list(library="*lib") == res assert unstable_r.function_list(withcode=True)[0][9] == function - @pytest.mark.onlyoncluster + @pytest.mark.onlycluster def test_function_list_on_cluster(self, unstable_r): unstable_r.function_load("Lua", "mylib", function) function_list = [ From 7c370e183149f90c9a91a0fc3af2bb9e378e8083 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 14:18:57 +0200 Subject: [PATCH 4/9] cluster unstable url --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 3ef01ddead..9769e67b5b 100644 --- a/tox.ini +++ b/tox.ini @@ -267,11 +267,11 @@ extras = hiredis: hiredis ocsp: cryptography, pyopenssl, requests setenv = - CLUSTER_URL = "redis://localhost:16379/0" + CLUSTER_URL = "redis://localhost:6372/0" commands = standalone: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' {posargs} standalone-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} - cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} {posargs} + cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} --redis-unstable-url{env:CLUSTER_URL:}={posargs} cluster-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} [testenv:redis5] From 465c21e3515ac5b8777816d24627f63e77ae956f Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 14:30:14 +0200 Subject: [PATCH 5/9] fix --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index e99cb991b8..04581974cf 100644 --- a/tox.ini +++ b/tox.ini @@ -286,7 +286,7 @@ setenv = commands = standalone: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' {posargs} standalone-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} - cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} --redis-unstable-url{env:CLUSTER_URL:}={posargs} + cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} --redis-unstable-url={env:CLUSTER_URL:} {posargs} cluster-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} [testenv:redis5] From 0801cf8a04c573c3e187d3eaf198c5c510d59539 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 14:44:49 +0200 Subject: [PATCH 6/9] fix cluster url --- tox.ini | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 04581974cf..e9174a66c5 100644 --- a/tox.ini +++ b/tox.ini @@ -282,11 +282,12 @@ extras = hiredis: hiredis ocsp: cryptography, pyopenssl, requests setenv = - CLUSTER_URL = "redis://localhost:6372/0" + CLUSTER_URL = "redis://localhost:16379/0" + UNSTABLE_CLUSTER_URL = "redis://localhost:6372/0" commands = standalone: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' {posargs} standalone-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} - cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} --redis-unstable-url={env:CLUSTER_URL:} {posargs} + cluster: pytest --cov=./ --cov-report=xml:coverage_cluster.xml -W always -m 'not onlynoncluster and not redismod' --redis-url={env:CLUSTER_URL:} --redis-unstable-url={env:UNSTABLE_CLUSTER_URL:} {posargs} cluster-uvloop: pytest --cov=./ --cov-report=xml:coverage_redis.xml -W always -m 'not onlycluster' --uvloop {posargs} [testenv:redis5] From 7fd19d7024be4ce9fef513e1619b0e87eef0c1c6 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 17:30:32 +0200 Subject: [PATCH 7/9] skip tests --- tests/test_function.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_function.py b/tests/test_function.py index 7a83d45344..38fcb5b4b5 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -1,6 +1,7 @@ import pytest from redis.exceptions import ResponseError +from .conftest import skip_if_server_version_lt function = "redis.register_function('myfunc', function(keys, args) return args[1] end)" function2 = "redis.register_function('hello', function() return 'Hello World' end)" @@ -10,7 +11,7 @@ return redis.call('GET', keys[1]) end)" -# @skip_if_server_version_lt("7.0.0") turn on after redis 7 release +@skip_if_server_version_lt("7.0.0") class TestFunction: @pytest.fixture(autouse=True) def reset_functions(self, unstable_r): From cc0d7b8e4b0393cec712fc3c26377049dc2f3c72 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 17:32:48 +0200 Subject: [PATCH 8/9] linters --- tests/test_function.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_function.py b/tests/test_function.py index 38fcb5b4b5..0ec01079b4 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -1,6 +1,7 @@ import pytest from redis.exceptions import ResponseError + from .conftest import skip_if_server_version_lt function = "redis.register_function('myfunc', function(keys, args) return args[1] end)" From 66e58991c506e1decb40710c4967045a5e282a59 Mon Sep 17 00:00:00 2001 From: dvora-h Date: Sun, 6 Mar 2022 17:34:33 +0200 Subject: [PATCH 9/9] linters --- tests/test_function.py | 110 ++++++++++++++++++++--------------------- 1 file changed, 55 insertions(+), 55 deletions(-) diff --git a/tests/test_function.py b/tests/test_function.py index 0ec01079b4..6f0a6ec1e1 100644 --- a/tests/test_function.py +++ b/tests/test_function.py @@ -15,39 +15,39 @@ @skip_if_server_version_lt("7.0.0") class TestFunction: @pytest.fixture(autouse=True) - def reset_functions(self, unstable_r): - unstable_r.function_flush() + def reset_functions(self, r): + r.function_flush() - def test_function_load(self, unstable_r): - assert unstable_r.function_load("Lua", "mylib", function) - assert unstable_r.function_load("Lua", "mylib", function, replace=True) + def test_function_load(self, r): + assert r.function_load("Lua", "mylib", function) + assert r.function_load("Lua", "mylib", function, replace=True) with pytest.raises(ResponseError): - unstable_r.function_load("Lua", "mylib", function) + r.function_load("Lua", "mylib", function) with pytest.raises(ResponseError): - unstable_r.function_load("Lua", "mylib2", function) + r.function_load("Lua", "mylib2", function) - def test_function_delete(self, unstable_r): - unstable_r.function_load("Lua", "mylib", set_function) + def test_function_delete(self, r): + r.function_load("Lua", "mylib", set_function) with pytest.raises(ResponseError): - unstable_r.function_load("Lua", "mylib", set_function) - assert unstable_r.fcall("set", 1, "foo", "bar") == "OK" - assert unstable_r.function_delete("mylib") + r.function_load("Lua", "mylib", set_function) + assert r.fcall("set", 1, "foo", "bar") == "OK" + assert r.function_delete("mylib") with pytest.raises(ResponseError): - unstable_r.fcall("set", 1, "foo", "bar") - assert unstable_r.function_load("Lua", "mylib", set_function) + r.fcall("set", 1, "foo", "bar") + assert r.function_load("Lua", "mylib", set_function) - def test_function_flush(self, unstable_r): - unstable_r.function_load("Lua", "mylib", function) - assert unstable_r.fcall("myfunc", 0, "hello") == "hello" - assert unstable_r.function_flush() + def test_function_flush(self, r): + r.function_load("Lua", "mylib", function) + assert r.fcall("myfunc", 0, "hello") == "hello" + assert r.function_flush() with pytest.raises(ResponseError): - unstable_r.fcall("myfunc", 0, "hello") + r.fcall("myfunc", 0, "hello") with pytest.raises(ResponseError): - unstable_r.function_flush("ABC") + r.function_flush("ABC") @pytest.mark.onlynoncluster - def test_function_list(self, unstable_r): - unstable_r.function_load("Lua", "mylib", function) + def test_function_list(self, r): + r.function_load("Lua", "mylib", function) res = [ [ "library_name", @@ -60,13 +60,13 @@ def test_function_list(self, unstable_r): [["name", "myfunc", "description", None]], ], ] - assert unstable_r.function_list() == res - assert unstable_r.function_list(library="*lib") == res - assert unstable_r.function_list(withcode=True)[0][9] == function + assert r.function_list() == res + assert r.function_list(library="*lib") == res + assert r.function_list(withcode=True)[0][9] == function @pytest.mark.onlycluster - def test_function_list_on_cluster(self, unstable_r): - unstable_r.function_load("Lua", "mylib", function) + def test_function_list_on_cluster(self, r): + r.function_load("Lua", "mylib", function) function_list = [ [ "library_name", @@ -79,42 +79,42 @@ def test_function_list_on_cluster(self, unstable_r): [["name", "myfunc", "description", None]], ], ] - primaries = unstable_r.get_primaries() + primaries = r.get_primaries() res = {} for node in primaries: res[node.name] = function_list - assert unstable_r.function_list() == res - assert unstable_r.function_list(library="*lib") == res + assert r.function_list() == res + assert r.function_list(library="*lib") == res node = primaries[0].name - assert unstable_r.function_list(withcode=True)[node][0][9] == function + assert r.function_list(withcode=True)[node][0][9] == function - def test_fcall(self, unstable_r): - unstable_r.function_load("Lua", "mylib", set_function) - unstable_r.function_load("Lua", "mylib2", get_function) - assert unstable_r.fcall("set", 1, "foo", "bar") == "OK" - assert unstable_r.fcall("get", 1, "foo") == "bar" + def test_fcall(self, r): + r.function_load("Lua", "mylib", set_function) + r.function_load("Lua", "mylib2", get_function) + assert r.fcall("set", 1, "foo", "bar") == "OK" + assert r.fcall("get", 1, "foo") == "bar" with pytest.raises(ResponseError): - unstable_r.fcall("myfunc", 0, "hello") + r.fcall("myfunc", 0, "hello") - def test_fcall_ro(self, unstable_r): - unstable_r.function_load("Lua", "mylib", function) - assert unstable_r.fcall_ro("myfunc", 0, "hello") == "hello" - unstable_r.function_load("Lua", "mylib2", set_function) + def test_fcall_ro(self, r): + r.function_load("Lua", "mylib", function) + assert r.fcall_ro("myfunc", 0, "hello") == "hello" + r.function_load("Lua", "mylib2", set_function) with pytest.raises(ResponseError): - unstable_r.fcall_ro("set", 1, "foo", "bar") + r.fcall_ro("set", 1, "foo", "bar") - def test_function_dump_restore(self, unstable_r): - unstable_r.function_load("Lua", "mylib", set_function) - payload = unstable_r.function_dump() - assert unstable_r.fcall("set", 1, "foo", "bar") == "OK" - unstable_r.function_delete("mylib") + def test_function_dump_restore(self, r): + r.function_load("Lua", "mylib", set_function) + payload = r.function_dump() + assert r.fcall("set", 1, "foo", "bar") == "OK" + r.function_delete("mylib") with pytest.raises(ResponseError): - unstable_r.fcall("set", 1, "foo", "bar") - assert unstable_r.function_restore(payload) - assert unstable_r.fcall("set", 1, "foo", "bar") == "OK" - unstable_r.function_load("Lua", "mylib2", get_function) - assert unstable_r.fcall("get", 1, "foo") == "bar" - unstable_r.function_delete("mylib") - assert unstable_r.function_restore(payload, "FLUSH") + r.fcall("set", 1, "foo", "bar") + assert r.function_restore(payload) + assert r.fcall("set", 1, "foo", "bar") == "OK" + r.function_load("Lua", "mylib2", get_function) + assert r.fcall("get", 1, "foo") == "bar" + r.function_delete("mylib") + assert r.function_restore(payload, "FLUSH") with pytest.raises(ResponseError): - unstable_r.fcall("get", 1, "foo") + r.fcall("get", 1, "foo")