From 8d3efd5d4e9f8e78b3a2210a9716aed29383a9d0 Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Tue, 11 Jun 2024 10:26:29 -0700 Subject: [PATCH 1/8] upgrade redhat8 python/ansible --- base/redhat-8/Dockerfile | 4 ++-- base/redhat-8/install.sh | 2 +- py23-image/redhat-8/Dockerfile | 10 +++++----- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/base/redhat-8/Dockerfile b/base/redhat-8/Dockerfile index 2f71cb49..c52fe57c 100644 --- a/base/redhat-8/Dockerfile +++ b/base/redhat-8/Dockerfile @@ -28,8 +28,8 @@ LABEL name="splunk" \ ARG BUSYBOX_URL ENV BUSYBOX_URL=${BUSYBOX_URL} \ - PYTHON_VERSION=3.7.16 \ - PYTHON_GPG_KEY_ID=0D96DF4D4110E5C43FBFB17F2D347EA6AA65421D + PYTHON_VERSION=3.8.19 \ + PYTHON_GPG_KEY_ID=E3FF2839C048B25C084DEBE9B26995E310250568 COPY install.sh /install.sh diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index 63e7950c..017b8e8a 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -74,7 +74,7 @@ ln -sf /usr/bin/pip${PY_SHORT} /usr/bin/pip # Install splunk-ansible dependencies cd / -/usr/bin/python3.7 -m pip install --upgrade pip +/usr/bin/python3.8 -m pip install --upgrade pip pip -q --no-cache-dir install --upgrade "requests_unixsocket<2.29" "requests<2.29" six wheel Mako "urllib3<2.0.0" certifi jmespath future avro cryptography lxml protobuf setuptools ansible # Remove tests packaged in python libs diff --git a/py23-image/redhat-8/Dockerfile b/py23-image/redhat-8/Dockerfile index 89676579..4dd297a8 100644 --- a/py23-image/redhat-8/Dockerfile +++ b/py23-image/redhat-8/Dockerfile @@ -5,8 +5,8 @@ USER root RUN microdnf -y --nodocs update \ && microdnf -y --nodocs install python2-pip python2-devel \ && pip2 --no-cache-dir install requests pyyaml jmespath \ - && ln -sf /usr/bin/python3.7 /usr/bin/python3 \ - && ln -sf /usr/bin/pip3.7 /usr/bin/pip3 \ - && ln -sf /usr/bin/python3.7 /usr/bin/python \ - && ln -sf /usr/bin/pip3.7 /usr/bin/pip \ - && pip3 install --upgrade ansible==3.4.0 requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 + && ln -sf /usr/bin/python3.8 /usr/bin/python3 \ + && ln -sf /usr/bin/pip3.8 /usr/bin/pip3 \ + && ln -sf /usr/bin/python3.8 /usr/bin/python \ + && ln -sf /usr/bin/pip3.8 /usr/bin/pip \ + && pip3 install --upgrade requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 From 27201c1bf600c5655015f5886fc2221ab1bceb1f Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Tue, 11 Jun 2024 12:07:18 -0700 Subject: [PATCH 2/8] add py3 interpreter path --- py23-image/redhat-8/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/py23-image/redhat-8/Dockerfile b/py23-image/redhat-8/Dockerfile index 4dd297a8..22353cee 100644 --- a/py23-image/redhat-8/Dockerfile +++ b/py23-image/redhat-8/Dockerfile @@ -9,4 +9,5 @@ RUN microdnf -y --nodocs update \ && ln -sf /usr/bin/pip3.8 /usr/bin/pip3 \ && ln -sf /usr/bin/python3.8 /usr/bin/python \ && ln -sf /usr/bin/pip3.8 /usr/bin/pip \ - && pip3 install --upgrade requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 + && pip3 install --upgrade requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 \ + && sed -i '/^\[defaults\]/a\interpreter_python = /usr/bin/python3' /opt/ansible/ansible.cfg From c2560fa29863f3968c68355d30c1b02834b30b23 Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Wed, 12 Jun 2024 15:35:11 -0700 Subject: [PATCH 3/8] use py3.9, resolve CVEs --- base/redhat-8/Dockerfile | 2 +- base/redhat-8/install.sh | 8 ++++++-- py23-image/redhat-8/Dockerfile | 11 ++++++----- 3 files changed, 13 insertions(+), 8 deletions(-) diff --git a/base/redhat-8/Dockerfile b/base/redhat-8/Dockerfile index c52fe57c..d839e582 100644 --- a/base/redhat-8/Dockerfile +++ b/base/redhat-8/Dockerfile @@ -28,7 +28,7 @@ LABEL name="splunk" \ ARG BUSYBOX_URL ENV BUSYBOX_URL=${BUSYBOX_URL} \ - PYTHON_VERSION=3.8.19 \ + PYTHON_VERSION=3.9.19 \ PYTHON_GPG_KEY_ID=E3FF2839C048B25C084DEBE9B26995E310250568 COPY install.sh /install.sh diff --git a/base/redhat-8/install.sh b/base/redhat-8/install.sh index 017b8e8a..2763a84a 100755 --- a/base/redhat-8/install.sh +++ b/base/redhat-8/install.sh @@ -17,7 +17,8 @@ set -e # Generate UTF-8 char map and locale # Reinstalling local English def for now, removed in minimal image: https://bugzilla.redhat.com/show_bug.cgi?id=1665251 -microdnf -y --nodocs install glibc-langpack-en +# Comment below install until glibc update is available in minimal image: https://access.redhat.com/errata/RHSA-2024:2722 +#microdnf -y --nodocs install glibc-langpack-en # Currently there is no access to the UTF-8 char map. The following command is commented out until # the base container can generate the locale. @@ -74,9 +75,12 @@ ln -sf /usr/bin/pip${PY_SHORT} /usr/bin/pip # Install splunk-ansible dependencies cd / -/usr/bin/python3.8 -m pip install --upgrade pip +/usr/bin/python3.9 -m pip install --upgrade pip pip -q --no-cache-dir install --upgrade "requests_unixsocket<2.29" "requests<2.29" six wheel Mako "urllib3<2.0.0" certifi jmespath future avro cryptography lxml protobuf setuptools ansible +# Avoid vulnerability on old pip version +/usr/libexec/platform-python -m pip install --upgrade pip + # Remove tests packaged in python libs find /usr/lib/ -depth \( -type d -a -not -wholename '*/ansible/plugins/test' -a \( -name test -o -name tests -o -name idle_test \) \) -exec rm -rf '{}' \; find /usr/lib/ -depth \( -type f -a -name '*.pyc' -o -name '*.pyo' -o -name '*.a' \) -exec rm -rf '{}' \; diff --git a/py23-image/redhat-8/Dockerfile b/py23-image/redhat-8/Dockerfile index 22353cee..c2dd5471 100644 --- a/py23-image/redhat-8/Dockerfile +++ b/py23-image/redhat-8/Dockerfile @@ -3,11 +3,12 @@ FROM ${SPLUNK_PRODUCT}-redhat-8:latest USER root RUN microdnf -y --nodocs update \ - && microdnf -y --nodocs install python2-pip python2-devel \ + && microdnf -y --nodocs install python2 \ + && pip2 install --upgrade pip \ && pip2 --no-cache-dir install requests pyyaml jmespath \ - && ln -sf /usr/bin/python3.8 /usr/bin/python3 \ - && ln -sf /usr/bin/pip3.8 /usr/bin/pip3 \ - && ln -sf /usr/bin/python3.8 /usr/bin/python \ - && ln -sf /usr/bin/pip3.8 /usr/bin/pip \ + && ln -sf /usr/bin/python3.9 /usr/bin/python3 \ + && ln -sf /usr/bin/pip3.9 /usr/bin/pip3 \ + && ln -sf /usr/bin/python3.9 /usr/bin/python \ + && ln -sf /usr/bin/pip3.9 /usr/bin/pip \ && pip3 install --upgrade requests==2.25.1 pyyaml==5.4.1 jmespath==0.10.0 \ && sed -i '/^\[defaults\]/a\interpreter_python = /usr/bin/python3' /opt/ansible/ansible.cfg From ef02f58976629ae6988d1f4a786a1ae40979a72e Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Fri, 14 Jun 2024 10:54:10 -0700 Subject: [PATCH 4/8] initial commit --- tests/executor.py | 6 ++++++ tests/test_single_splunk_image.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/tests/executor.py b/tests/executor.py index c6026adc..856cf687 100644 --- a/tests/executor.py +++ b/tests/executor.py @@ -380,3 +380,9 @@ def check_dmc_groups(self, splunkd_port, num_idx, num_sh, num_cm, num_lm): assert status == 200 output = json.loads(content) assert len(output["entry"][0]["content"]["member"]) == num_sh + + def uds_enabled(self, container_name): + # Check for cli.socket file + exec_command = self.client.exec_create(container_name, "ls /opt/splunkforwarder/var/run/splunk") + file_output = self.client.exec_start(exec_command) + return "cli.socket" in file_output diff --git a/tests/test_single_splunk_image.py b/tests/test_single_splunk_image.py index 0743fe1f..1a7319f2 100644 --- a/tests/test_single_splunk_image.py +++ b/tests/test_single_splunk_image.py @@ -2778,3 +2778,16 @@ def test_compose_1hf_splunk_add(self): assert self.check_splunkd("admin", self.password) # Check Splunkd using the new users assert self.check_splunkd("jerry", "seinfeld") + + def test_compose_1uf_uds(self): + # TODO: perform curl request using unix socket once bug is resolved: https://splunk.atlassian.net/browse/SPL-257022 + self.compose_file_name = "1uf_uds.yaml" + self.project_name = self.generate_random_string() + container_count, rc = self.compose_up() + assert rc == 0 + assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) + container_name = "{}_uf1_1".format(self.project_name) + output = self.get_container_logs() + self.check_ansible(output) + # Check that cli.socket file is created + assert self.uds_enabled(container_name) From cfe8ab076e7422d2a91b3395a9b245df4fd393a5 Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Mon, 17 Jun 2024 14:36:02 -0700 Subject: [PATCH 5/8] use create_container instead of docker-compose --- tests/requirements.txt | 2 +- tests/test_single_splunk_image.py | 37 ++++++++++++++++++++++--------- 2 files changed, 27 insertions(+), 12 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 0fff1856..6b5520e7 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,6 +1,6 @@ pytest==4.4.0 pyrsistent==0.16.1 -requests +requests==2.31.0 docker PyYAML docker-compose diff --git a/tests/test_single_splunk_image.py b/tests/test_single_splunk_image.py index 1a7319f2..0fc9de93 100644 --- a/tests/test_single_splunk_image.py +++ b/tests/test_single_splunk_image.py @@ -2780,14 +2780,29 @@ def test_compose_1hf_splunk_add(self): assert self.check_splunkd("jerry", "seinfeld") def test_compose_1uf_uds(self): - # TODO: perform curl request using unix socket once bug is resolved: https://splunk.atlassian.net/browse/SPL-257022 - self.compose_file_name = "1uf_uds.yaml" - self.project_name = self.generate_random_string() - container_count, rc = self.compose_up() - assert rc == 0 - assert self.wait_for_containers(container_count, label="com.docker.compose.project={}".format(self.project_name)) - container_name = "{}_uf1_1".format(self.project_name) - output = self.get_container_logs() - self.check_ansible(output) - # Check that cli.socket file is created - assert self.uds_enabled(container_name) + container_name = self.generate_random_string() + self.DIR = os.path.join(self.FIXTURES_DIR, container_name) + cid = None + try: + cid = self.client.create_container(self.UF_IMAGE_NAME, tty=True, command="start", + volumes=["/tmp/defaults/"], name=container_name, + environment={"DEBUG": "true", "SPLUNK_START_ARGS": "--accept-license", + "SPLUNK_PASSWORD": "foobar", "ENABLE_TCP_MODE": "false"}, + host_config=self.client.create_host_config(binds=[ + os.path.join(self.FIXTURES_DIR, container_name) + ":/tmp/defaults/"]) + ) + cid = cid.get("Id") + self.client.start(cid) + assert self.wait_for_containers(1, name=container_name) + assert self.uds_enabled(container_name) + except Exception as e: + self.logger.error(e) + raise e + finally: + if cid: + self.client.remove_container(cid, v=True, force=True) + try: + os.remove(os.path.join(self.DIR, "default.yml")) + os.rmdir(self.DIR) + except OSError: + pass From a7c11cbb4e9c9f51763809e0027381bf0ad6c6af Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Mon, 17 Jun 2024 14:48:52 -0700 Subject: [PATCH 6/8] add decoding, fix password --- tests/executor.py | 10 ++++++++-- tests/test_single_splunk_image.py | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/tests/executor.py b/tests/executor.py index 856cf687..de88ef92 100644 --- a/tests/executor.py +++ b/tests/executor.py @@ -61,7 +61,7 @@ def setup_class(cls, platform): # Define images by name to be validated cls.BASE_IMAGE_NAME = "base-{}".format(platform) cls.SPLUNK_IMAGE_NAME = "splunk-{}".format(platform) - cls.UF_IMAGE_NAME = "uf-{}".format(platform) + cls.UF_IMAGE_NAME = "uf-redhat-8" # Define new, random password for each executor cls.password = Executor.generate_random_string() cls.compose_file_name = None @@ -101,6 +101,8 @@ def get_container_logs(self, container_id): stream = self.client.logs(container_id, stream=True) output = "" for char in stream: + if type(char) is bytes: + char = char.decode("utf-8") if "Ansible playbook complete" in char: break output += char @@ -148,6 +150,8 @@ def wait_for_containers(self, count, label=None, name=None, timeout=500): # The healthcheck on our Splunk image is not reliable - resorting to checking logs if container.get("Labels", {}).get("maintainer") == "support@splunk.com": output = self.client.logs(container["Id"], tail=5) + if type(output) is bytes: + output = output.decode("utf-8") if "unable to" in output or "denied" in output or "splunkd.pid file is unreadable" in output: self.logger.error("Container {} did not start properly, last log line: {}".format(container["Names"][0], output)) elif "Ansible playbook complete" in output: @@ -231,7 +235,9 @@ def extract_json(self, container_name): retries = 15 for i in range(retries): exec_command = self.client.exec_create(container_name, "cat /opt/container_artifact/ansible_inventory.json") - json_data = self.client.exec_start(exec_command) + json_data = self.client.exec_start(exec_command["Id"]) + if type(json_data) is bytes: + json_data = json_data.decode("utf-8") if "No such file or directory" in json_data: time.sleep(5) else: diff --git a/tests/test_single_splunk_image.py b/tests/test_single_splunk_image.py index 0fc9de93..30181297 100644 --- a/tests/test_single_splunk_image.py +++ b/tests/test_single_splunk_image.py @@ -2787,7 +2787,7 @@ def test_compose_1uf_uds(self): cid = self.client.create_container(self.UF_IMAGE_NAME, tty=True, command="start", volumes=["/tmp/defaults/"], name=container_name, environment={"DEBUG": "true", "SPLUNK_START_ARGS": "--accept-license", - "SPLUNK_PASSWORD": "foobar", "ENABLE_TCP_MODE": "false"}, + "SPLUNK_PASSWORD": "Changeme", "ENABLE_TCP_MODE": "false"}, host_config=self.client.create_host_config(binds=[ os.path.join(self.FIXTURES_DIR, container_name) + ":/tmp/defaults/"]) ) From abadbcd7f5cdacfde9fec6756fa611eccd8cb624 Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Mon, 17 Jun 2024 15:37:22 -0700 Subject: [PATCH 7/8] revert uf image --- tests/executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/executor.py b/tests/executor.py index de88ef92..b92fb593 100644 --- a/tests/executor.py +++ b/tests/executor.py @@ -61,7 +61,7 @@ def setup_class(cls, platform): # Define images by name to be validated cls.BASE_IMAGE_NAME = "base-{}".format(platform) cls.SPLUNK_IMAGE_NAME = "splunk-{}".format(platform) - cls.UF_IMAGE_NAME = "uf-redhat-8" + cls.UF_IMAGE_NAME = "uf-{}".format(platform) # Define new, random password for each executor cls.password = Executor.generate_random_string() cls.compose_file_name = None From 7da21c00d141edf29ce94c3a2cd09e7bbff50886 Mon Sep 17 00:00:00 2001 From: Jack Meixensperger Date: Tue, 18 Jun 2024 10:25:21 -0700 Subject: [PATCH 8/8] fix file check + output fetch, add output assertion --- tests/executor.py | 10 +++++----- tests/test_single_splunk_image.py | 4 +++- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/tests/executor.py b/tests/executor.py index b92fb593..a34ee162 100644 --- a/tests/executor.py +++ b/tests/executor.py @@ -103,9 +103,9 @@ def get_container_logs(self, container_id): for char in stream: if type(char) is bytes: char = char.decode("utf-8") - if "Ansible playbook complete" in char: - break output += char + if "Ansible playbook complete" in output: + break return output def cleanup_files(self, files): @@ -387,8 +387,8 @@ def check_dmc_groups(self, splunkd_port, num_idx, num_sh, num_cm, num_lm): output = json.loads(content) assert len(output["entry"][0]["content"]["member"]) == num_sh - def uds_enabled(self, container_name): + def check_uds_socket_file(self, container_name): # Check for cli.socket file - exec_command = self.client.exec_create(container_name, "ls /opt/splunkforwarder/var/run/splunk") + exec_command = self.client.exec_create(container_name, "ls /opt/splunkforwarder/var/run/splunk", user="splunk") file_output = self.client.exec_start(exec_command) - return "cli.socket" in file_output + return "cli.socket" in file_output.decode("utf-8") diff --git a/tests/test_single_splunk_image.py b/tests/test_single_splunk_image.py index 30181297..48e788dd 100644 --- a/tests/test_single_splunk_image.py +++ b/tests/test_single_splunk_image.py @@ -2794,7 +2794,9 @@ def test_compose_1uf_uds(self): cid = cid.get("Id") self.client.start(cid) assert self.wait_for_containers(1, name=container_name) - assert self.uds_enabled(container_name) + output = self.get_container_logs(cid) + assert "Allows UDS" in output + assert self.check_uds_socket_file(container_name) except Exception as e: self.logger.error(e) raise e