Merge pull request #621 from stackhpc/update/timestamps

Update dnf repos using latest Pulp timestamps (plus tooling)

Author: bertiethorpe

ansible/ci/update_timestamps.yml

@@ -0,0 +1,16 @@
+- hosts: localhost
+  tasks:
+    - name: Get latest timestamps from sources
+      latest_timestamps:
+        repos_dict: "{{ appliances_pulp_repos }}"
+        content_url: "https://ark.stackhpc.com/pulp/content"
+      register: _result
+
+    - name: Overwrite repo timestamps with latest
+      ansible.builtin.copy:
+        dest: "{{ appliances_repository_root }}/environments/common/inventory/group_vars/all/timestamps.yml"
+        content: "{{ repo_template | to_nice_yaml(indent=2) }}"
+        backup: true
+      vars:
+        repo_template:
+          appliances_pulp_repos: "{{ _result.timestamps }}"

ansible/library/latest_timestamps.py

@@ -0,0 +1,76 @@
+__metaclass__ = type
+
+DOCUMENTATION = r'''
+---
+module: latest_timestamps
+short_description: Gets the latest set of snapshots from Pulp
+version_added: "1.0.0"
+description: Gets the latest set of snapshots from given source URLs and returns dictionary to replace 'appliances_repo_timestamps' with
+author:
+    - William Tripp
+    - Steve Brasier
+'''
+
+EXAMPLES = r'''
+- name: Get latest timestamps
+  latest_timestamps:
+    repos_dict: "{{ appliances_repo_timestamp_sources }}"
+    content_url: "https://ark.stackhpc.com/pulp/content"
+  register: result
+'''
+
+RETURN = r'''
+latest_dict:
+    description: Dictionary with updated timestamps
+    type: dict
+    returned: always
+changed_timestamps:
+    description: List of repos that have updated timestamps
+    type: str[]
+    returned: always
+'''
+
+from ansible.module_utils.basic import AnsibleModule
+import requests
+from bs4 import BeautifulSoup
+
+def run_module():
+    module_args = dict(
+        repos_dict=dict(type='dict', required=True),
+        content_url=dict(type='str', required=True)
+    )
+
+    result = dict(
+        changed=False,
+        original_message='',
+        message=''
+    )
+
+    module = AnsibleModule(
+        argument_spec=module_args,
+        supports_check_mode=True
+    )
+
+    timestamps = dict(module.params['repos_dict'])
+    for repo in timestamps:
+        for version in timestamps[repo]:
+
+            html_txt = requests.get(
+                    url= module.params['content_url'] + '/' + timestamps[repo][version]['path']
+                ).text
+            timestamp_link_list = BeautifulSoup(html_txt,features="html.parser").body.find('pre').find_all() # getting raw list of timestamps from html
+            timestamp_link_list = map(lambda x: x.string,timestamp_link_list) # stripping xml tags
+            latest_timestamp = list(timestamp_link_list)[-1][:-1] # last timestamp in list with trailing / removed
+            timestamps[repo][version]['timestamp'] = latest_timestamp
+            
+    result['timestamps'] = dict(sorted(timestamps.items()))
+
+    module.exit_json(**result)
+
+
+def main():
+    run_module()
+
+
+if __name__ == '__main__':
+    main()

environments/.stackhpc/ansible.cfg

@@ -10,6 +10,7 @@ inventory = ../common/inventory,inventory
 collections_path = ../../ansible/collections
 roles_path = ../../ansible/roles
 filter_plugins = ../../ansible/filter_plugins
+library = ../../ansible/library
 
 [ssh_connection]
 ssh_args = -o ServerAliveInterval=10 -o ControlMaster=auto -o ControlPath=~/.ssh/%r@%h-%p -o ControlPersist=240s -o PreferredAuthentications=publickey -o UserKnownHostsFile=/dev/null

environments/.stackhpc/tofu/cluster_image.auto.tfvars.json

@@ -1,6 +1,6 @@
 {
     "cluster_image": {
-        "RL8": "openhpc-RL8-250331-1627-cccd6c9c",
-        "RL9": "openhpc-RL9-250331-1627-cccd6c9c"
+        "RL8": "openhpc-RL8-250401-1100-9a3cffdb",
+        "RL9": "openhpc-RL9-250401-1100-9a3cffdb"
     }
 }

environments/common/inventory/group_vars/all/defaults.yml

@@ -98,75 +98,3 @@ appliances_extra_packages_default:
 appliances_extra_packages_other: []
 
 appliances_extra_packages: "{{ appliances_extra_packages_default + appliances_extra_packages_other }}"
-
-###################### ark repo timestamps ###################################################
-
-appliances_pulp_repos:
-  baseos:
-    '9.4':
-      timestamp: 20241115T011711
-      path: rocky/9.4/BaseOS/x86_64/os
-    '9.5':
-      timestamp: 20241216T013503
-      path: rocky/9.5/BaseOS/x86_64/os
-    '8.10':
-      timestamp: 20241217T123729
-      path: rocky/8.10/BaseOS/x86_64/os
-  appstream:
-    '9.4':
-      timestamp: 20241112T003151
-      path: rocky/9.4/AppStream/x86_64/os
-    '9.5':
-      timestamp: 20241217T005008
-      path: rocky/9.5/AppStream/x86_64/os
-    '8.10':
-      timestamp: 20241217T123729
-      path: rocky/8.10/AppStream/x86_64/os
-  crb:
-    '9.4':
-      timestamp: 20241115T003133
-      path: rocky/9.4/CRB/x86_64/os
-    '9.5':
-      timestamp: 20241217T005008
-      path: rocky/9.5/CRB/x86_64/os
-    '8.10':
-      timestamp: 20241217T123729
-      path: rocky/8.10/PowerTools/x86_64/os
-  extras:
-    '9.4':
-      timestamp: 20241118T002802
-      path: rocky/9.4/extras/x86_64/os
-    '9.5':
-      timestamp: 20241218T004632
-      path: rocky/9.5/extras/x86_64/os
-    '8.10':
-      timestamp: 20241217T123729
-      path: rocky/8.10/extras/x86_64/os
-  epel:
-    '9':
-      timestamp: 20241213T010218
-      path: epel/9/Everything/x86_64
-    '8':
-      timestamp: 20241216T235733
-      path: epel/8/Everything/x86_64
-  openhpc_base:
-    '8':
-      path: OpenHPC/2/EL_8
-      timestamp: 20241218T154614
-    '9':
-      path: OpenHPC/3/EL_9
-      timestamp: 20241218T154614
-  openhpc_updates:
-    '8':
-      path: OpenHPC/2/updates/EL_8
-      timestamp: 20241218T154614
-    '9':
-      path: OpenHPC/3/updates/EL_9
-      timestamp: 20241218T154614
-  ceph:
-    '8':
-      timestamp: 20231104T015751
-      path: centos/8-stream/storage/x86_64/ceph-quincy
-    '9':
-      timestamp: 20240923T233036
-      path: centos/9-stream/storage/x86_64/ceph-reef

environments/common/inventory/group_vars/all/timestamps.yml

@@ -0,0 +1,69 @@
+appliances_pulp_repos:
+  appstream:
+    '8.10':
+      path: rocky/8.10/AppStream/x86_64/os
+      timestamp: 20250328T030013
+    '9.4':
+      path: rocky/9.4/AppStream/x86_64/os
+      timestamp: 20241112T003151
+    '9.5':
+      path: rocky/9.5/AppStream/x86_64/os
+      timestamp: 20250328T031822
+  baseos:
+    '8.10':
+      path: rocky/8.10/BaseOS/x86_64/os
+      timestamp: 20250328T030013
+    '9.4':
+      path: rocky/9.4/BaseOS/x86_64/os
+      timestamp: 20241115T011711
+    '9.5':
+      path: rocky/9.5/BaseOS/x86_64/os
+      timestamp: 20250326T030636
+  ceph:
+    '8':
+      path: centos/8-stream/storage/x86_64/ceph-quincy
+      timestamp: 20231104T015751
+    '9':
+      path: centos/9-stream/storage/x86_64/ceph-reef
+      timestamp: 20240923T233036
+  crb:
+    '8.10':
+      path: rocky/8.10/PowerTools/x86_64/os
+      timestamp: 20250328T030013
+    '9.4':
+      path: rocky/9.4/CRB/x86_64/os
+      timestamp: 20241115T003133
+    '9.5':
+      path: rocky/9.5/CRB/x86_64/os
+      timestamp: 20250325T031428
+  epel:
+    '8':
+      path: epel/8/Everything/x86_64
+      timestamp: 20250326T000103
+    '9':
+      path: epel/9/Everything/x86_64
+      timestamp: 20250326T000103
+  extras:
+    '8.10':
+      path: rocky/8.10/extras/x86_64/os
+      timestamp: 20250327T030422
+    '9.4':
+      path: rocky/9.4/extras/x86_64/os
+      timestamp: 20241118T002802
+    '9.5':
+      path: rocky/9.5/extras/x86_64/os
+      timestamp: 20250328T031822
+  openhpc_base:
+    '8':
+      path: OpenHPC/2/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/EL_9
+      timestamp: 20241218T154614
+  openhpc_updates:
+    '8':
+      path: OpenHPC/2/updates/EL_8
+      timestamp: 20241218T154614
+    '9':
+      path: OpenHPC/3/updates/EL_9
+      timestamp: 20241218T154614

requirements.txt

@@ -10,3 +10,4 @@ selinux # this is a shim to avoid having to use --system-site-packages, you stil
 netaddr
 matplotlib
 pulp-cli==0.23.2
+beautifulsoup4==4.13.3