Support additional nodegroups (#704)

* Support fixed IP addresses for control node

The variable control_ip_address was documented but not implemented.
Since we support multiple networks, change it to control_ip_addresses
and implement it.

Closes #642.

* Update environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

* tf style tweaks

* update production docs for control_ip_addresses

* fix control IP address logic

* add validation for control_ip_addresses

* remove fixed IPs from production docs - not standard process

* support ip_addresses fo all nodes

* make stackhpc tofu format consistent

* add support for additional_nodegroups

* add missing additional nodegroup file

* rename additional tf file for consistency

* support changing security groups for additional nodes

* add node_fqdn from PR#702 to additional_nodes

---------

Co-authored-by: Pierre Riteau <[email protected]>

Author: sjpb

environments/skeleton/{{cookiecutter.environment}}/tofu/additional.tf

@@ -0,0 +1,68 @@
+module "additional" {
+  source = "./node_group"
+
+  for_each = var.additional_nodegroups
+
+  # must be set for group:
+  nodes = each.value.nodes
+  flavor = each.value.flavor
+
+  # always taken from top-level value:
+  cluster_name = var.cluster_name
+  cluster_domain_suffix = var.cluster_domain_suffix
+  key_pair = var.key_pair
+  environment_root = var.environment_root
+  
+  # can be set for group, defaults to top-level value:
+  image_id = lookup(each.value, "image_id", var.cluster_image_id)
+  vnic_types = lookup(each.value, "vnic_types", var.vnic_types)
+  volume_backed_instances = lookup(each.value, "volume_backed_instances", var.volume_backed_instances)
+  root_volume_size = lookup(each.value, "root_volume_size", var.root_volume_size)
+  root_volume_type = lookup(each.value, "root_volume_type", var.root_volume_type)
+  gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip)
+  nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template)
+  
+  # optionally set for group:
+  networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
+  # here null means "use module var default"
+  extra_volumes = lookup(each.value, "extra_volumes", null)
+  fip_addresses = lookup(each.value, "fip_addresses", null)
+  fip_network = lookup(each.value, "fip_network", null)
+  match_ironic_node = lookup(each.value, "match_ironic_node", null)
+  availability_zone = lookup(each.value, "availability_zone", null)
+  ip_addresses = lookup(each.value, "ip_addresses", null)
+  security_group_ids = lookup(each.value, "security_group_ids", [for o in data.openstack_networking_secgroup_v2.nonlogin: o.id])
+
+  # can't be set for additional nodes
+  compute_init_enable = []
+  ignore_image_changes = false
+
+  # computed
+  # not using openstack_compute_instance_v2.control.access_ip_v4 to avoid
+  # updates to node metadata on deletion/recreation of the control node:
+  control_address = openstack_networking_port_v2.control[var.cluster_networks[0].network].all_fixed_ips[0]
+  baremetal_nodes = data.external.baremetal_nodes.result
+
+  # input dict validation:
+  group_name = each.key
+  group_keys = keys(each.value)
+  allowed_keys = [
+    "nodes",
+    "flavor",
+    "image_id",
+    "extra_networks",
+    "vnic_types",
+    "volume_backed_instances",
+    "root_volume_size",
+    "root_volume_type",
+    "extra_volumes",
+    "fip_addresses",
+    "fip_network",
+    "match_ironic_node",
+    "availability_zone",
+    "ip_addresses",
+    "gateway_ip",
+    "nodename_template",
+    "security_group_ids",
+  ]
+}

environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tf

@@ -7,6 +7,7 @@ resource "local_file" "hosts" {
                             "control_fqdn": local.control_fqdn
                             "login_groups": module.login
                             "compute_groups": module.compute
+                            "additional_groups": module.additional
                             "state_dir": var.state_dir
                             "cluster_home_volume": var.home_volume_provisioning != "none"
                           },

environments/skeleton/{{cookiecutter.environment}}/tofu/inventory.tpl

@@ -15,6 +15,7 @@ control:
     vars:
         appliances_state_dir: ${state_dir} # NB needs to be set on group not host otherwise it is ignored in packer build!
 
+# --- login nodes ---
 %{ for group_name in keys(login_groups) ~}
 ${cluster_name}_${group_name}:
     hosts:
@@ -34,6 +35,7 @@ login:
         ${cluster_name}_${group_name}:
 %{ endfor ~}
 
+# --- compute nodes ---
 %{ for group_name in keys(compute_groups) ~}
 ${cluster_name}_${group_name}:
     hosts:
@@ -47,10 +49,37 @@ ${cluster_name}_${group_name}:
     vars:
         # NB: this is the target image, not necessarily what is provisioned
         image_id: ${compute_groups[group_name]["image_id"]}
+
+${group_name}:
+    children:
+        ${cluster_name}_${group_name}:
+
 %{ endfor ~}
 
 compute:
     children:
 %{ for group_name in keys(compute_groups) ~}
         ${cluster_name}_${group_name}:
 %{ endfor ~}
+
+# --- additional nodes ---
+%{ for group_name in keys(additional_groups) ~}
+${cluster_name}_${group_name}:
+    hosts:
+%{ for nodename, node in additional_groups[group_name]["compute_instances"] ~}
+        ${ node.name }:
+            ansible_host: ${node.access_ip_v4}
+            instance_id: ${ node.id }
+            networks: ${jsonencode({for n in node.network: n.name => {"fixed_ip_v4": n.fixed_ip_v4, "fixed_ip_v6": n.fixed_ip_v6}})}
+            node_fqdn: ${additional_groups[group_name]["fqdns"][nodename]}
+%{ endfor ~}
+${group_name}:
+    children:
+        ${cluster_name}_${group_name}:
+
+%{ endfor ~}
+additional:
+    children:
+%{ for group_name in keys(additional_groups) ~}
+        ${cluster_name}_${group_name}:
+%{ endfor ~}

environments/skeleton/{{cookiecutter.environment}}/tofu/login.tf

@@ -22,7 +22,7 @@ module "login" {
   gateway_ip = lookup(each.value, "gateway_ip", var.gateway_ip)
   nodename_template = lookup(each.value, "nodename_template", var.cluster_nodename_template)
 
-  # optionally set for group
+  # optionally set for group:
   networks = concat(var.cluster_networks, lookup(each.value, "extra_networks", []))
   # here null means "use module var default"
   extra_volumes = lookup(each.value, "extra_volumes", null)

environments/skeleton/{{cookiecutter.environment}}/tofu/node_group/variables.tf

@@ -72,7 +72,8 @@ variable "extra_volumes" {
 }
 
 variable "security_group_ids" {
-    type = list
+    type = list(string)
+    nullable = false
 }
 
 variable "control_address" {

environments/skeleton/{{cookiecutter.environment}}/tofu/variables.tf

@@ -125,11 +125,38 @@ variable "compute" {
             availability_zone: Name of availability zone - ignored unless match_ironic_node is true (default: "nova")
             gateway_ip: Address to add default route via
             nodename_template: Overrides variable cluster_nodename_template
+
+        Nodes are added to the following inventory groups:
+        - $group_name
+        - $cluster_name + '_' + $group_name - this is used for the stackhpc.openhpc role
+        - 'compute'
     EOF
 
     type = any # can't do any better; TF type constraints can't cope with heterogeneous inner mappings
 }
 
+variable "additional_nodegroups" {
+    default = {}
+    description = <<-EOF
+        Mapping defining homogenous groups of nodes for arbitrary purposes.
+        These nodes are not in the compute or login inventory groups so they
+        will not run slurmd.
+
+        Keys are names of groups.
+        Values are a mapping as for the "login" variable, with the addition of
+        the optional entry:
+        
+            security_group_ids: List of strings giving IDs of security groups
+                                to apply. If not specified the groups from the
+                                variable nonlogin_security_groups are applied.
+
+        Nodes are added to the following inventory groups:
+        - $group_name
+        - $cluster_name + '_' + $group_name
+        - 'additional'
+    EOF
+}
+
 variable "environment_root" {
     type = string
     description = "Path to environment root, automatically set by activate script"