btl/uct: allow connections to be formed using a separate memory domain

It is possible that the current memory domain does not have an adequate
transport for forming endpoint to endpoint connections. When this is the case
the btl will fail to function. To support these situations this CL adds
support for using an alternate transport (usually tcp) which can be used to
make the endpoint connections.

Signed-off-by: Nathan Hjelm <[email protected]>

Author: hjelmn

opal/mca/btl/uct/btl_uct.h

@@ -64,6 +64,9 @@ struct mca_btl_uct_module_t {
     /** base BTL interface */
     mca_btl_base_module_t super;
 
+    /** module index in the component module array */
+    int module_index;
+
     /** whether the module has been fully initialized or not */
     bool initialized;
 
@@ -141,9 +144,15 @@ struct mca_btl_uct_component_t {
 
     /** allowed UCT memory domains */
     char *memory_domains;
+    mca_btl_uct_include_list_t memory_domain_list;
 
     /** allowed transports */
     char *allowed_transports;
+    mca_btl_uct_include_list_t allowed_transport_list;
+
+    /** transports to consider for forming connections */
+    char *connection_domains;
+    mca_btl_uct_include_list_t connection_domain_list;
 
     /** number of worker contexts to create */
     int num_contexts_per_module;
@@ -158,6 +167,10 @@ struct mca_btl_uct_component_t {
 
     /** connection retry timeout */
     unsigned int connection_retry_timeout;
+
+    /** alternate connection-only module that can be used if no suitable
+     * connection tl is found. this is usually a tcp tl. */
+    mca_btl_uct_module_t *conn_module;
 };
 typedef struct mca_btl_uct_component_t mca_btl_uct_component_t;
 
@@ -294,7 +307,8 @@ struct mca_btl_base_endpoint_t *mca_btl_uct_get_ep(struct mca_btl_base_module_t
                                                    opal_proc_t *proc);
 
 int mca_btl_uct_query_tls(mca_btl_uct_module_t *module, mca_btl_uct_md_t *md,
-                          uct_tl_resource_desc_t *tl_descs, unsigned tl_count);
+                          uct_tl_resource_desc_t *tl_descs, unsigned tl_count,
+                          bool evaluate_for_conn_only);
 int mca_btl_uct_process_connection_request(mca_btl_uct_module_t *module,
                                            mca_btl_uct_conn_req_t *req);
 
@@ -341,5 +355,15 @@ static inline bool mca_btl_uct_tl_requires_connection_tl(mca_btl_uct_tl_t *tl)
     return !(MCA_BTL_UCT_TL_ATTR(tl, 0).cap.flags & UCT_IFACE_FLAG_CONNECT_TO_IFACE);
 }
 
+/**
+ * @brief Find the rank of `name` in the include list `list`.
+ *
+ * @param[in] name   name to find
+ * @param[in] list   list to search
+ *
+ * A negative result means the name is not present or the list is negated.
+ */
+int mca_btl_uct_include_list_rank (const char *name, const mca_btl_uct_include_list_t *list);
+
 END_C_DECLS
 #endif

opal/mca/btl/uct/btl_uct_am.c

@@ -55,7 +55,6 @@ static inline void _mca_btl_uct_send_pack(void *data, void *header, size_t heade
 {
     uint32_t iov_count = 1;
     struct iovec iov;
-    size_t length;
 
     if (header_size > 0) {
         assert(NULL != header);