c/partition_balancer: track detailed information about failed reallocations

When partition replicas area moved in the cluster from whichever reason
Redpanda should expose all information required to identify issues
experienced during that operation. Previously partition balancer planner
included only the basic information about the failures in the status.
That was the list of `ntps` that replicas failed to be reallocated and
the total number of reallocations failures. In order to obtain more
detailed information looking into the partition balancer log was
required. Moreover the reallocations that were not even attempted during
preparing an allocation plan were not reported at all. This lead to
confusion as f.e. partitions with only one replica on the node that is
offline that can not be moved was not reported ans the failed one.

In order to address this problem we introduced a new
`allocation_failure_details` map in the partition balancer planner
result. The map provides a detailed information about the failed
reallocation. Additionally reallocation failure is reported when
partition is marked as `immutable` in the process of planning balancer
action. This makes it easy for the operator to identify any partition
balancer issues without the need to look for the specific log entries.

The change is fully backward compatible as all the changes are
incremental i.e. when no detailed information about failures is
available Redpanda will fallback to the old basic `ntp` list.

Signed-off-by: Michał Maślanka <[email protected]>

Author: mmaslankaprv

src/v/cluster/partition_balancer_backend.cc

@@ -414,8 +414,8 @@ ss::future<> partition_balancer_backend::do_tick() {
 
     _cur_term->last_tick_time = clock_t::now();
     _cur_term->last_violations = std::move(plan_data.violations);
-    _cur_term->last_tick_decommission_realloc_failures = std::move(
-      plan_data.decommission_realloc_failures);
+    _cur_term->last_tick_reallocation_failures = std::move(
+      plan_data.reallocation_failures);
     if (
       _state.topics().has_updates_in_progress()
       || plan_data.status == planner_status::actions_planned) {
@@ -571,8 +571,7 @@ partition_balancer_overview_reply partition_balancer_backend::overview() const {
 
     ret.status = _cur_term->last_status;
     ret.violations = _cur_term->last_violations;
-    ret.decommission_realloc_failures
-      = _cur_term->last_tick_decommission_realloc_failures;
+    ret.set_reallocation_failures(_cur_term->last_tick_reallocation_failures);
     ret.partitions_pending_force_recovery_count
       = _state.topics().partitions_to_force_recover().size();
     if (ret.partitions_pending_force_recovery_count > 0) {

src/v/cluster/partition_balancer_backend.h

@@ -139,8 +139,8 @@ class partition_balancer_backend {
           = partition_balancer_status::starting;
         size_t last_tick_in_progress_updates = 0;
 
-        absl::flat_hash_map<model::node_id, absl::btree_set<model::ntp>>
-          last_tick_decommission_realloc_failures;
+        chunked_hash_map<model::ntp, reallocation_failure_details>
+          last_tick_reallocation_failures;
 
         bool _ondemand_rebalance_requested = false;
         bool _force_health_report_refresh = false;

src/v/cluster/partition_balancer_planner.cc

@@ -66,6 +66,33 @@ distinct_from(const absl::flat_hash_set<model::node_id>& nodes) {
     return hard_constraint(std::make_unique<impl>(nodes));
 }
 
+reallocation_failure_details map_result_to_failure_details(
+  change_reason reason, std::error_code ec, model::node_id replica_to_move) {
+    reallocation_failure_details details{
+      .replica_to_move = replica_to_move,
+      .reason = reason,
+      .error = details.error = reallocation_error::unknown_error};
+    if (ec.category() == cluster::error_category()) {
+        switch (static_cast<cluster::errc>(ec.value())) {
+        case cluster::errc::no_eligible_allocation_nodes:
+            details.error = reallocation_error::no_eligible_node_found;
+            break;
+        case cluster::errc::topic_invalid_partitions_fd_limit:
+            details.error = reallocation_error::over_partition_fd_limit;
+            break;
+        case cluster::errc::topic_invalid_partitions_core_limit:
+            details.error = reallocation_error::over_partition_core_limit;
+            break;
+        case cluster::errc::topic_invalid_partitions_memory_limit:
+            details.error = reallocation_error::over_partition_memory_limit;
+            break;
+        default:
+            break;
+        }
+    }
+
+    return details;
+}
 } // namespace
 
 partition_balancer_planner::partition_balancer_planner(
@@ -187,8 +214,8 @@ class partition_balancer_planner::request_context {
 
     void increment_missing_size_count() { _partitions_with_missing_size++; }
 
-    void
-    report_decommission_reallocation_failure(model::node_id, const model::ntp&);
+    void report_reallocation_failure(
+      const model::ntp&, reallocation_failure_details);
 
     node2count_t& get_topic_to_node_count(model::topic_namespace_view tp_ns) {
         auto it = _topic2node_counts.find(tp_ns);
@@ -240,10 +267,9 @@ class partition_balancer_planner::request_context {
     // we track missing partition size info separately as it requires force
     // refresh of health report
     size_t _partitions_with_missing_size = 0;
-    // Tracks ntps with allocation failures grouped by decommissioning node.
-    static constexpr size_t max_ntps_with_reallocation_falures = 25;
-    absl::flat_hash_map<model::node_id, absl::btree_set<model::ntp>>
-      _decommission_realloc_failures;
+    static constexpr size_t max_reallocation_failures_reported = 25;
+    chunked_hash_map<model::ntp, reallocation_failure_details>
+      _reallocation_failures;
     absl::node_hash_set<model::ntp> _cancellations;
     bool _counts_rebalancing_finished = false;
     ss::abort_source& _as;
@@ -544,13 +570,13 @@ bool partition_balancer_planner::request_context::increment_failure_count() {
     }
 }
 
-void partition_balancer_planner::request_context::
-  report_decommission_reallocation_failure(
-    model::node_id node, const model::ntp& ntp) {
-    auto& ntps = _decommission_realloc_failures.try_emplace(node).first->second;
-    if (ntps.size() < max_ntps_with_reallocation_falures) {
-        ntps.emplace(ntp);
+void partition_balancer_planner::request_context::report_reallocation_failure(
+  const model::ntp& ntp, reallocation_failure_details details) {
+    if (_reallocation_failures.size() >= max_reallocation_failures_reported) {
+        return;
     }
+
+    _reallocation_failures.try_emplace(ntp, details);
 }
 
 static bool has_quorum(
@@ -821,6 +847,54 @@ class partition_balancer_planner::immutable_partition {
           change_reason);
     }
 
+    bool contains_replica(model::node_id replica) const {
+        return contains_node(_replicas, replica);
+    }
+
+    reallocation_failure_details map_immutability_reason_to_failure_details(
+      change_reason reason, model::node_id replica_to_move) {
+        reallocation_failure_details details{
+          .replica_to_move = replica_to_move,
+          .reason = reason,
+          .error = details.error = reallocation_error::unknown_error};
+
+        switch (_reason) {
+        case immutability_reason::no_quorum:
+            details.error = reallocation_error::no_quorum;
+            break;
+        case immutability_reason::no_size_info:
+            details.error = reallocation_error::missing_partition_size_info;
+            break;
+        case immutability_reason::reconfiguration_state:
+            details.error = reallocation_error::reconfiguration_in_progress;
+            break;
+        case immutability_reason::disabled:
+            details.error = reallocation_error::partition_disabled;
+            break;
+        case immutability_reason::batch_full:
+            break;
+        }
+
+        return details;
+    }
+
+    void report_immutable_partition_as_reallocation_failure(
+      request_context& ctx,
+      const std::vector<model::node_id>& replicas_to_move,
+      change_reason reason) {
+        if (_reason == immutability_reason::batch_full) {
+            return;
+        }
+        // report per replica reallocation failures
+        for (const auto replica : replicas_to_move) {
+            if (contains_replica(replica)) {
+                ctx.report_reallocation_failure(
+                  ntp(),
+                  map_immutability_reason_to_failure_details(reason, replica));
+            }
+        }
+    }
+
 private:
     friend class request_context;
 
@@ -1497,8 +1571,10 @@ ss::future<> partition_balancer_planner::get_node_drain_actions(
                         ctx.config().hard_max_disk_usage_ratio,
                         reason);
                       if (!result) {
-                          ctx.report_decommission_reallocation_failure(
-                            replica, part.ntp());
+                          ctx.report_reallocation_failure(
+                            part.ntp(),
+                            map_result_to_failure_details(
+                              reason, result.error(), replica));
                       }
                   }
               }
@@ -1522,7 +1598,11 @@ ss::future<> partition_balancer_planner::get_node_drain_actions(
               }
           },
           [&](force_reassignable_partition&) {},
-          [&](immutable_partition& part) { part.report_failure(reason); });
+          [&](immutable_partition& part) {
+              part.report_failure(reason);
+              part.report_immutable_partition_as_reallocation_failure(
+                ctx, to_move, reason);
+          });
 
         return ss::stop_iteration::no;
     });
@@ -1589,15 +1669,25 @@ ss::future<> partition_balancer_planner::get_rack_constraint_repair_actions(
                       if (part.is_original(replica)) {
                           // only move replicas that haven't been moved for
                           // other reasons
-                          (void)part.move_replica(
+                          auto r = part.move_replica(
                             replica,
                             ctx.config().soft_max_disk_usage_ratio,
                             change_reason::rack_constraint_repair);
+                          if (r.has_error()) {
+                              ctx.report_reallocation_failure(
+                                part.ntp(),
+                                map_result_to_failure_details(
+                                  change_reason::rack_constraint_repair,
+                                  r.error(),
+                                  replica));
+                          }
                       }
                   }
               },
-              [](immutable_partition& part) {
+              [&](immutable_partition& part) {
                   part.report_failure(change_reason::rack_constraint_repair);
+                  part.report_immutable_partition_as_reallocation_failure(
+                    ctx, to_move, change_reason::rack_constraint_repair);
               },
               [](moving_partition&) {},
               [](force_reassignable_partition&) {});
@@ -1814,12 +1904,23 @@ partition_balancer_planner::get_full_node_actions(request_context& ctx) {
                         });
 
                       for (const auto& replica : full_node_replicas) {
-                          (void)part.move_replica(
+                          auto r = part.move_replica(
                             replica.node_id,
                             ctx.config().soft_max_disk_usage_ratio,
                             change_reason::disk_full);
+                          if (r.has_error()) {
+                              ctx.report_reallocation_failure(
+                                part.ntp(),
+                                map_result_to_failure_details(
+                                  change_reason::disk_full,
+                                  r.error(),
+                                  replica.node_id));
+                          }
                       }
                   },
+                  [&](immutable_partition& part) {
+                      part.report_failure(change_reason::disk_full);
+                  },
                   [](auto&) {});
             });
         }
@@ -1940,6 +2041,12 @@ ss::future<> partition_balancer_planner::get_counts_rebalancing_actions(
                   ctx.config().soft_max_disk_usage_ratio,
                   change_reason::partition_count_rebalancing);
                 if (!res) {
+                    ctx.report_reallocation_failure(
+                      part.ntp(),
+                      map_result_to_failure_details(
+                        change_reason::partition_count_rebalancing,
+                        res.error(),
+                        node));
                     return;
                 }
 
@@ -1959,6 +2066,8 @@ ss::future<> partition_balancer_planner::get_counts_rebalancing_actions(
             },
             [&](immutable_partition& p) {
                 p.report_failure(change_reason::partition_count_rebalancing);
+                p.report_immutable_partition_as_reallocation_failure(
+                  ctx, {node}, change_reason::partition_count_rebalancing);
                 should_stop = false;
             },
             [](auto&) {});
@@ -2048,8 +2157,7 @@ void partition_balancer_planner::request_context::collect_actions(
 
     result.counts_rebalancing_finished = _counts_rebalancing_finished;
 
-    result.decommission_realloc_failures = std::move(
-      _decommission_realloc_failures);
+    result.reallocation_failures = std::move(_reallocation_failures);
 
     if (
       !result.cancellations.empty() || !result.reassignments.empty()

src/v/cluster/partition_balancer_planner.h

@@ -84,11 +84,13 @@ class partition_balancer_planner {
         partition_balancer_violations violations;
         std::vector<ntp_reassignment> reassignments;
         std::vector<model::ntp> cancellations;
-        absl::flat_hash_map<model::node_id, absl::btree_set<model::ntp>>
-          decommission_realloc_failures;
+        chunked_hash_map<model::ntp, reallocation_failure_details>
+          reallocation_failures;
         bool counts_rebalancing_finished = false;
         size_t failed_actions_count = 0;
         status status = status::empty;
+
+        void maybe_add_reallocation_failure();
     };
 
     ss::future<plan_data>

src/v/cluster/partition_balancer_types.cc

@@ -75,6 +75,43 @@ partition_balancer_violations::partition_balancer_violations(
   : unavailable_nodes(std::move(un))
   , full_nodes(std::move(fn)) {}
 
+void partition_balancer_overview_reply::set_reallocation_failures(
+  const chunked_hash_map<model::ntp, reallocation_failure_details>&
+    reallocations) {
+    reallocation_failures.reserve(reallocations.size());
+    for (const auto& [ntp, details] : reallocations) {
+        reallocation_failures.emplace(ntp, details);
+        /**
+         * Fill in the decommission_realloc_failures map with the reallocation
+         * failures for backward compatibility.
+         */
+        if (details.reason == change_reason::node_decommissioning) {
+            auto& failed_ntps
+              = decommission_realloc_failures[details.replica_to_move];
+            failed_ntps.insert(ntp);
+        }
+    }
+}
+
+partition_balancer_overview_reply
+partition_balancer_overview_reply::copy() const {
+    partition_balancer_overview_reply copy;
+    copy.error = error;
+    copy.last_tick_time = last_tick_time;
+    copy.status = status;
+    copy.violations = violations;
+    copy.partitions_pending_force_recovery_count
+      = partitions_pending_force_recovery_count;
+    copy.partitions_pending_force_recovery_sample
+      = partitions_pending_force_recovery_sample;
+    copy.decommission_realloc_failures = decommission_realloc_failures;
+    copy.reallocation_failures.reserve(reallocation_failures.size());
+    for (const auto& [ntp, details] : reallocation_failures) {
+        copy.reallocation_failures.emplace(ntp, details);
+    }
+    return copy;
+}
+
 std::ostream&
 operator<<(std::ostream& o, const partition_balancer_violations& v) {
     fmt::print(
@@ -117,12 +154,16 @@ operator<<(std::ostream& o, const partition_balancer_overview_reply& rep) {
     fmt::print(
       o,
       "{{ error: {} last_tick_time: {} status: {} violations: {}, "
-      "partitions_pending_force_recovery: {}}}",
+      "partitions_pending_force_recovery: {}, "
+      "decommission_reallocation_failures_count: {}, reallocation_failures: "
+      "{}}}",
       rep.error,
       rep.last_tick_time,
       rep.status,
       rep.violations,
-      rep.partitions_pending_force_recovery_count);
+      rep.partitions_pending_force_recovery_count,
+      rep.decommission_realloc_failures.size(),
+      fmt::join(rep.reallocation_failures | std::views::values, ", "));
     return o;
 }