diff --git a/R/geom-label.R b/R/geom-label.R
index d83434b386..c55ed6a332 100644
--- a/R/geom-label.R
+++ b/R/geom-label.R
@@ -143,20 +143,27 @@ labelGrob <- function(label, x = unit(0.5, "npc"), y = unit(0.5, "npc"),
   descent <- font_descent(
     text.gp$fontfamily, text.gp$fontface, text.gp$fontsize, text.gp$cex
   )
+  # To balance labels, we ensure the top includes at least the descent height
+  # and subtract the descent height from the bottom padding
+  padding[1] <- unit.pmax(padding[1], descent)
+  padding[3] <- unit.pmax(padding[3] - descent, unit(0, "pt"))
+
   hjust <- resolveHJust(just, NULL)
   vjust <- resolveVJust(just, NULL)
 
   text <- titleGrob(
-    label = label, hjust = hjust, vjust = vjust, x = x, y = y,
+    label = label, hjust = hjust, vjust = vjust, x = x,
+    y = y + (1 - vjust) * descent,
     margin = padding, margin_x = TRUE, margin_y = TRUE,
     gp = text.gp
   )
 
+  height <- heightDetails(text)
   box <- roundrectGrob(
-    x = x, y = y - (1 - vjust) * descent,
+    x = x, y = y + (0.5 - vjust) * height,
     width  = widthDetails(text),
-    height = heightDetails(text),
-    just   = c(hjust, vjust),
+    height = height,
+    just   = c(hjust, 0.5),
     r = r, gp = rect.gp, name = "box"
   )
 
diff --git a/tests/testthat/_snaps/geom-sf/labels-for-north-carolina.svg b/tests/testthat/_snaps/geom-sf/labels-for-north-carolina.svg
index ab6663e489..cf5ffdbbf2 100644
--- a/tests/testthat/_snaps/geom-sf/labels-for-north-carolina.svg
+++ b/tests/testthat/_snaps/geom-sf/labels-for-north-carolina.svg
@@ -27,7 +27,7 @@
 </defs>
 <g clip-path='url(#cpNTcuMjd8NzE0LjUyfDIyLjc4fDU0NS4xMQ==)'>
 <rect x='57.27' y='22.78' width='657.25' height='522.33' style='stroke-width: 1.07; stroke: none; fill: #FFFFFF;' />
-<polygon points='372.60,293.35 399.19,293.35 399.11,293.35 399.43,293.34 399.74,293.27 400.04,293.16 400.32,293.00 400.57,292.80 400.78,292.56 400.95,292.29 401.08,291.99 401.15,291.68 401.18,291.36 401.18,291.36 401.18,278.82 401.18,278.82 401.15,278.51 401.08,278.20 400.95,277.90 400.78,277.63 400.57,277.39 400.32,277.19 400.04,277.03 399.74,276.92 399.43,276.85 399.19,276.84 372.60,276.84 372.84,276.85 372.52,276.84 372.20,276.88 371.89,276.97 371.60,277.10 371.34,277.29 371.11,277.51 370.92,277.76 370.77,278.05 370.67,278.35 370.62,278.66 370.61,278.82 370.61,291.36 370.62,291.20 370.62,291.52 370.67,291.84 370.77,292.14 370.92,292.43 371.11,292.68 371.34,292.90 371.60,293.08 371.89,293.22 372.20,293.31 372.52,293.35 ' style='stroke-width: 0.53; fill: #FFFFFF;' />
+<polygon points='372.60,291.06 399.19,291.06 399.11,291.06 399.43,291.04 399.74,290.98 400.04,290.87 400.32,290.71 400.57,290.50 400.78,290.27 400.95,289.99 401.08,289.70 401.15,289.39 401.18,289.07 401.18,289.07 401.18,278.82 401.18,278.82 401.15,278.51 401.08,278.20 400.95,277.90 400.78,277.63 400.57,277.39 400.32,277.19 400.04,277.03 399.74,276.92 399.43,276.85 399.19,276.84 372.60,276.84 372.84,276.85 372.52,276.84 372.20,276.88 371.89,276.97 371.60,277.10 371.34,277.29 371.11,277.51 370.92,277.76 370.77,278.05 370.67,278.35 370.62,278.66 370.61,278.82 370.61,289.07 370.62,288.91 370.62,289.23 370.67,289.55 370.77,289.85 370.92,290.13 371.11,290.39 371.34,290.61 371.60,290.79 371.89,290.93 372.20,291.02 372.52,291.06 ' style='stroke-width: 0.53; fill: #FFFFFF;' />
 <text x='385.89' y='287.75' text-anchor='middle' style='font-size: 11.04px; font-family: sans;' textLength='23.95px' lengthAdjust='spacingAndGlyphs'>ashe</text>
 <rect x='57.27' y='22.78' width='657.25' height='522.33' style='stroke-width: 1.07; stroke: #333333;' />
 </g>