Geom_label conditional change color text and padding

I am creating a pie chart and I would like the color of the label text and padding to be white when the segment being labelled is filled with black. At the moment other labels follow the color of the segment labelled and I want to keep it so. The black filled segments also have black label text and can therefore not be seen, despite being there.

ggplot(pigs2.proportions,
       aes(x="", y=comp_percent, fill = percent_class, width = 0.4)) +
  geom_bar(stat="identity") +
  geom_label(
    aes(
      label = ifelse(comp_percent %% 1 != 0, sprintf("%.2f%%", comp_percent),
                     sprintf("%.0f%%", comp_percent))),
            position = position_stack(vjust = 0.5), 
    size = 4,
    label.padding = unit(0.25, "lines"),
    show.legend = FALSE) +
  coord_polar("y", start=0) +
  facet_wrap(~factor(source, levels = c("fish", "soy", "skim")), labeller = custom_labeller("source")) +
  scale_fill_manual(values = c("low" = "black", "moderate" = "darkgray", "high" = "lightgray")) +
  theme(
    axis.text.x = element_blank(), # Hide labels on polar coordinate axis
    panel.grid = element_blank(),  # Remove grid lines
    panel.border = element_blank(),  # Remove round outline
    axis.text = element_blank(),  # Remove axis text
    axis.title = element_blank()  # Remove axis titles
  )

structure(list(source = structure(c(1L, 1L, 1L, 2L, 2L, 2L, 3L, 
3L, 3L), levels = c("fish", "soy", "skim"), class = "factor"), 
    percent_class = structure(c(3L, 1L, 2L, 3L, 1L, 2L, 3L, 1L, 
    2L), levels = c("low", "moderate", "high"), class = "factor"), 
    count = c(3L, 2L, 5L, 1L, 3L, 6L, 1L, 3L, 5L), comp_percent = c(30, 
    20, 50, 10, 30, 60, 11.1111111111111, 33.3333333333333, 55.5555555555556
    ), pos = c(15, 10, 25, 35, 35, 80, 45.5555555555556, 66.6666666666667, 
    137.777777777778)), class = c("grouped_df", "tbl_df", "tbl", 
"data.frame"), row.names = c(NA, -9L), groups = structure(list(
    percent_class = structure(1:3, levels = c("low", "moderate", 
    "high"), class = "factor"), .rows = structure(list(c(2L, 
    5L, 8L), c(3L, 6L, 9L), c(1L, 4L, 7L)), ptype = integer(0), class = c("vctrs_list_of", 
    "vctrs_vctr", "list"))), class = c("tbl_df", "tbl", "data.frame"
), row.names = c(NA, -3L), .drop = TRUE))

As suggested by @AllanCameron in the comments you could e.g. map the condition percent_class == "low" on the color aes and set your desired text colors using scale_color_manual. However, doing so changes the grouping of the data for geom_label and hence the stacking of the labels no longer aligns with the bars. To fix that you have to explicitly map percent_class on the group aes.

library(ggplot2)

p <- ggplot(
  pigs2.proportions,
  aes(x = "", y = comp_percent, fill = percent_class, width = 0.4)
) +
  geom_bar(stat = "identity") +
  coord_polar("y", start = 0) +
  facet_wrap(
    ~ factor(source, levels = c("fish", "soy", "skim")),
    # labeller = custom_labeller("source")
  ) +
  scale_fill_manual(
    values = c("low" = "black", "moderate" = "darkgray", "high" = "lightgray")
  ) +
  theme(
    axis.text.x = element_blank(), # Hide labels on polar coordinate axis
    panel.grid = element_blank(), # Remove grid lines
    panel.border = element_blank(), # Remove round outline
    axis.text = element_blank(), # Remove axis text
    axis.title = element_blank() # Remove axis titles
  )

p +
  geom_label(
    aes(
      label = ifelse(comp_percent %% 1 != 0,
        sprintf("%.2f%%", comp_percent),
        sprintf("%.0f%%", comp_percent)
      ),
      color = percent_class == "low",
      group = percent_class
    ),
    position = position_stack(vjust = 0.5),
    size = 4,
    label.padding = unit(0.25, "lines"),
    show.legend = FALSE
  ) +
  scale_color_manual(values = c("black", "white"))

A second option would be to use after_scale to conditionally set the color based on the value of fill like so:

p +
  geom_label(
    aes(
      label = ifelse(comp_percent %% 1 != 0,
        sprintf("%.2f%%", comp_percent),
        sprintf("%.0f%%", comp_percent)
      ),
      color = after_scale(
        ifelse(fill == "black", "white", "black")
      )
    ),
    position = position_stack(vjust = 0.5),
    size = 4,
    label.padding = unit(0.25, "lines"),
    show.legend = FALSE
  )