R: Manually change the color of stat_pvalue_manual?

I want to colour the significance bars below, so that the all the bars comparing VC across different doses are yellow and the ones comparing OJ across doses are grey. The ones comparing VC vs OJ on the same day can stay black.

It works when I do stat_pvalue_manual(color = 'supp') but I want to use my own color palette over the default ones.

# sample data
plot_df <- ToothGrowth

# Summarize the data
summary_df <- plot_df %>%
  group_by(dose, supp) %>%
  summarise(
    Mean = mean(len),
    SD = sd(len),
    SE = sd(len) / sqrt(n())  # Standard Error
  )

# p values for supp between dose
p_sup <- plot_df %>%
  group_by(supp) %>%
  t_test(len ~ dose, p.adjust.method = "fdr")

p_sup <- p_sup %>% 
  add_xy_position(x = "dose", group = "supp")

# Group the data by dose and then compare the levels of the supps variable
p_dose <- plot_df %>%
  group_by(dose) %>%
  t_test(len ~ supp) %>%
  adjust_pvalue(method = "fdr") %>%
  add_significance("p.adj")
p_dose

# Add p-values onto the box plots
p_dose <- p_dose %>%
  add_xy_position(x = "dose")

# change dose to character
summary_df$dose <- as.character(summary_df$dose)

ggplot(summary_df, aes(x = dose, y = Mean, fill = supp)) +
  geom_bar(stat = "identity", color = "black", position = position_dodge()) +
  geom_errorbar(aes(ymin = Mean - SE, ymax = Mean + SE), width = 0.2, position = position_dodge(.9)) +
  theme_minimal() +
  scale_fill_manual(values=c('#999999','#E69F00')) +
  labs(title = 'ToothGrowth',
       x = "dose",
       y = "len ± SE")+
  theme(
    plot.title = element_text(hjust = 0.5)
  )+
  stat_pvalue_manual(
    p_sup,
    tip.length = 0.01, 
    hide.ns = TRUE
  )+
  stat_pvalue_manual(
    p_dose, tip.length = 0.01,
    hide.ns = TRUE,
    inherit.aes = FALSE
  )

Similar to scale_fill_manual you can specify a custom color scale using scale_color_manual or simply add aesthetics = c("color", "fill") to scale_fill_manual to use the same palette for both aesthetics:

library(ggplot2)
library(rstatix)
library(ggpubr)

ggplot(summary_df, aes(x = dose, y = Mean, fill = supp)) +
  geom_bar(stat = "identity", color = "black", position = position_dodge()) +
  geom_errorbar(aes(ymin = Mean - SE, ymax = Mean + SE), width = 0.2, position = position_dodge(.9)) +
  theme_minimal() +
  scale_fill_manual(
    values = c("#999999", "#E69F00"),
    aesthetics = c("color", "fill")
  ) +
  labs(
    title = "ToothGrowth",
    x = "dose",
    y = "len ± SE"
  ) +
  theme(
    plot.title = element_text(hjust = 0.5)
  ) +
  stat_pvalue_manual(
    p_sup,
    tip.length = 0.01,
    hide.ns = TRUE,
    color = "supp"
  ) +
  stat_pvalue_manual(
    p_dose,
    tip.length = 0.01,
    hide.ns = TRUE,
    inherit.aes = FALSE
  )