Reordering color palette with factors after using forcats::fct_infreq()

I'm working on displaying some survey data using geom_histogram and want to sort by the occurrence of the factors using forcats::fct_infreq.

I'm using a custom color-palette to match a theme and unfortunately the sequential palette gets all jumbled. Just reordering the palette to match the order of the factors doesn't seem to work either for some reason.

Example below. Hoping someone can answer on a better approach or a simple solution for this relatively minor, but annoying, problem.

library(tidyverse)

df <- tibble(
  response = c(1,1,1,1,1,2,2,2,2,2,2,2,3,3,3,3,3,3,3,3,4,4,4,4,4,5,5,5,5,5,6,6,6,6,6,6,6,7,7,7,7,7,7,8,8,8,8,8,9,9,9,9,9,10, 10,10, 10,10, 10,10, 10,11, 12,12, 12,12, 12,13, 13,13, 13,13, 14,14, 14,14, 14,14, 14,14, 14,14, 15,15, 15,16, 16,17, 17,17, 18,18, 19,19, 19,20, 20,20, 20,20, 21,21, 21,21, 22,23, 23,23, 23,23, 24,24, 24,24, 25,25, 25,26, 26,27, 28,29, 29,29, 29,29, 29,30, 30,30, 30,30, 31,31, 32,32, 32,33, 33,33, 33,33, 34,34,34,34, 34,34, 34,34, 34,35, 35,35, 35,35, 35,36, 36,36, 36,1),
  Q4 = c("Ratings","Reviews","Reading challenge","Search books","My Year in Books","Shelves","Ratings","Reviews","Progress tracking","Reading challenge","Recommendations","Search books","Shelves","Ratings","Reviews","Reading challenge","Recommendations","Search books","Explore","My Year in Books","Ratings","Reviews","Progress tracking","Search books","Lists","Shelves","Ratings","Reading challenge","Lists","My Year in Books","Ratings","Reviews","Progress tracking","Reading challenge","Recommendations","Search books","My Year in Books","Shelves","Ratings","Reviews","Reading challenge","Search books","My Year in Books","Ratings","Reviews","Groups","Lists","Explore","Ratings","Reading challenge","Search books","Lists","My Year in Books","Ratings","Reviews","Progress tracking","Reading challenge","Recommendations","Search books","Explore","My Year in Books","Search books","Ratings","Reviews","Progress tracking","Reading challenge","Search books","Progress tracking","Reading challenge","Recommendations","Search books","My Year in Books","Shelves","Ratings","Reviews","Progress tracking","Groups","Reading challenge","Recommendations","Search books","Lists","My Year in Books","Ratings","Groups","Reading challenge","Reviews","Recommendations","Ratings","Reviews","Search books","Ratings","Reviews","Shelves","Search books","My Year in Books","Shelves","Ratings","Reviews","Progress tracking","Search books","Shelves","Ratings","Search books","Lists","Shelves","Ratings","Reviews","Progress tracking","Search books","Lists","Ratings","Reviews","Progress tracking","Recommendations","Ratings","Reviews","Progress tracking","Shelves","Progress tracking","Search books","Progress tracking","Shelves","Ratings","Reviews","Recommendations","Search books","My Year in Books","Ratings","Reviews","Recommendations","Search books","Explore","Ratings","Reviews","Progress tracking","Recommendations","Search books","Shelves","Ratings","Progress tracking","Reading challenge","Search books","Shelves","Ratings","Reviews","Progress tracking","Recommendations","Search books","Lists","Explore","My Year in Books","Shelves","Ratings","Reviews","Reading challenge","Search books","Lists","Shelves","Ratings","Reviews","Recommendations","Scan books at library/bookstores")
  ) %>% 
  mutate_if(is.character, factor)

palette <- c('#e9e5cd', '#ddd6bb', '#d1c7aa', '#c4b89a', '#b8a98a', '#ac9b7b', '#9f8d6d', '#937f60', '#8d6f52', '#885f3b', '#7f5025', '#75420e')

# factors NOT in correct order, palette in correct order
df %>% 
  ggplot(aes(y=Q4, fill=Q4))+
  geom_histogram(stat="count")+
  scale_fill_manual(values=palette)+
  theme(legend.position = "none")

# factors in correct order, palette NOT in correct order
df %>% 
  ggplot(aes(y=forcats::fct_infreq(Q4), fill=Q4))+
  geom_histogram(stat="count")+
  scale_fill_manual(values=palette)+
  theme(legend.position = "none")

I'd used dplyr::tally in place of forcats::fct_infreq.

df2 <- df %>% group_by(Q4) %>% tally() %>% arrange(desc(n))

df2
# A tibble: 12 × 2
   Q4                                   n
   <fct>                            <int>
 1 Ratings                             27
 2 Search books                        24
 3 Reviews                             23
 4 Progress tracking                   16
 5 Shelves                             15
 6 Reading challenge                   14
 7 Recommendations                     13
 8 My Year in Books                    12
 9 Lists                                9
10 Explore                              5
11 Groups                               3
12 Scan books at library/bookstores     1

#set levels based on the descending order
df2$Q4 <- factor(df2$Q4, levels=df2$Q4)  

#reverse order of palette
palette <- rev(palette)

df2 %>% 
  ggplot(aes(x=Q4, y=n, fill=Q4))+
  geom_col() + #plot with bar chart instead of relying on gghistogram
  scale_fill_manual(values=palette)+
  theme(legend.position = "none") +
  coord_flip()