Customize colors in ggplot2's geom_polygon using brewer palette

I want to replicate a map built using the exact code from this link - just from "Step 1, Building the Initial Map":

https://www.r-bloggers.com/user-question-how-to-add-a-state-border-to-a-zip-code-map/

However, I want the colors to be not blue but something else.So, I've added just one line to the code - right after ggplot_polygon:

+scale_fill_brewer(palette = 'OrRd') 

But it's not working. Any idea why?

Below is my code:

library(choroplethr)
library(ggplot2)
library(devtools)
install_github('arilamstein/choroplethrZip@v1.3.0')
library(choroplethrZip)

# load the data 
data(df_zip_demographics)
str(df_zip_demographics)
df_zip_demographics$value = df_zip_demographics$percent_white

# create the map
zip_map = ZipChoropleth$new(df_zip_demographics)
zip_map$ggplot_polygon = geom_polygon(aes(fill = value), 
                                      color = NA) +
  scale_fill_brewer(palette = 'OrRd')
zip_map$set_zoom_zip(state_zoom  = c("new york", "new jersey"), 
                     county_zoom = NULL, 
                     msa_zoom    = NULL, 
                     zip_zoom    = NULL) 
zip_map$title = "New York and New Jersey ZCTAs"
zip_map$legend = "Percent White"
zip_map$set_num_colors(4)
choro = zip_map$render()
choro

From examining code, choro is the ggplot2 object that is rendered so this is where further ggplot2 elements are added, not to zip_map$ggplot_polygon which only takes the geom_polygon assignment.

# create the map
zip_map = ZipChoropleth$new(df_zip_demographics)
zip_map$ggplot_polygon = geom_polygon(aes(fill = value), 
                                  color = NA)
zip_map$set_zoom_zip(state_zoom  = c("new york", "new jersey"), 
                 county_zoom = NULL, 
                 msa_zoom    = NULL, 
                 zip_zoom    = NULL) 
zip_map$title = "New York and New Jersey ZCTAs"
zip_map$legend = "Percent White"
zip_map$set_num_colors(4)
choro = zip_map$render()
choro

# Change fill to brewer palette
choro + scale_fill_brewer(palette = 'OrRd')