color only mean group of PCA

Let's say I have a PCA (either using prcomp() or PCA() from FactoMineR. I'd like to plot all points per each group in black, but label and color only the mean point per group. For example,

library(FactoMineR)
library(factoextra)
iris
res.ACP<-PCA(iris[,c(1:3)],scale.unit = TRUE,ncp= 5)
fviz_pca_ind(res.ACP,geom.ind=c("point"),point.size=3,pointshape=16,
             col.ind=iris$Species,
             col="Set1",legend.title="Species",addlabel=TRUE, mean.point.size=5)

How do I keep the mean points colored as above but plot all the data points in black? Ideally, I'd also like to label the means and then I'll drop the legend.

Thanks for the help.

Here is a possibile solution.

library(FactoMineR)
library(factoextra)
library(dplyr)

res.ACP <- PCA(iris[,c(1:3)], scale.unit = TRUE, ncp= 5)

df <- data.frame(res.ACP$ind$coord[,1:2], Species=iris$Species)
df <- df %>%
      group_by(Species) %>%
      summarize(Dim1=mean(Dim.1), Dim2=mean(Dim.2))

fviz_pca_ind(res.ACP, geom.ind=c("point"), point.size=3, pointshape=16,
             legend.title="Species", addlabel=TRUE) +
  geom_text(data=df, aes(x=Dim1, y=Dim2, label=Species, color=Species), 
            size=6, nudge_y=0.25) +
  geom_point(data=df, aes(x=Dim1, y=Dim2, color=Species), size=5)