Remove points if condition is met while keeping original color scale

I am trying to remove the left half of points (long < 0.5) from the plot below while retaining the original legend color scale.

However, the code I have below produces the following plot. The color scale changes from (-0.5, 0, 0.5, 1) to (0, 0.25, 0.5, 0.75, 1). How can I modify the code below to retain the original color scale? I would like to keep the x-axis limits from 0 to 1.

library(fields);library(ggplot2);library(colorRamps);library(dplyr)

loc<-as.matrix(expand.grid(seq(0,1,.03),seq(0,1,.03)))
n=nrow(loc)

matCov<-function(distMat,phi){
  (1+(sqrt(5)*(distMat/phi))+((5*distMat^2)/(3*(phi^2))))*exp(-(sqrt(5)*(distMat/phi)))
}

distMat<-as.matrix(rdist(loc))
CovMat <- matCov(distMat,phi=1)
set.seed(123)
Y <- as.numeric(t(chol(CovMat))%*%rnorm(n))
dat = as.data.frame(cbind(loc[,1:2],Y))
names(dat) = c("long","lat","Y")

ggplot(dat, aes(long, lat)) + 
  geom_raster(aes(fill = Y), interpolate = TRUE) + 
  scale_fill_gradientn(colours=matlab.like(10))+
  theme(legend.title=element_blank())

pick <- function(condition){
  function(d) d %>% filter_(condition)
}

ggplot(dat, aes(long, lat)) + 
  geom_raster(aes(fill = Y), interpolate = TRUE,data = pick(~long > 0.5)) + 
  scale_fill_gradientn(colours=matlab.like(10))+
  theme(legend.title=element_blank())+
  scale_x_continuous(limits=c(0,1))

Silly me. Using the suggestion from @zephryl, the code below does the trick.

ggplot(dat, aes(long, lat)) + 
  geom_raster(aes(fill = Y), interpolate = TRUE,data = pick(~long > 0.5)) + 
  scale_fill_gradientn(colours=matlab.like(10),limits=range(Y))+
  theme(legend.title=element_blank())+
  scale_x_continuous(limits=c(0,1))