Close socket after select()

I'm coding an IRC client and I would like implement a "/server" command to switch the connection of my client to an other server. Before initialize the new connection I want to close the sockect's fd but the close() call fail. Anybody could say me why ? Here is my code :

/* Main execution loop */
FD_ZERO(&irc->rdfs);
FD_SET(STDIN_FILENO, &irc->rdfs);
FD_SET(irc->socket_fd, &irc->rdfs);
if ((select(irc->socket_fd + 1, &irc->rdfs, NULL, NULL, NULL)) == -1)
{
  if ((close(irc->socket_fd)) == -1)
    exit(usage(CLOSE_ERROR));
  exit(usage(SELECT_ERROR));
}
if (FD_ISSET(STDIN_FILENO, &irc->rdfs))
{
  fgets(irc->buffer, SIZE - 1, stdin);
  {
    p = strstr(irc->buffer, RET);
    if (p != NULL)
      *p = 0;
    else
      irc->buffer[SIZE - 1] = 0;
  }
  write_on_server(irc, irc->buffer); /* The function where I call switch_server() in */
}
else if (FD_ISSET(irc->socket_fd, &irc->rdfs))
{
  if ((read_on_server(irc)) == 0)
    exit(usage(SERVER_DISCONNECT));
  puts(irc->buffer);
}

And here is where I'm trying to close my socket's fd :

void            switch_server(t_irc *irc)
{
if ((close(irc->socket_fd)) == -1) /* This is the close which fail */
    exit(EXIT_FAILURE);
}

void            write_on_server(t_irc *irc, const char * buffer)
{
if (!(strncmp("/server", buffer, strlen("/server"))))
  switch_server(irc);
else
  if ((send(irc->socket_fd, buffer, strlen(buffer), 0)) < 0)
   {
     if ((close(irc->socket_fd)) == -1)
       exit(usage(CLOSE_ERROR));
     exit(usage(CLIENT_SEND_ERROR));
   }
}

If you want to know why a syscall like close() failed, use perror() to print an error message to stderr, or strerror(errno) to convert the error code to a string and output it some other way.