where does the glibc's implementation of syscall open set ENAMETOOLONG errno when pathname is too long?

From the man open:

int open(const char *pathname, int flags);

...

       ENAMETOOLONG
              pathname was too long.

I couldn't find the assignment of errno to ENAMETOOLONG in the source code of glibc's open function. It is possible that this assignment is done by invoking kernel functions, but I couldn't find any evidence of that either.

Here are the functions I have already explored in glibc:

/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
   a third argument is the file protection.  */
int
__libc_open (const char *file, int oflag)
{
  int mode;

  if (file == NULL)
    {
      __set_errno (EINVAL);
      return -1;
    }

  if (__OPEN_NEEDS_MODE (oflag))
    {
      va_list arg;
      va_start(arg, oflag);
      mode = va_arg(arg, int);
      va_end(arg);
    }

  __set_errno (ENOSYS);
  return -1;
}

int
__open_nocancel (const char *file, int oflag, ...)
{
  int mode = 0;

  if (__OPEN_NEEDS_MODE (oflag))
    {
      va_list arg;
      va_start (arg, oflag);
      mode = va_arg (arg, int);
      va_end (arg);
    }

  return INLINE_SYSCALL_CALL (openat, AT_FDCWD, file, oflag, mode);
}

/* Open FILE with access OFLAG.  Interpret relative paths relative to
   the directory associated with FD.  If O_CREAT or O_TMPFILE is in OFLAG, a
   third argument is the file protection.  */
int
__openat (int fd, const char *file, int oflag, ...)
{
  int mode;

  if (file == NULL)
    {
      __set_errno (EINVAL);
      return -1;
    }

  if (fd != AT_FDCWD && file[0] != '/')
    {
      /* Check FD is associated with a directory.  */
      struct stat64 st;
      if (__fxstat64 (_STAT_VER, fd, &st) != 0)
    return -1;

      if (!S_ISDIR (st.st_mode))
    {
      __set_errno (ENOTDIR);
      return -1;
    }
    }

  if (__OPEN_NEEDS_MODE (oflag))
    {
      va_list arg;
      va_start (arg, oflag);
      mode = va_arg (arg, int);
      va_end (arg);

      ignore_value (mode);
    }

  __set_errno (ENOSYS);
  return -1;
}

/* Issue a syscall defined by syscall number plus any other argument
   required.  Any error will be handled using arch defined macros and errno
   will be set accordingly.
   It is similar to INLINE_SYSCALL macro, but without the need to pass the
   expected argument number as second parameter.  */
#define INLINE_SYSCALL_CALL(...) \
  __INLINE_SYSCALL_DISP (__INLINE_SYSCALL, __VA_ARGS__)

#define SYSCALL_CANCEL(...) \
  ({                                         \
    long int sc_ret;                                 \
    if (SINGLE_THREAD_P)                             \
      sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__);                \
    else                                     \
      {                                      \
    int sc_cancel_oldtype = LIBC_CANCEL_ASYNC ();                \
    sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__);              \
        LIBC_CANCEL_RESET (sc_cancel_oldtype);                   \
      }                                      \
    sc_ret;                                  \
  })

It's handled by the filesystem because the rules for what names are too long could be very complex and file-system specific. See, for example, this code from EXT4.