Not Authorized to access this resource/api while trying to access directory api

I am using node with the googleapis and google-auth-library packages for accessing the users of G-Suite domain. For that, a service account was created with the domain-wide-delegation enabled:

The domain admin gave access to the service account to access the following scopes:

"https://www.googleapis.com/auth/admin.directory.group.readonly",
"https://www.googleapis.com/auth/admin.directory.group.member.readonly",
"https://www.googleapis.com/auth/admin.directory.user.readonly"

My code looks like this:

import { JWT } from "google-auth-library/build/src/auth/jwtclient";
import * as google from "googleapis";
const keys = require("../google-credentials.json");

async function main() {
  const client = new JWT(keys.client_email, undefined, keys.private_key, [
    "https://www.googleapis.com/auth/admin.directory.group.readonly",
    "https://www.googleapis.com/auth/admin.directory.group.member.readonly",
    "https://www.googleapis.com/auth/admin.directory.user.readonly"
  ]);
  await client.authorize();
  const service = google.admin("directory_v1");
  service.users.list(
    {
      auth: client,
      domain: "my_domain.com",
      maxResults: 10,
      orderBy: "email"
    },
    function(err, response) {
      if (err) {
        console.log("The API returned an error: " + err);
        return;
      }
      var users = response.users;
      if (users.length == 0) {
        console.log("No users in the domain.");
      } else {
        console.log("Users:");
        for (var i = 0; i < users.length; i++) {
          var user = users[i];
          console.log("%s (%s)", user.primaryEmail, user.name.fullName);
        }
      }
    }
  );
}

main().catch(console.error);

A JWT client gets initialised with the credentials received for the service account. Whatever, the client gives the following message back: Not Authorized to access this resource/api

You have to impersonate the service account with an email of a admin of your google domain.

const client = new JWT(
      keys.client_email,
      undefined,
      keys.private_key,
      [
        "https://www.googleapis.com/auth/admin.directory.group.readonly",
        "https://www.googleapis.com/auth/admin.directory.group.member.readonly",
        "https://www.googleapis.com/auth/admin.directory.user.readonly"
      ],
      "admin@yourdomain.com"
    );

This is mentioned somewhere in the docs in a box, however not really documented anywhere how to implement...