Traefik serversTransport Not Taking Effect: TLS Verification Issues

I'm encountering issues with Traefik's serversTransport configuration. I'm trying to set up a reverse proxy that provides a client certificate to upstream services. However, despite insecureSkipVerify being set to true, Traefik is still attempting to validate the certificate.

I'm able to curl to my back end from traefik's container: curl --cert /run/secrets/gateway_cert --key /run/secrets/gateway_cert_key --cacert /run/secrets/ca_pem --http2 https://core_api:5021/core-api/healthcheck it gives me a correct response.

The error that I'm getting is "tls: failed to verify certificate: x509: certificate is valid for ..." as if traefik is trying to validate the connection. It appears as " 500 Internal Server Error" in the front end.

here it is the compose file (to shorten this post I've left out the secrets and volume declarations):

services:
  traefik:
    image: traefik:v3.1
    command:
      - "--api.dashboard=true"
      - "--providers.docker=true"
      - "--entrypoints.web.address=:9091"
      - "--entrypoints.web.address=:9091/core-api"
      - "--entrypoints.web.address=:9091/auth-api"
    secrets:
      - ca_pem
      - gateway_cert
      - gateway_cert_key
    ports:
      - "9091:9091"  # Custom port for HTTP
      - "8080:8080"  # Traefik Dashboard
    volumes:
      - "/var/run/docker.sock:/var/run/docker.sock"
      - "./traefik.yml:/etc/traefik/traefik.yml"
    labels:
      - "traefik.http.services.gateway.loadbalancer.server.scheme=https"
      - "traefik.http.services.gateway.loadbalancer.server.certificateauthorityfile=/run/secrets/ca_pem"
  
  website:
    image: champ_web
    environment:
      - RELEASE_NAME=0
      - APP_VERSION=1
    networks:
      default:
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.website.entrypoints=website"
      - "traefik.http.routers.website.rule=Host(`localhost`)"

  auth_api:
    image: auth_api:latest
    environment: 
      ASPNETCORE_ENVIRONMENT: Release
    secrets:
      - gateway_pfx
      - service_pfx
      - service_pfx_password
    volumes:
      - auth_keys:/auth_keys
    labels:
      - "com.docker.compose.volume.access=rw"
      - "traefik.enable=true"
      - "traefik.http.routers.auth_api.rule=Host(`localhost`) && PathPrefix(`/auth-api`)"
      - "traefik.http.services.auth_api.loadbalancer.server.port=5011"
      - "traefik.http.services.auth_api.loadbalancer.server.scheme=https"
      
  core_api:
    image: core_api:latest
    environment: 
      ASPNETCORE_ENVIRONMENT: Release
      RELEASE_NAME: test
    secrets:
      - gateway_pfx
      - core_pfx
      - core_pfx_password
    volumes:
      - core_keys:/core_keys
    labels:
      - "com.docker.compose.volume.access=rw"
      - "traefik.enable=true"
      - "traefik.http.routers.core_api.rule=Host(`localhost`) && PathPrefix(`/core-api`)"
      - "traefik.http.services.core_api.loadbalancer.server.port=5021"
      - "traefik.http.services.core_api.loadbalancer.server.scheme=https"

and the traefik yml file:

api:
  dashboard: true
  insecure: true

log:
  level: DEBUG
  filePath: "/var/log/traefik.log"

accessLog:
  filePath: "/var/log/access.log"

entryPoints:
  website:
    address: ":9091"

services:
  core_api:
    loadBalancer:
      servers:
        - url: https://core_api:5021
          serversTransport: gatewayTransport
  auth_api:
    loadBalancer:
      servers:
        - url: https://auth_api:5011
          serversTransport: gatewayTransport

http:
  serversTransports:
    gatewayTransport:
      insecureSkipVerify: true
      rootCAs:
        - /run/secrets/ca_pem
      certificates:
        certFile: /run/secrets/gateway_cert     # Gateway's client certificate
        keyFile: /run/secrets/gateway_cert_key        # Gateway's client key

providers:
  docker:
    endpoint: "unix:///var/run/docker.sock"
    exposedByDefault: false

Any insights or suggestions would be greatly appreciated.

serversTransports can only be set through dynamic configuration from a file provider, it doesn't work with labels.

so your static config traefik.yml should include the dynamic config file:

api:
  insecure: true
  dashboard: true

providers:
  file:
    watch: true
    filename: /etc/traefik/dynamic_conf.yml

and your dynamic_conf.yml you will be able to set the serversTransports

http:
  routers:
    website:
      rule: Host(`localhost`)
      service: web_service
      entryPoints: web_entrypoint

    auth_api:
      rule: Host(`localhost`) && PathPrefix(`/auth-api`)
      service: auth_api
      entryPoints: web_entrypoint

  services:
    web_service:
      loadBalancer:
        servers: 
          - url: http://website:80

    auth_api:
      loadBalancer:
        serversTransport: gatewayTransport
        servers: 
          - url: https://auth_api:5011

  serversTransports:
    gatewayTransport:
      certificates:
        - certFile: /run/secrets/gateway_cert
          keyFile: /run/secrets/gateway_cert_key
      rootCAs:
        - /run/secrets/ca_pem
      insecureSkipVerify: true