Prometheus doesn't scrape data from the /metrics endpoint

I'm following Microsoft's documentation for adding observability to .NET with OpenTelemetry. The issue is that Prometheus isn't able to scrape data from my ASP.NET Core Web API, although Jaeger is collecting data correctly.

I’m using a Docker Compose setup from a Microsoft repository: TaskWeaver Docker Compose.

Here’s what I suspect might be the problem: In my prometheus-config.yml, I've set the target to optl-collector:9464. However, since my ASP.NET Core Web API exposes metrics at https://localhost:5001/metrics, I think the target should be host.docker.internal:5000. This is because host.docker.internal would point to my machine’s localhost where the API is running, rather than within Docker. But if I change the target to host.docker.internal, I’m unsure of the role of optl-collector in this setup.

public static class ServiceCollectionExtensions
{
    public static IServiceCollection AddObservabilityServices(this IServiceCollection services, IHostEnvironment environment, IConfiguration configuration)
    {
        ArgumentNullException.ThrowIfNull(services);
        ArgumentNullException.ThrowIfNull(environment);
        ArgumentNullException.ThrowIfNull(configuration);

        services.AddOpenTelemetry()
            .ConfigureResource(resource => resource
                .AddService(serviceName: environment.ApplicationName)
                .AddAttributes(new Dictionary<string, object>
                {
                    ["deployment.environment"] = environment.EnvironmentName
                }))
            .WithMetrics(metrics => metrics
                .AddAspNetCoreInstrumentation()
                .AddMeter(Metrics.SourceName)
                .AddMeter("Microsoft.AspNetCore.Hosting")
                .AddMeter("Microsoft.AspNetCore.Server.Kestrel")
                .AddPrometheusExporter())
            .WithTracing(tracing =>
            {
                tracing.AddAspNetCoreInstrumentation();
                tracing.AddHttpClientInstrumentation();
                tracing.AddSource(DistributedTracing.SourceName);
                
                // Export to Jaeger
                tracing.AddOtlpExporter(otlpOptions =>
                {
                    otlpOptions.Endpoint = new Uri("http://localhost:4317");
                });
            });

        return services;
    }

    public static IApplicationBuilder MapObservabilityEndpoints(this IApplicationBuilder app)
    {
        ArgumentNullException.ThrowIfNull(app);

        app.UseOpenTelemetryPrometheusScrapingEndpoint();

        return app;
    }
}

docker-compose.yaml

version: '3.9'

services:

  optl-collector:
    image: otel/opentelemetry-collector:0.96.0
    command: ["--config=/etc/collector-config.yaml"]
    volumes:
      - ./collector-config.yaml:/etc/collector-config.yaml
    ports:
      - "4317:4317" # Expose the gRPC receiver port for the first collector
    depends_on:
      - jaeger

  jaeger:
    image: jaegertracing/all-in-one:1.54
    ports:
      - "16686:16686" # Jaeger UI

  prometheus:
    image: prom/prometheus:latest
    ports:
      - "9090:9090" # Prometheus UI
    volumes:
      - ./prometheus-config.yml:/etc/prometheus/prometheus.yml
    command: ["--config.file=/etc/prometheus/prometheus.yml"]
    depends_on:
      - optl-collector

  grafana:
    image: grafana/grafana-enterprise:latest
    ports:
      - "3000:3000" # Grafana UI
    environment:
      - GF_SECURITY_ADMIN_PASSWORD=secret # You should change 'secret' to a password of your choosing
      - GF_USERS_ALLOW_SIGN_UP=false
    volumes:
      - grafana_data:/var/lib/grafana
    depends_on:
      - prometheus

volumes:
  grafana_data:

collector-config.yaml

receivers:
  otlp:
    protocols:
      grpc:
        endpoint: 0.0.0.0:4317
      http:
        endpoint: 0.0.0.0:4318

exporters:
  debug:
    verbosity: detailed
  otlp:
    endpoint: "jaeger:4317"
    tls:
      insecure: true
  prometheus:
    endpoint: "0.0.0.0:9464"

service:
  pipelines:
    traces:
      receivers: [otlp]
      exporters: [otlp]
    metrics:
      receivers: [otlp]
      exporters: [prometheus]
    logs:
      receivers: [otlp]
      exporters: [debug]

prometheus-config.yml

scrape_configs:
  - job_name: optl-collector
    scrape_interval: 5s
    static_configs:
      - targets: ["optl-collector:9464"]

I decided to keep the docker-compose as it is. Instead, I updated the code accordinly:

.WithMetrics(metrics =>
{
    ...
    if (otlpEndpoint != null)
    {
        metrics.AddOtlpExporter(otlpOptions =>
        {
            otlpOptions.Endpoint = new Uri(otlpEndpoint);
            otlpOptions.Protocol = OpenTelemetry.Exporter.OtlpExportProtocol.Grpc;
        });
     }
}