[SOLVED] ASP.NET Core Web API on Azure App Service - POST and PUT requests failing 500 for a short period

The web application is a REST API and a SPA. It's maintained and currently on .NET 6.0 and have been working steadily for years.

The requests are CORS and the server is properly configured for this.

Suddenly we have several outbursts per day of POST and PUT requests consistently failing with 500 server error. And they are failing quickly, only 30 ms. This goes on for 5-15 minutes and then returns to normal again.

All GET requests still working perfectly fine in between, which is strange.

Even stranger these failing requests are not logged, like they never reach the web server. Checked both web server logs (IIS) and ASP.NET Core application exceptions and traces.

The response header X-Powered-By: ASP.NET is also missing from these failing requests. Which would be present for normal 500 server errors. Also suggesting the requests never reach the server.

The App Service Plan is only using 30% of it's resources currently.

Same behaviour is confirmed across Chrome, Edge and Safari. But you can have the issue in Chrome, while a session in Edge is working flawlessly on the same PC.

If we close the browser and re-open, the issues are gone.

It all started this month: May 2024.

Also worth mentioning that we have a DNS load balancer, Azure Traffic Manager.

This only operates on DNS queries and returns the closest of the two instances of the REST API services.

Traffic Manager does therefore not log any requests.

Update confirmed that error also occurs without Traffic Manager.

Update 2 We have tested deploying on a brand new App Service. And tried upgrading to .NET8.0. To no avail

That leaves us with browser's network log the only place to inspect. We have exported numbers of HAR files and looked for differences between failing requests and working requests, and found none.

Has anyone experienced similar behaviour of any kind?

Configuration of ASP.NET Core in Startup.cs:

public class Startup
{
    readonly string _corsPolicyName = "corsOrigins";
    public IConfiguration Configuration { get; }
    public IWebHostEnvironment Environment { get; }

    private static IConfiguration config;
    public static IConfiguration GetConfiguration()
    {
        return config;
    }

    public Startup(IConfiguration configuration, IWebHostEnvironment environment)
    {
        Configuration = configuration;
        config = configuration;
        Environment = environment;
    }

    public void ConfigureServices(IServiceCollection services)
    {
        if (Environment.IsDevelopment())
            IdentityModelEventSource.ShowPII = true;

        // logging
        services.AddApplicationInsightsTelemetry();
        services.AddLogging(logging => 
        {
            logging.AddSimpleConsole();
            logging.AddAzureWebAppDiagnostics();
        });
        services.Configure<AzureFileLoggerOptions>(options =>
        {
            options.FileName = "filelog-";
            options.FileSizeLimit = 50 * 1024;
            options.RetainedFileCountLimit = 5;
        });
        services.Configure<AzureBlobLoggerOptions>(options =>
        {
            options.BlobName = "Backend.txt";
        });

        // so our claims will not be translated
        JwtSecurityTokenHandler.DefaultInboundClaimTypeMap.Clear();

        // configure languages
        services.Configure<RequestLocalizationOptions>(options =>
        {
            var supportedCultures = new[]
            {
                new CultureInfo("en"),
                new CultureInfo("no")
            };
            options.DefaultRequestCulture = new RequestCulture("en");
            options.SupportedCultures = supportedCultures;
            options.SupportedUICultures = supportedCultures;
        });

        // add Identity
        services.AddIdentity<ApplicationUser, ApplicationRole>()
            .AddEntityFrameworkStores<AuthContext>()
            .AddRoleManager<RoleManager<ApplicationRole>>()
            .AddDefaultTokenProviders();
        services.AddUserAndPasswordPolicies();

        services.AddCors(options =>
        {
            {
                options.AddPolicy(_corsPolicyName, builder =>
                builder.SetIsOriginAllowedToAllowWildcardSubdomains()
                .WithOrigins("https://*.our.domain")
                .AllowAnyMethod()
                .AllowAnyHeader()
                .AllowCredentials());
            }
        });

        services.AddAuthentication(options =>
        {
            options.DefaultAuthenticateScheme = JwtBearerDefaults.AuthenticationScheme;
            options.DefaultChallengeScheme = JwtBearerDefaults.AuthenticationScheme;
        }).AddJwtBearer(o =>
        {
            o.MapInboundClaims = false;
            o.Authority = Configuration.GetValue<string>("authServerUrl");
            o.Audience = "backend";
            o.RequireHttpsMetadata = true;
            o.SaveToken = true;
            o.TokenValidationParameters = new TokenValidationParameters
            {
                NameClaimType = "name",
                RoleClaimType = "role",
                ValidateIssuer = true,
                ValidateAudience = true
            };
        });

        services.AddControllersWithViews(options =>
        {
            options.Filters.Add<WebCustomExceptionFilter>();
        })
            .AddNewtonsoftJson(options =>
            {
                options.SerializerSettings.ReferenceLoopHandling = ReferenceLoopHandling.Ignore;
                options.SerializerSettings.DateTimeZoneHandling = DateTimeZoneHandling.Utc;
            });

        services.AddSignalRManager();

        services.AddRazorPages()
            .AddRazorRuntimeCompilation();

        services.AddSwaggerGenNewtonsoftSupport();
        services.AddSwaggerGen(options =>
        {
            options.SwaggerDoc("v1", new OpenApiInfo { Title = "server debug api spec", Version = "v1" });
            options.SchemaFilter<EnumSchemaFilter>();
        });
    }

    public void Configure(IApplicationBuilder app, IWebHostEnvironment env)
    {
        StaticEnvironment.IsDebug = env.IsDevelopment();
        StaticEnvironment.ContentRootPath = env.ContentRootPath;

        app.UseCors(_corsPolicyName);

        if (env.IsDevelopment())
        {
            app.UseStaticFiles();
            app.UseDeveloperExceptionPage();
            app.UseSwagger();
            app.UseSwaggerUI(c => {
                c.DocumentTitle = "Backend Swagger docs";
                var sidebar = Path.Combine(env.ContentRootPath, "wwwroot/sidebar.html");
                c.HeadContent = File.ReadAllText(sidebar);
                c.InjectStylesheet("/colors.css");
                c.InjectStylesheet("/style.css");
                c.SwaggerEndpoint("/swagger/v1/swagger.json", "server");
            });
        }
        else
        {
            app.UseHttpsRedirection();
            app.UseHsts();
        }

        // Localization
        app.UseRequestLocalization();
        app.UseRouting();

        app.UseAuthentication();
        app.UseAuthorization();

        app.UseEndpoints(endpoints =>
        {
            endpoints.MapControllers();
        });
     

        // Enable static files for use in MVC views. 
        app.UseStaticFiles();
        app.UseStaticFiles(new StaticFileOptions()
        {
            FileProvider = new PhysicalFileProvider(
                        Path.Combine(Directory.GetCurrentDirectory(), @"Style")),
            RequestPath = new PathString("/Style")
        });
    }
}

Typical controller signature:

[Authorize]
[ApiController]
[Route("api/[controller]")]
public class ImprovementController : ControllerBase

The Web client fetch code

const fullUrl = `${baseUrl}/${url}`
const bearer = getBearer()
const connectionId = getConnectionId()
    
const req = {
    method: "POST",
    headers: {
        "Authorization": bearer,
        "Content-Type": "application/json"
    },
    body: JSON.stringify(data)
}
    
const res = await fetch(fullUrl, req)