How to use ActionFilter on Prometheus mapPath in standard .Net Web API?

I want to filter the range of client IPs who can route to Prometheus metrics. So in startup I have

    public void Configuration(IAppBuilder app)
    {
        ConfigureAuth(app);            
        app.UsePrometheusServer(q =>
        {
            q.MapPath = "/metrics";
        });
        app.UseWebApi(config);
    }

And this is my custom actionFilter class

public class IpFilter : ActionFilterAttribute
{
    public override void OnActionExecuting(ActionExecutingContext actionContext)
    {            
        string clinetIP = GetClientIpAddress(actionContext.HttpContext.Items["MS_HttpRequestMessage"] as HttpRequestMessage);
        if (IpAllowed(clinetIP))  
            base.OnActionExecuting(actionContext);
    }       

But I have no idea how to use IpFilter since it cannot be use as an attribute on a controller action. I tried to use it by adding a middleware using owin but the next.Invoke doesn't work properly

    public void Configuration(IAppBuilder app)
    {     
        app.Map("/metrics", metricsApp =>
        {             
           metricsApp.Use<TestIpMid>(deniedIps);
           metricsApp.UsePrometheusServer(q => q.MapPath = "/metrics");              
        });
        app.UsePrometheusServer(q =>
        {
            q.MapPath = "/metrics";
        });
        app.UseWebApi(config);
    }

and this is the middleware:

    public class TestIpMid : OwinMiddleware
{
    private readonly HashSet<string> _deniedIps;

    public TestIpMid(OwinMiddleware next, HashSet<string> deniedIps) : base(next)
    {
        _deniedIps = deniedIps;
    }

    public override async Task Invoke(IOwinContext context)
    {
        var ipAddress = context.Request.RemoteIpAddress;
        if (_deniedIps.Contains(ipAddress))
        {
            context.Response.StatusCode = 403;
            return;
        }           
        await Next.Invoke(context);            
    }
}

please help me :'(

this solution worked for me but other ways I was thinking of didn't work

    public void Configuration(IAppBuilder app)
    {
        ConfigureAuth(app);

        var config = new HttpConfiguration();
        var allowedIps = ProtectedSettings.Read(ProtectedSettings.protheusIpWhitelist).Split(new string[] { "," }, StringSplitOptions.RemoveEmptyEntries);             
        app.Use(async (Context, next) =>
        {
            var ipAddress = Context.Request.RemoteIpAddress;
            if ((!allowedIps.Contains(ipAddress)) && Context.Request.Path.Value == "/metrics")
            {
                Context.Response.StatusCode = 403;
                return;
            }
            await next.Invoke();
        });
        app.UsePrometheusServer(q =>
        {
            q.MapPath = "/metrics";

        });
        app.UseWebApi(config);
    }