[SOLVED] Content written to response writer shows erratic behaviour

File: main.go

import (
  "github.com/gorilla/mux"
  )
  
func setupRouting(pBus *bus.Bus) *mux.Router {

    log.Debug("Setting up routing")

    r := mux.NewRouter()

    pInvoiceHandler := handlers.NewInvoiceHandler(pBus)

    postRouter.HandleFunc("/invoice", pInvoiceHandler.HandleInvoiceGenerationRequest)


    return r
}

main() {
    pRouter := setupRouting(pBus)
    
    s := &http.Server{
        Addr:         address,
        Handler:      pRouter,
        IdleTimeout:  time.Duration(idlTimeout) * time.Second,
        ReadTimeout:  time.Duration(rdTimeout) * time.Second,
        WriteTimeout: time.Duration(wrtTimeout) * time.Second,
    }
}

File: InvoiceGenerationHandler.go

func (ih *InvoiceHandler) HandleInvoiceGenerationRequest(rw http.ResponseWriter, pReq *http.Request) {

    log.Info("Handling invoice generation request")

    query := pReq.URL.Query()
    resourceGrp := query.Get("rGrp")

    if resourceGrp == "" {
        http.Error(rw, "Invalid URL, rGrp parameter should be present in the URL ", http.StatusBadRequest)
        return
    }

    pSliceCostDetailsCtx, err := ih.pApiLoadTesting.CalculateCost(resourceGrp)

    if err != nil {
        log.Error("Could not calculate cost :", err.Error())
        http.Error(rw, err.Error(), http.StatusInternalServerError)
        return
    }

    pResponseJson, err := json.Marshal(pSliceCostDetailsCtx)

    if err != nil {
        log.Error("Could not convert price context slice to json: ", err.Error())
        http.Error(rw, "Could not generate cost response", http.StatusInternalServerError)
        log.Error(pResponseJson)
        return
    }

    rw.Write(pResponseJson)
}

File: apiloadtestpricing.go

type ApiLoadTestPricing struct {
    mpHttpClient *http.Client
}

func NewApiLoadTesting() *ApiLoadTestPricing {
    var pHttpClient = &http.Client{
        Timeout: time.Second * 120,
    }
    return &ApiLoadTestPricing{pHttpClient}
}

func getBearerToken(pHttpClient *http.Client, microsoftTokenEndpoint string) (string, error) {

    v := url.Values{}
    v.Set("grant_type", "client_credentials")
    v.Set("client_id", "******************************")
    v.Set("client_secret", "**************************")

    v.Set("resource", "https://management.core.windows.net")

    params := strings.NewReader(v.Encode())

    req, err := http.NewRequest(http.MethodPost, microsoftTokenEndpoint, params)

    if err != nil {
        log.Error("Could not create a new request")
        log.Error("Encountered error: ", err.Error())
        return "", err
    }

    req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
    req.Header.Set("Host", "login.microsoftonline.com")

    resp, err := pHttpClient.Do(req)

    if err != nil {
        log.Error("Could not get token from microsoft token endpoint ", microsoftTokenEndpoint)
        log.Error("Encountered error: ", err.Error())
        return "", err
    }
    
    /* More Code here not shown */
}

func getCostFromAzureCostManagement(pHttpClient *http.Client, bearerToken string, resourcegroup string) (*[]models.CostDetailsContext, error) {

    log.Info("Getting Cost from Azure")

    url := "https://management.azure.com/subscriptions/6a877c8f-4ca3-4135-9a0c-8e92db2384b4/resourceGroups/" + resourcegroup + "/providers/Microsoft.CostManagement/query?api-version=2021-10-01"
    postBody := `{
                "type": "Usage",
                "timeframe": "MonthToDate",
                "dataset": {
                  "granularity": "Daily",
                  "aggregation": {
                   "totalCost": {
                      "name": "PreTaxCost",
                      "function": "Sum"
                    }
                  },
                 "grouping": [
                    {
                     "type": "Dimension",
                      "name": "resourceGroup"
                    }
                  ]
                }
              }`

    req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(postBody))
    if err != nil {
        panic(err)
    }
    req.Header.Set("Content-Type", "application/json")
    req.Header.Set("Authorization", bearerToken)

    resp, err := pHttpClient.Do(req)
    
    /* More Code here not shown */
}

func (pPrice *ApiLoadTestPricing) CalculateCost(rGrp string) (*[]models.CostDetailsContext, error) {

    log.Info("Calculating cost from azure for resource group: ", rGrp)

    token, err := getBearerToken(pPrice.mpHttpClient, "https://login.microsoftonline.com/7a9376d4-7c43-480f-82ba-a090647f651d/oauth2/token")

    if err != nil {
        log.Error("Cost calculation has failed because proper bearer token was not present")
        return nil, err
    }

    pSliceCostDetailsContext, err := getCostFromAzureCostManagement(pPrice.mpHttpClient, "Bearer "+token, rGrp)

    return pSliceCostDetailsContext, err
}

Problem Statement:

I have written a small API in golang which does two things: 1. Get Bearer token from azure active directory 2. Call the Microsoft cost management API to get the resource group cost

Issue: Case 1: When getBearerToken and getCostFromAzureCostManagement get called. The response written to the response writer in InvoiceGenerationHandler.go sometimes reach and sometimes doesn't to the client.

Case 2: When only getBearerToken is called and the token string is returned it reaches the client on every write.

Note:

Both getBearerToken and getCostFromAzureCostManagement use the same http client for the request
gorilla mux is used for routing

What have I done ?

Tried commenting out the CalculateCost code - response writer works.
Tried commenting out getCostFromAzureCostManagement - responsewriter works
Tried with both of them uncommented - reponsewriter doesnt work
Read about the possible issue of shared httpclient and tried the follwing code https://github.com/hashicorp/go-cleanhttp - Responsewriter doesnt work.

I am stuck with this and have run out of ideas can someone please help me debug this ?