google-apps-scripthubspothubspot-crm

Error 429 when trying to loop through Hubspot Deals API JSON


I'm trying to get all the deals in my Hubspot API into a Google Sheet. The following Google App Script worked fine when the deal count was under 250. Now that I have more than 250 deals in my pipeline I'm getting 429 errors, 'Service invoked too many times for one day: urlfetch.', and other errors.

function getDeals() {
   // Prepare authentication to Hubspot
   
   var service = getService();
   var headers = {headers: {'Authorization': 'Bearer '+ service.getAccessToken()}};
   
   // Prepare pagination
   // Hubspot lets you take max 250 deals per request.
   // We need to make multiple request until we get all the deals.
   
   var keep_going = true;
   var offset = 0;
   var deals = Array();
   while(keep_going) {
      
      // We’ll take three properties from the deals: the source, the stage, the amount of the deal
      
      var url = API_URL + "/deals/v1/deal/paged?&includeAssociations=true&properties=dealstage&properties=source&properties=amount&properties=dealname&properties=num_associated_contacts&limit=250&offset&properties=hubspot_owner_id&limit=250&offset="+offset;
      var response = UrlFetchApp.fetch(url, headers);
      var result = JSON.parse(response.getContentText());
      Logger.log(result.deal)
      
      // Are there any more results, should we stop the pagination
      
      keep_going = result.hasMore;
      offset = result.offset;
      
      // For each deal, we take the stageId, source, amount, dealname, num_associated_contacts & hubspot_owner_id
      
      result.deals.forEach(function(deal) {
         var stageId = (deal.properties.hasOwnProperty("dealstage")) ? deal.properties.dealstage.value : "unknown";
         var source = (deal.properties.hasOwnProperty("source")) ? deal.properties.source.value : "unknown";
         var amount = (deal.properties.hasOwnProperty("amount")) ? deal.properties.amount.value : 0;
         var dealname = (deal.properties.hasOwnProperty("dealname")) ? deal.properties.dealname.value : "unknown";
         var hubspot_owner_id = (deal.properties.hasOwnProperty("hubspot_owner_id")) ? deal.properties.hubspot_owner_id.value : "unknown";
         var num_associated_contacts = (deal.properties.hasOwnProperty("num_associated_contacts")) ? deal.properties.num_associated_contacts.value : "unknown";
         deals.push([stageId,source,amount,dealname,num_associated_contacts,hubspot_owner_id]);
      });
   }
   return deals;
}


Solution

  • I think your issue is from the multiple specification of both the offset and limit URL parameters in your url:

    "...&limit=250&offset&...&limit=250&offset=" + offset;
    

    HubSpot's API may only expect a single value for certain keywords (such as limit and offset), which would mean you are always accessing only the first page of results - if more than one page exists, you would never stop calling this function until you exhaust your UrlFetchApp quota and the script exits via an unhandled exception, as result.hasMore would always be true.

    I would rewrite your script to use a do-while loop (and also to simplify your property extraction).

    function getDeals() {
      // Prepare authentication to Hubspot
      const service = getService();
      const fetchParams = {
        headers: {'Authorization': 'Bearer '+ service.getAccessToken()}
      };
    
      // Properties to collect from each deal:
      const desiredProps = [
        "dealstage",
        "source",
        "amount",
        "dealname",
        "num_associated_contacts",
        "hubspot_owner_id"
      ];
      const deals = [];
    
      // Hubspot lets you take max 250 deals per request.
      // Make multiple requests until we get all the deals.
      var offset = 0;
      var remainingPages = 100; // just in case.
      const url = API_URL + "/deals/v1/deal/paged?&includeAssociations=true&properties=dealstage&properties=source&properties=amount&properties=dealname&properties=num_associated_contacts&properties=hubspot_owner_id"
          + "&limit=250&offset=";
    
      do {
        var resp = UrlFetchApp.fetch(url + offset, fetchParams);
        var result = JSON.parse(response.getContentText());
        offset = result.offset;
    
        var pageDealInfo = result.deals.map(function (deal) {
          var dealInfo = desiredProperties.map(function (propName) {
            var val = deal.properties[propName];
            return (val === undefined ? "Unknown" : val;
          });
          /** add other things to dealInfo that aren't members of deal.properties
          dealInfo.push(deal.<something>);
           */
          return dealInfo;
        });
    
        // Add all the info for all the deals from this page of results.
        if (pageDealInfo.length)
          Array.prototype.push.apply(deals, pageDealInfo);
      } while (result.hasMore && --remainingPages);
    
      if (!remainingPages)
        console.warn({message: "Stopped deal queries due to own page limit - more deals exist!", currentOffset: offset, gatheredDealCount: deals.length});
      else
        console.log({message: "Finished deal queries", totalDeals: deals.length});
    
      return deals;
    }