google-apps-scriptgoogle-sheetsweb-scrapingwebformsurlfetch

How to refresh the token and cookies using Google Apps Script?


I want to auto-refresh the verification token and Cookie each time I make a request using Google Apps Script? I post Pin on this web form and extract the results back to Google sheet, I used the following snippet:

 function submitForm() {
  var url = "https://taxdelinquent.cookcountyclerkil.gov/";
  var formData = {
     '__RequestVerificationToken': 'q5mCwHyTYkN3HgYB3NZXS-JvsGLfvH5bmRtpJKPWTmml_Oskif_gIW3vnU8jNbhZC_XW2xlmCyGqHAhISUjtX-zbZdoIQBiOulF44suHU1c1',
    'Pin': '13-30-300-016-0000'
  };

  var encodedData = Object.keys(formData).map(function(key) {
    return encodeURIComponent(key) + '=' + encodeURIComponent(formData[key]);
  }).join('&');

  var headers = {
    "Accept": "text/html,application/xhtml",
    "Cookie": "__RequestVerificationToken=wOpZrdbvOL7wiQJcqlWYSAlsLIBc5yImjoHSGqSixBZewlt2FZgNLQXh9xH5APz2_bCkcgxPmckSipc1sRExCbLEOYcmCRrWTqNhX8mVtqk1; AWSALB=KsF6dcBJ9rQjYyLrj1veloVO595F9XgoQyhW2wcXz8AC6XuxVBcklT3NdYfQ8p32s2w/pzyVw5w7QcJ7NBnAhb3LcpBoWAyN6hwsXjzXMvr4fZhbEZ4KAMXHjbU8; AWSALBCORS=KsF6dcBJ9rQjYyLrj1veloVO595F9XgoQyhW2wcXz8AC6XuxVBcklT3NdYfQ8p32s2w/pzyVw5w7QcJ7NBnAhb3LcpBoWAyN6hwsXjzXMvr4fZhbEZ4KAMXHjbU8"
  };

  var options = {
    "method": "post",  // or "get" depending on your needs
    "headers": headers,
    "payload": encodedData
  };

  var response = UrlFetchApp.fetch(url, options).getContentText();
}

It produces the required data. If you go to inspect element of this website, you will see the payload like this:

Form Elements.

The data to be extracted looks like this:

Data to be parsed

However I believe that when this verification token and Cookie expire, script would not be able to get the data, I need your guidance to resolve this. Thank you for your assistance.


Solution

  • ok, using the following script I am able to get verification token and cookies and use it in above script submitForm():

    function getTokenCookie(url) {
    
      var options = {
        "method": "get", 
      };
    
      // Make the HTTP request
      var response = UrlFetchApp.fetch(url, options);
      var cookies = response.getAllHeaders()["Set-Cookie"];
    
      var htmlContent = response.getContentText();
      var match = htmlContent.match(/<input name="__RequestVerificationToken" type="hidden" value="([^"]+)" \/>/);
    
      if (match && match.length > 1) {
        var verificationToken = match[1];
        return [verificationToken,cookies];
      } else {
        Logger.log("Unable to find __RequestVerificationToken in the HTML content.");
      }
    }
    

    This script gets both the values (token,cookies) from the webpage to be used for data scraping.