I was making a shoe raffle bot with Node.js and was originally using headless Puppeteer to automate the process of filling out the raffle form and submitting it. I was told that Puppeteer is very CPU-intensive and was slower than the request modules within Node.js such as fetch, Axios etc.
I’ve been messing around with Axios for the past 2 days but I don’t actually know how to fill in and submit a form. How can I fill in and sumbit a form like I described above in Axios? Also, is Axios going to be the best choice (in terms of speed & CPU usage) or is there a better option?
Here is an example for a form I would like to fill in.
This is my puppeteer code which fills out the form:
const { sizeSelectorsTitolo } = require('./selectors/sizes');
const accounts = require('./profiles/savedaccounts');
const { proxyList1 } = require('./profiles/proxylists');
async function titoloMain(url, size, shippingprofile , ppaccountnumber, proxygroup, instaaccountnumber){
let splitProxy = proxygroup.split(':');
let proxyUserLocal = splitProxy[2];
let proxyPassLocal = splitProxy[3];
let proxyPortLocal = splitProxy[1];
let proxyMainLocal = splitProxy[0];
let countrySelector = '';
//Getting size selector ready
let OurSizeSelector = 'sizeSelectorsTitolo.'
OurSizeSelector = OurSizeSelector.concat(size);
delete OurSizeSelector.property;
OurSizeSelector = eval(OurSizeSelector);
//Getting country selector
switch (shippingprofile.country){
case shippingprofile.country = "UK":
countrySelector = 'United Kingdom'
break;
case shippingprofile.country = "USA":
countrySelector = 'United States of America'
break;
case shippingprofile.country = "France":
countrySelector = 'France'
break;
case shippingprofile.country = "Spain":
countrySelector = 'Spain'
break;
case shippingprofile.country = "Germany":
countrySelector = 'Germany'
break;
case shippingprofile.country = "Canada":
countrySelector = 'Canada'
break;
};
//getting gender selectors
let localGender = '';
if (shippingprofile.gender == 'Male'){
localGender = 'Male'
} else {
localGender = 'Female'
};
const browser = await puppetteer.launch( {
headless: true,
args: ['--disable-infobars',
`--window-size=${1000.},${1000.}`,
'--disable-features=IsolateOrigins,site-per-process',
//ip and port
`${proxyMainLocal}:${proxyPortLocal}`
],
ignoreDefaultArgs: ['--enable-automation']
});
const page = await browser.newPage();
//proxy settings
await page.authenticate({
username: proxyUserLocal,
password: proxyPassLocal
});
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/84.0.4147.125 Safari/537.36');
//now at the raffle page
console.log('on the raffle page')
await page.goto(url, {waitUntil:"networkidle2"});
console.log('filing out form...')
console.log('paypal')
await page.waitForSelector('#mce-EMAIL')
await page.type('#mce-EMAIL',accounts.paypalAccounts[ppaccountnumber].main,{delay:20})
console.log('name')
await page.type('#mce-FNAME',shippingprofile.firstname, {delay:20});
await page.type('#mce-LNAME',shippingprofile.surname, {delay:20});
console.log('shipping address')
await page.type('#mce-MMERGE5',shippingprofile.houseNum.concat(` ${shippingprofile.street}`), {delay:20});
await page.type('#mce-MMERGE8', shippingprofile.postcode , {delay:20});
await page.type('#mce-MMERGE12', shippingprofile.city , {delay:20});
await page.select('#mce-MMERGE3',countrySelector);
console.log('phone number')
await page.type('#mce-PHONE', shippingprofile.phoneNumber, {delay:20});
console.log('additional info')
await page.select('#mce-MMERGE9',localGender);
await page.select('#mce-MMERGE10','en');
console.log('instagram name')
await page.type('#mce-MMERGE7',accounts.instagram[instaaccountnumber].accountname,{delay:20});
console.log('selecting size')
await page.select('#mce-MMERGE6',OurSizeSelector)
console.log('clicking terms')
await page.waitForSelector('#mce-group\\[199\\]-199-0');
await page.waitFor(500);
await page.click('#mce-group\\[199\\]-199-0');
console.log('submitting')
await page.click('#mc-embedded-subscribe');
await page.waitFor(1000);
console.log(`done, check ${accounts.paypalAccounts[ppaccountnumber].main}`);
await browser.close()
};
This is the code for axios, i don't know how i would target the input fields to post the data so i am just logging the headers at the moment:
const axios = require('axios')
async function test() {
axios.get('https://en.titoloshop.com/titolo/air-jordan-1-retro-high-og-bio-hack/#raffle')
.then(res => {
console.log(res.headers)
})
.catch(err => {
if (err.response) {
console.log('there was an error');
console.log(err.response.data);
console.log(err.response.status);
console.log(err.response.headers);
}
});
}
TLDR: Make a POST request with the appropriate form data with the appropriate encoding to the form's action element. The keys to send can be found by looking at the name attributes of the input or select elements of the form. Scroll down for the code.
First of all, it's important to know the difference between Puppeteer and request modules such as axios and node-fetch. Puppeteer is used to control a browser and use websites as if a user was using it, whereas axios is used to simply make HTTP requests. Therefore, you can't do things like click buttons or fill in forms.
However, submitting a form usually simply makes an HTTP request with the form data to another URL. For example, here is the HTML of that raffle form (simplified):
<form
action="https://titolo.us6.list-manage.com/subscribe/post?u=652f80ec0adf2d7ac9588d0a1&id=8093f364b8"
method="post"
>
<div>
<label for="mce-EMAIL">Email Address* (PayPal)</label>
<input type="email" name="EMAIL" id="mce-EMAIL">
</div>
<!-- first name, last name, address, postcode, city -->
<div>
<label for="mce-MMERGE3">Country*</label>
<select name="MMERGE3" id="mce-MMERGE3">
<option value=""></option>
<option value="Switzerland">Switzerland</option>
<!-- more countries... -->
</select>
</div>
<!-- the rest of the inputs... -->
<div>
<strong>Terms & Conditions </strong>
<ul>
<li>
<input type="checkbox" name="group[243][1]" id="mce-group[243]-243-0">
<label for="mce-group[243]-243-0">I agree</label>
</li>
</ul>
</div>
<!-- more stuff... -->
<!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->
<div style="position: absolute; left: -5000px">
<input
type="text"
name="b_652f80ec0adf2d7ac9588d0a1_8093f364b8"
tabindex="-1"
value=""
>
</div>
<div>
<input type="submit" value="Subscribe" name="subscribe" id="mc-embedded-subscribe">
</div>
</form>
Let's break this down.
<form action="..." method="post'>: The form will POST its data in the post body to the URL specified in the action attribute. This will be encoded depending on the enctype attribute. In this case, it will be a application/x-www-form-urlencoded string (e.g. key1=value1&key2=value2), which is the default.<input type="email" name="EMAIL" id="mce-EMAIL">: The inputted email will be the value of the EMAIL key (indicated by the name attribute) in the post body.<select name="MMERGE3" id="mce-MMERGE3">: The selected option will be the value of the MMERGE3 key.
<option value="Switzerland"> Each of the options has a value attribute, which specifies the string to send in the post body. This does not need to be the same as the text content. If this is not present, the element's text content will be sent.<input type="checkbox" value="1" name="group[243][1]" id="mce-group[243]-243-0"> This is a checkbox input. If the checkbox is selected, group[243][1] (the name attribute) will be 1 (the value attribute, which defaults to on) in the form body. If it isn't selected,group[243][1] will be absent from the body.<input type="text" name="b_652f80ec0adf2d7ac9588d0a1_8093f364b8" tabindex="-1" value=""> According to the comment, this is an input that should never be filled in and is just to prevent bots signing up for the raffle (ironic isn't it?).<input type="submit" value="Subscribe" name="subscribe" id="mc-embedded-subscribe"> This is the submit button. The value of this is not submitted unless there is a name attribute (just like any other input), so in this case there will be a key called subscribe with the value Subscribe (from the value attribute). The default value attribute varies by browser:
<input type="submit">For more information on how submitting forms work, see Sending form data on MDN.
So, to submit this form without using Puppeteer, you need to make a POST request with the appropriate URL-encoded form values as the form data. You can do this in axios like this (documentation):
const url = 'https://titolo.us6.list-manage.com/subscribe/post?u=652f80ec0adf2d7ac9588d0a1&id=8093f364b8'
axios.post(url, new URLSearchParams({
EMAIL: accounts.paypalAccounts[ppaccountnumber].main,
FNAME: shippingprofile.firstname,
LNAME: shippingprofile.surname,
// Address
MMERGE5: shippingprofile.houseNum.concat(` ${shippingprofile.street}`),
// Postcode
MMERGE8: shippingprofile.postcode,
// City
MMERGE12: shippingprofile.city,
// Country
MMERGE3: countrySelector,
PHONE: shippingprofile.phoneNumber,
// Gender
MMERGE9: localGender,
// Language
MMERGE10: 'en',
// Size
MMERGE6: OurSizeSelector,
// Instagram account name
MMERGE7: accounts.instagram[instaaccountnumber].accountname,
// T&Cs
'group[243][1]': '1',
// That invisible input
b_652f80ec0adf2d7ac9588d0a1_8093f364b8: '',
// Subscribe button
subscribe: 'Subscribe'
}))
This code should submit the form just as if a user filled it in and submitted on their website. Most likely, not all of these are actually needed (like the subscribe button and the invisible input), but I'm including them here just in case.
Note: The website requires you to select a way you would like to hear from the company: the newsletter (name="gdpr[695]") and/or raffle information (name="gdpr[699]"). Both of these checkboxes have value="Y", so if you would like to submit these as part of the form just add 'gdpr[695]': 'Y' and/or 'gdpr[699]': 'Y' to the URLSearchParams constructor.