Puppeteer@24.1.0/chrome@132.0.6834.83 (headless: true) fail to GET pages with basic auth. net::ERR_BLOCKED_BY_CLIENT
I've encountered this problem while test crawling our customer service site. The problem was reproduced with mimimal test case in my dev env.
Am I missing any additional setting required for puppeteer@24.1.0/chrome@132, or is the software degraded, or anything else?
Test env
OS: Rocky Linux 9.5
Web server: Apache httpd 2.4.62
Node.js 20.18.2 (from nodesource)
Puppeteer/Chrome Versions
OK list:
all have no problem. (can GET pages with username/password)
NG list:
fails.
error trace:
net::ERR_BLOCKED_BY_CLIENT
at http://[myhost]/[mydir]/
at navigate (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/cdp/Frame.js:183:27)
at async Deferred.race (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/util/Deferred.js:36:20)
at async CdpFrame.goto (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/cdp/Frame.js:149:25)
at async CdpPage.goto (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/api/Page.js:574:20)
at async [my_js]
REPRODUCER
Basic auth sample settings for Apache httpd on Rocky Linux 9.5:
/etc/httpd/conf.d/basic_auth.conf
<Directory "/var/www/html/basic_auth">
AuthType Basic
AuthName "Restricted"
AuthUserFile "/mypwdir/htpasswd"
Require valid-user
</Directory>
script to generate /mypwdir/htpasswd:
# htpasswd -c /mypwdir/htpasswd testuser
(enter password when a prompt is displayed)
target page:
/var/www/html/basic_auth/test.html
<html>
<head><title>test page</title></head>
<body>test page</body>
</html>
Puppeteer script:
pt_test1.js
// reproducer: puppeteer >= 24.1.1 ERR_BLOCKED_BY_CLIENT w/Basic auth
'use strict';
const ptdelay = 0;
const pttmo = 10000; // msec.
//const TDIR = '/tmp/test1';
//const hfile = TDIR + '/' + 'http_header';
//const pfile = TDIR + '/' + 'http_body';
const puppeteer = require('puppeteer');
const fs = require('fs');
process.on('unhandledRejection', (reason, p) => {
console.log('Error:unhandled Rejection at: Promise '+p+' reason: '+reason);
process.exit();
});
// /usr/bin/node = process.argv[0];
const jsfile = process.argv[1];
var url = process.argv[2]; // url
if (url === undefined) { ERX('no url'); }
var uname = process.argv[3]; // -wwwauth username:password
var passwd = process.argv[4]; // -wwwauth username:password
// for basic auth
var wwwauth = {};
if (uname) { wwwauth.username = uname; }
if (passwd) { wwwauth.password = passwd; }
// chrome arguments
var ptargs = [];
ptargs.push('--no-sandbox');
ptargs.push('--disable-setuid-sandbox');
//ptargs.push('--ignore-certificate-errors'); //*** problem disappears with this option ***//
// puppeteer options
var ptopts = {
args: ptargs
,headless: true
,env: {}
//,ignoreHTTPSErrors: false
};
console.log('-');
console.log('url:['+url+']');
console.log('-');
// main
( async () => {
const browser = await puppeteer.launch(ptopts);
const page = await browser.newPage();
// basic auth
if (Object.keys(wwwauth).length > 0) {
await page.authenticate(wwwauth);
}
var tstart = Date.now();
var response;
try {
response = await page.goto(url, {timeout: pttmo, waituntil: 'load'});
} catch (e) {
console.log(e);
browser.close();
process.exit();
}
//fs.writeFileSync(hfile, 'HTTP/1.1 '+response.status()+' '+'FILLER'+"\n");
await console.log('HTTP/1.1 '+response.status()+' '+'FILLER');
await console.log('-');
var h = response.headers();
Object.keys(h).forEach(function(key){
//fs.appendFileSync(hfile, key+':'+this[key]+"\n");
console.log(key+':'+this[key]);
}, h);
await sleep(ptdelay);
//
const c = await page.content();
//await fs.writeFileSync(pfile, c);
await console.log('-');
await console.log(c);
await console.log("\n");
await console.log('-');
await console.log((Date.now() - tstart)/1000);
await page.close();
await browser.close();
})();
//
function ERX(emsg) {
console.log('ERROR: '+emsg);
console.log('Usage: node '+jsfile+' <url> [<username> <password>]');
process.exit();
}
function sleep(time) {
return new Promise((resolve, reject) => {
setTimeout(() => {
resolve();
}, time);
});
}
run sample1 (problem reproduced):
# /usr/bin/node /mydir/pt_test1.js http://mydomain/basic_auth/test.html testuser testpw
-
url:[http://mydomain/basic_auth/test.html]
-
Error: net::ERR_BLOCKED_BY_CLIENT at http://mydomain/basic_auth/test.html
at navigate (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/cdp/Frame.js:183:27)
at async Deferred.race (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/util/Deferred.js:36:20)
at async CdpFrame.goto (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/cdp/Frame.js:149:25)
at async CdpPage.goto (/mylib/node_modules/puppeteer-core/lib/cjs/puppeteer/api/Page.js:574:20)
at async /mydir/pt_test1.js:63:16
run sample2 (problem disappeared with '--ignore-certificate-errors', using empty test.html/size 0)
# /usr/bin/node /mydir/pt_test1.js http://mydomain/basic_auth/test.html testuser testpw
-
url:[http://mydomain/basic_auth/test.html]
-
HTTP/1.1 200 FILLER
-
accept-ranges:bytes
connection:Keep-Alive
content-length:0
content-type:text/html; charset=UTF-8
date:Thu, 06 Feb 2025 06:30:43 GMT
etag:"0-62d73543364b9"
keep-alive:timeout=5, max=99
last-modified:Thu, 06 Feb 2025 06:26:57 GMT
server:Apache/2.4.62 (Rocky Linux) OpenSSL/3.0.7 mod_fcgid/2.3.9 mod_perl/2.0.12 Perl/v5.32.1
strict-transport-security:max-age=31536000; includeSubDomains; preload
-
<html><head></head><body></body></html>
-
0.137
You probably encountering this issue: https://issues.chromium.org/issues/378022921
Note that you can still catch the error and click through the warning using Puppeteer. This is the behavior in latest Chrome that insecure HTTP requests could trigger this error for users. It is also possible to disable this feature as mentioned in the issue or make sure your host is exempt from this (e.g., by using localhost).