node.jsweb-scrapingpuppeteergoogle-chrome-headlessnode-cluster

Puppeteer - Protocol error (Page.navigate): Target closed

As you can see with the sample code below, I'm using Puppeteer with a cluster of workers in Node to run multiple requests of websites screenshots by a given URL:

const cluster = require('cluster');
const express = require('express');
const bodyParser = require('body-parser');
const puppeteer = require('puppeteer');

async function getScreenshot(domain) {
    let screenshot;
    const browser = await puppeteer.launch({ args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'] });
    const page = await browser.newPage();

    try {
        await page.goto('http://' + domain + '/', { timeout: 60000, waitUntil: 'networkidle2' });
    } catch (error) {
        try {
            await page.goto('http://' + domain + '/', { timeout: 120000, waitUntil: 'networkidle2' });
            screenshot = await page.screenshot({ type: 'png', encoding: 'base64' });
        } catch (error) {
            console.error('Connecting to: ' + domain + ' failed due to: ' + error);
        }

    await page.close();
    await browser.close();

    return screenshot;
}

if (cluster.isMaster) {
    const numOfWorkers = require('os').cpus().length;
    for (let worker = 0; worker < numOfWorkers; worker++) {
        cluster.fork();
    }

    cluster.on('exit', function (worker, code, signal) {
        console.debug('Worker ' + worker.process.pid + ' died with code: ' + code + ', and signal: ' + signal);
        Cluster.fork();
    });

    cluster.on('message', function (handler, msg) {
        console.debug('Worker: ' + handler.process.pid + ' has finished working on ' + msg.domain + '. Exiting...');
        if (Cluster.workers[handler.id]) {
            Cluster.workers[handler.id].kill('SIGTERM');
        }
    });
} else {
    const app = express();
    app.use(bodyParser.json());
    app.listen(80, function() {
        console.debug('Worker ' + process.pid + ' is listening to incoming messages');
    });

    app.post('/screenshot', (req, res) => {
        const domain = req.body.domain;

        getScreenshot(domain)
            .then((screenshot) =>
                try {
                    process.send({ domain: domain });
                } catch (error) {
                    console.error('Error while exiting worker ' + process.pid + ' due to: ' + error);
                }

                res.status(200).json({ screenshot: screenshot });
            })
            .catch((error) => {
                try {
                    process.send({ domain: domain });
                } catch (error) {
                    console.error('Error while exiting worker ' + process.pid + ' due to: ' + error);
                }

                res.status(500).json({ error: error });
            });
    });
}

Some explanation:

  1. Each time a request arrives a worker will process it and kill itself at the end
  2. Each worker creates a new browser instance with a single page, and if a page took more than 60sec to load, it will retry reloading it (in the same page because maybe some resources has already been loaded) with timeout of 120sec
  3. Once finished both the page and the browser will be closed

My problem is that some legitimate domains get errors that I can't explain:

Error: Protocol error (Page.navigate): Target closed.

Error: Protocol error (Runtime.callFunctionOn): Session closed. Most likely the page has been closed.

I read at some git issue (that I can't find now) that it can happen when the page redirects and adds 'www' at the start, but I'm hoping it's false... Is there something I'm missing?

Solution

  • What "Target closed" means

    When you launch a browser via puppeteer.launch it will start a browser and connect to it. From there on any function you execute on your opened browser (like page.goto) will be send via the Chrome DevTools Protocol to the browser. A target means a tab in this context.

    The Target closed exception is thrown when you are trying to run a function, but the target (tab) was already closed.

    Similar error messages

    The error message was recently changed to give more meaningful information. It now gives the following message:

    Error: Protocol error (Target.activateTarget): Session closed. Most likely the page has been closed.

    Why does it happen

    There are multiple reasons why this could happen.