javascriptweb-scrapingphantomjs

Scraping multiple pages with Phantomjs/Pjscrape


trying to scrape multiple pages but can't get the urlid array to work within the pjscrape .js file.

I'm pretty sure I might be making a newbie mistake but I would appreciate some help though. Thanks :)

pjs.config({

    timeoutInterval: 6000,
    timeoutLimit: 10000,

})

pjs.addSuite({
    // single URL or array
    url: abolaURLs,
    scraper: function(){
        var abolaURLs = [366762,366764,366763];
        for (var i = 0; i<abolaURLs.length; i++) {
            abolaURLs[i] = 'http://abola.pt/nnh/ver.aspx?id=' + abolaURLs[i];
        };
        var results[];
        var cenas1 = $('div#a5g2').text();
        var cenas2 = $('span#noticiatext').text();
        var cenas3 = $('div#a5x').text();
        results.push(cenas1, cenas2, cenas3);
        return results;
    }
});

Solution

  • That will work for you:

    var abolaURLs = [366762,366764,366763];
    
    for (var i = 0; i < abolaURLs.length; i++) {
        abolaURLs[i] = 'http://abola.pt/nnh/ver.aspx?id=' + abolaURLs[i];
    };
    
    pjs.addSuite({
        url: abolaURLs,
        scraper: function() {
                var results = []; // !! you have the wrong array declaration result[]
                var cenas1 = $('div#a5g2').text();
                var cenas2 = $('span#noticiatext').text();
                var cenas3 = $('div#a5x').text();
                results.push(cenas1, cenas2, cenas3);
                return results;
        }
    });
    
    pjs.config({
        timeoutInterval: 6000,
        timeoutLimit: 10000,
    });