javascriptnode.jsweb-crawlerphantomjs

How can I scrape pages with dynamic content using node.js?


I am trying to scrape a website but I don't get some of the elements, because these elements are dynamically created.

I use the cheerio in node.js and My code is below.

var request = require('request');
var cheerio = require('cheerio');
var url = "http://www.bdtong.co.kr/index.php?c_category=C02";

request(url, function (err, res, html) {
    var $ = cheerio.load(html);
    $('.listMain > li').each(function () {
        console.log($(this).find('a').attr('href'));
    });
});

This code returns empty response, because when the page is loaded, the <ul id="store_list" class="listMain"> is empty.

The content has not been appended yet.

How can I get these elements using node.js? How can I scrape pages with dynamic content?


Solution

  • Here you go;

    var phantom = require('phantom');
    
    phantom.create(function (ph) {
      ph.createPage(function (page) {
        var url = "http://www.bdtong.co.kr/index.php?c_category=C02";
        page.open(url, function() {
          page.includeJs("http://ajax.googleapis.com/ajax/libs/jquery/1.6.1/jquery.min.js", function() {
            page.evaluate(function() {
              $('.listMain > li').each(function () {
                console.log($(this).find('a').attr('href'));
              });
            }, function(){
              ph.exit()
            });
          });
        });
      });
    });