I am trying to create an app using the NodeWebkit. I am scraping content using node-phantom-simple module. Using the module, I was able to scrape content off the website. But how should i access it on the html side. I don't think i can create a rest service for this case. Here is the code sample:
var file = require('file.js');
var gui = require('nw.gui');
var menu = new gui.Menu({ type: 'menubar' });
var express = require('express');
var fs = require('fs');
var request = require('request');
var cheerio = require('cheerio');
var app = express();
var driver = require('node-phantom-simple');
url = 'http://www.espncricinfo.com/';
request(url, function(error, response, html){
if(!error){
var $ = cheerio.load(html);
var title, release, rating;
var json = { title : "", release : "", rating : ""};
$('.scoreline-list').first().filter(function(){
var data = $(this);
var numOfMatches = data.children().length;
console.log("Number of Matches: ",numOfMatches);
var matches=[];
//GET URL FOR EACH MATCH
for(x=0;x<numOfMatches;x++)
{
var lielem = data.children().eq(x);
matches[x] = "http://www.espncricinfo.com" + lielem.children().first().attr('href');
$('#editor').val(matches[x]);
console.log(matches[x]);
}
//FOR EACH MATCH URL
for(x=0;x<numOfMatches;x++)
{
var matchurl = matches[x];
//console.log(matchurl);
driver.create({ path: require('phantomjs').path }, function (err, browser) {
return browser.createPage(function (err, page) {
return page.open(matchurl, function (err,status) {
console.log("opened site? ", status);
page.includeJs('http://ajax.googleapis.com/ajax/libs/jquery/1.7.2/jquery.min.js', function (err) {
// jQuery Loaded.
// Wait for a bit for AJAX content to load on the page. Here, we are waiting 5 seconds.
setTimeout(function () {
return page.evaluate(function () {
//Get what you want from the page using jQuery. A good way is to populate an object with all the jQuery commands that you need and then return the object.
var h2Arr = [];
$('.innings-information').each(function () { h2Arr.push($(this).html()); });
return {
h2: h2Arr
};
}, function (err,result) {
console.log(result);
browser.exit();
});
}, 5000);
});
});
});
});
setTimeout(function(){
//waiting for the jquery to load
}, 5000);
} //END FOR LOOP EACH MATCH URL
})
}
})
Thanks for the help!
Why wouldn't you be able to make a rest endpoint for this? Just cache the result for each scraping operation, and the return the cache on the http-endpoint.
var cache={};
app.get('/myendpoint', function(req, res) {
res.json(cache);
})
request(url, function(error, response, html){
...
setTimeout(function () {
...
console.log(result);
//set cache here
cache=result;
browser.exit();
});
}, 5000);
});
app.listen(1338);
If you want to cache to update, wrap the scraping function in a setInterval.
var cache={};
app.get('/myendpoint', function(req, res) {
res.json(cache);
})
function updateCache() {
request(url, function(error, response, html){
...
setTimeout(function () {
...
console.log(result);
//set cache here
cache=result;
browser.exit();
});
}, 5000);
});
}
//Update cache every 60 secs.
setInterval(updateCache, 60000);
app.listen(1338);