phpsimple-html-domgoogle-scholar

Web scraping publication from a google scholar profile via simplehtmldom PHP


I'm trying to scrape publication from a google scholar profile,but i have no idea how to scrape every publication from a profile, i know the maximum publication the profile page can show is 100 per page from this question :

Google Scholar profile scrape PHP

I just want to know how to apply the url to my php code so that i can get every publication from a profile and insert them to a array

I am able to put every publication in a single page to a array with this code:

<?php 
set_time_limit(0);  
include 'simple_html_dom.php';

$data = json_decode(file_get_contents('php://input'),true);
$scholarID =  $data["gScholarID"];
$kodeDosen = $data["kodeDosen"];
$page = 1;
$offset = ($page - 1)* 100;
$cStart = 0+$offset;
$profile = 'https://scholar.google.com/citations?user='.$scholarID.'&hl=en&cstart='.$cStart.'&view_op=list_works&pagesize=100';
$html = file_get_html($profile);
$table = $html->find('#gsc_a_t',0);
$rowData = array();

foreach($table->find('tr.gsc_a_tr') as $row){
    $paperjudul  = $row->find('td.gsc_a_t a', 0)->plaintext;
    $paper['kodeDosen'] = $kodeDosen;
    $paper['judul'] = $paperjudul;
    $cited   = $row->find('td.gsc_a_c', 0)->plaintext;
    if($cited === ''){
        $cited = 0;
    }
    $cited = preg_replace('/[\*]+/', '', $cited);
    $paper['citedBy'] = $cited;
    $paper['namaJurnal']    = $row->find('td.gsc_a_t .gs_gray', 1)->plaintext;
    if($paper['namaJurnal'] === ''){
        $paper['namaJurnal'] = 'n/a';
    }
    $paper['periode']   = $row->find('td.gsc_a_y', 0)->plaintext;
    if($paper['periode'] === ' '){
        $paper['periode'] = 'n/a';
    }
    $paper['status'] = 'Published';
    $rowData[] = $paper;
}

print_r($rowData);


?>

I just want to know how to apply this code to multiple pages to get all publication from a google scholar profile


Solution

  • I found a method that works, first, i create a loop that searches the web page for an indication that the page has no publication to show and inserts the url that contains the publication which i then loop to scrape the publication within the url. This is the code that i used:

    <?php 
    set_time_limit(0);  
    include 'simple_html_dom.php';
    include 'connectdb.php';
    
    $scholarID =  $_GET["gScholarID"];
    $kodeDosen = $_GET["kodeDosen"];
    
    $page = 1;
    $finalPage = false;
    
    $sqlTest = 'INSERT INTO tbl_publikasi(kodeDosen,jenis,namaJurnal,judul,status,tipePublikasi,periode,tahun,citedCount) VALUES ';
    $response = array();
    
    
    while (!$finalPage) {
        $offset = ($page - 1)* 100;
        $cStart = 0+$offset;
        $profile = 'https://scholar.google.com/citations?user='.$scholarID.'&hl=en&cstart='.$cStart.'&view_op=list_works&pagesize=100';
        $html = file_get_html($profile);
        if(is_object($html)){
            $empty = $html->find('td.gsc_a_e',0);
            if($empty){
                $finalPage = true;
                unset($html);
            }
            else{
                $urlArray[] = $profile;
                $page++;
            }
        }
        else{
            $response['success'] = 0;
            $response['message'] = "URL tidak valid ";
        }
    
    }
    
    if($finalPage){
        foreach ($urlArray as $urlPublikasi) {
            $html = file_get_html($urlPublikasi);
            $table = $html->find('#gsc_a_t',0);
            $rowData = array();
            if($table){
                foreach($table->find('tr.gsc_a_tr') as $row){
                    $paper['kodeDosen'] = $kodeDosen;
                    $paperjudul  = $row->find('td.gsc_a_t a', 0)->plaintext;
                    $paper['judul'] = $paperjudul;
                    $cited   = $row->find('td.gsc_a_c', 0)->plaintext;
                    if($cited === ''){
                        $cited = 0;
                    }
                    $cited = preg_replace('/[\*]+/', '', $cited);
                    $paper['citedBy'] = trim($cited);
                    $paper['jenis'] = 'Scholar';
                    $paper['namaJurnal']    = $row->find('td.gsc_a_t .gs_gray', 1)->plaintext;
                    if($paper['namaJurnal'] === ''){
                        $paper['namaJurnal'] = 'n/a';
                    }
                    $paper['periode'] = 'n/a';
                    $paper['tahun']   = $row->find('td.gsc_a_y', 0)->plaintext;
                    if($paper['tahun'] === ' '){
                        $paper['tahun'] = '0000';
                    }
                    $paper['tipePublikasi'] = 'Scholar'; 
                    $paper['status'] = 'Published';
                    $rowData[] = $paper;
                }
    
                foreach ($rowData as $paperValue) {
                    $judul = $paperValue['judul'];
                    $jenis = $paperValue['jenis'];
                    $citedCount = $paperValue['citedBy'];
                    $namaJurnal = $paperValue['namaJurnal'];
                    $periode = $paperValue['periode'];
                    $tahun = $paperValue['tahun'];
                    $status = $paperValue['status'];
                    $tipePublikasi = $paperValue['tipePublikasi'];
                    $sqlTest .= "('".$kodeDosen."','".$jenis."','".$namaJurnal."','".$judul."','".$status."','".$tipePublikasi."','".$periode."','".trim($tahun)."','".$citedCount."'),";
    
                }
                $query = rtrim($sqlTest, ',');
                $query .= "ON DUPLICATE KEY UPDATE idPublikasi=LAST_INSERT_ID(idPublikasi), kodeDosen = VALUES(kodeDosen), jenis = VALUES(jenis), 
                namaJurnal=VALUES(namaJurnal),status=VALUES(status),
                tipePublikasi = VALUES(tipePublikasi),periode=VALUES(periode),tahun = VALUES(tahun),citedCount = VALUES(citedCount)";
            }
            else{
                $response['success'] = 0;
                $response['message'] = "Tabel Publikasi tidak ditemukan ";
            }
    
        }
    
    
    
         if (mysqli_query($conn, $query)) {
            $response['success'] = 1;
            $response['message'] = "Array Uploaded Successfully";
         } 
         else {
            $response['success'] = 0;
            $response['message'] = "Array Upload Failed, Alasan : ".mysqli_error($conn);
         }
    
    
    }
    else{
        $response['success'] = 0;
        $response['message'] = "Gagal ditemukan ";
    }
    
    echo json_encode($response);
    
    
    
    
    ?>