[SOLVED] Cant get scrapy javascript full response

Cant get scrapy javascript full response

I cant seem to render a full html response from this link:

http://gabgoh.github.io/COVID

I'm using splash extension because what I have tried with common scrapy practice haven't work, yet this isn't working either.

Here is my python code, (I'm running splash docker with docker run -p 8050:8050 scrapinghub/splash)

import scrapy
from scrapy.utils.log import configure_logging
import scrapy_splash
from scrapy_splash import SplashRequest

class Covid_Spider(scrapy.Spider):
    name = "covid_spider"
    custom_settings = {
        'SPLASH_URL' : 'http://127.0.0.1:8050',
        'DOWNLOADER_MIDDLEWARES' : {
            'scrapy_splash.SplashCookiesMiddleware': 723,
            'scrapy_splash.SplashMiddleware': 725,
            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
            },
        'DUPEFILTER_CLASS' : 'scrapy_splash.SplashAwareDupeFilter',
        'SPIDER_MIDDLEWARES' : {
                'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,
                },
        'HTTPCACHE_STORAGE' : 'scrapy_splash.SplashAwareFSCacheStorage'
    }
    def start_requests(self):
        link = 'http://gabgoh.github.io/COVID/?CFR=0.02&D_hospital_lag=5&D_incbation=5.2&D_infectious=2.9&D_recovery_mild=11.1&D_recovery_severe=28.6&I0=10&InterventionAmt=0.09&InterventionTime=0&P_SEVERE=0.2&R0=2.2&Time_to_death=32&logN=14.1'
        print(link)
        splash_args = {
            'html': 1,
            'wait': 1,
            'render_all': 1
        }
        yield SplashRequest(url=link, callback=self.parse_covid,endpoint='render.html',args=splash_args)

    def parse_covid(self, response):
        print(response.css('body').getall()[0])

from scrapy.utils.project import get_project_settings
from scrapy.crawler import CrawlerProcess
if __name__ == "__main__":
    process = CrawlerProcess()
    process.crawl(Covid_Spider)
    process.start()

And this is my terminal output:

<body>
<!-- Global site tag (gtag.js) - Google Analytics -->
<script async="" src="https://www.googletagmanager.com/gtag/js?id=UA-65931696-1"></script>
<script>
  window.dataLayer = window.dataLayer || [];
  function gtag(){dataLayer.push(arguments);}
  gtag('js', new Date());

  gtag('config', 'UA-65931696-1');
</script>



</body>

I'm trying to get the full page. I'll appreciate any help. This is my first post btw.

Solution

Solved with Selenium

import os  
from selenium import webdriver  
from selenium.webdriver.chrome.options import Options  

# #short version
chrome_options = Options()  
chrome_options.add_argument("--headless")  
driver = webdriver.Chrome(chrome_options=chrome_options)  
driver.get('http://gabgoh.github.io/COVID')
body = driver.find_element_by_css_selector("body")
driver.close()