pythonqtwebenginepyqt6

Odd JS Errors when using QWebEnginePage


So I've been working on an authentication system using PyQt6-WebEngine to make web requests that will follow javascript redirects since Requests in Python doesn't support that. I've made a simple request script:

import sys
import json

# import PyQt6
import PyQt6
from PyQt6.QtCore import QUrl, QByteArray
from PyQt6.QtWidgets import QApplication
from PyQt6.QtWebEngineCore import QWebEnginePage, QWebEngineHttpRequest


class __Page__(QWebEnginePage):
    def __init__(self, app, url=None, method=None, headers=None, data=None) -> None:
        # Initiate the application
        self.app = app
        super().__init__()

        # Make sure the necessary variables aren't empty
        if url == None:
            raise ValueError("Passed url variable cannot be empty")
        if method == None:
            raise ValueError("Passed method variable cannot be empty")

        # Creates the main request variable
        self.req = QWebEngineHttpRequest()

        # Assign the variables used to setup the request later in self.setupRequest()
        self.method = method
        self.qurl = url
        self.headers = headers

        # Only assign this if the method is post
        if self.method == "post":
            self.postData = data
        else:
            self.postData = None

        # Setup the request
        self.setupRequest()

        # Create a new html variable and stuff, then load the request and launch the app
        self.html = ""
        self.loadFinished.connect(self.finish)
        self.load(self.req)
        self.app.exec()

    def setupRequest(self):

        # Assign the method variable
        if self.method == "get":
            method = QWebEngineHttpRequest.Method.Get
        elif self.method == "post":
            method = QWebEngineHttpRequest.Method.Post
        else:
            raise ValueError(
                "Expected 'get' or 'post' methods, but received '{}'".format(
                    self.method
                )
            )

        # Assign the method variable to the request method
        self.req.setMethod(method)
        self.req.setUrl(QUrl(self.qurl))

        # Assign the headers
        if self.headers != None:
            for i in self.headers:
                self.req.setHeader(
                    QByteArray(list(i.keys())[0].encode("ascii")),
                    QByteArray(list(i.values())[0].encode("ascii")),
                )

        # Assign the postData if the method is post
        if self.method == "post":
            self.req.setPostData(bytes(self.postData, "utf-8"))

    # This part is a bit over my head
    def finish(self):
        self.html = self.toHtml(self.callHtml)
        self.url = self.url()
        self.deleteLater()
        self.app.quit()

    def callHtml(self, html_str):
        self.html = html_str


class app:
    def __init__(self) -> None:
        self.app = QApplication(sys.argv)

    def requestPage(self, url=None, method=None, headers=None, data=None):
        page = __Page__(
            app=self.app, url=url, method=method, headers=headers, data=data
        )
        return page


def main():
    webapp = app()
    request = webapp.requestPage(url="https://google.com", method="get")


if __name__ == "__main__":
    main()

The issue with this is that it keeps giving me JS error in the output, which it started doing suddenly and without an apparent reason. With Google.com it gives 1-2 errors like: js: Error with Permissions-Policy header: Unrecognized feature: 'ch-ua-full-version-list'. With other websites, it has given me up to 30+.

This wouldn't be so annoying in and of itself except for the confusion, but I'm having another issue (which I am not quite sure is the fault of this script, but it came about the same time as the JS error issue) where the requested website just won't return any html whatsoever.

Any ideas?


Solution

  • I finally resolved this: the JS errors had nothing to do with the requests failing. I had just been making invalid requests (header issues and stuff). I'm still not sure why the errors seemed to pop up randomly, maybe I just made that conclusion mistakenly.