So I've been working on an authentication system using PyQt6-WebEngine to make web requests that will follow javascript redirects since Requests in Python doesn't support that. I've made a simple request script:
import sys
import json
# import PyQt6
import PyQt6
from PyQt6.QtCore import QUrl, QByteArray
from PyQt6.QtWidgets import QApplication
from PyQt6.QtWebEngineCore import QWebEnginePage, QWebEngineHttpRequest
class __Page__(QWebEnginePage):
def __init__(self, app, url=None, method=None, headers=None, data=None) -> None:
# Initiate the application
self.app = app
super().__init__()
# Make sure the necessary variables aren't empty
if url == None:
raise ValueError("Passed url variable cannot be empty")
if method == None:
raise ValueError("Passed method variable cannot be empty")
# Creates the main request variable
self.req = QWebEngineHttpRequest()
# Assign the variables used to setup the request later in self.setupRequest()
self.method = method
self.qurl = url
self.headers = headers
# Only assign this if the method is post
if self.method == "post":
self.postData = data
else:
self.postData = None
# Setup the request
self.setupRequest()
# Create a new html variable and stuff, then load the request and launch the app
self.html = ""
self.loadFinished.connect(self.finish)
self.load(self.req)
self.app.exec()
def setupRequest(self):
# Assign the method variable
if self.method == "get":
method = QWebEngineHttpRequest.Method.Get
elif self.method == "post":
method = QWebEngineHttpRequest.Method.Post
else:
raise ValueError(
"Expected 'get' or 'post' methods, but received '{}'".format(
self.method
)
)
# Assign the method variable to the request method
self.req.setMethod(method)
self.req.setUrl(QUrl(self.qurl))
# Assign the headers
if self.headers != None:
for i in self.headers:
self.req.setHeader(
QByteArray(list(i.keys())[0].encode("ascii")),
QByteArray(list(i.values())[0].encode("ascii")),
)
# Assign the postData if the method is post
if self.method == "post":
self.req.setPostData(bytes(self.postData, "utf-8"))
# This part is a bit over my head
def finish(self):
self.html = self.toHtml(self.callHtml)
self.url = self.url()
self.deleteLater()
self.app.quit()
def callHtml(self, html_str):
self.html = html_str
class app:
def __init__(self) -> None:
self.app = QApplication(sys.argv)
def requestPage(self, url=None, method=None, headers=None, data=None):
page = __Page__(
app=self.app, url=url, method=method, headers=headers, data=data
)
return page
def main():
webapp = app()
request = webapp.requestPage(url="https://google.com", method="get")
if __name__ == "__main__":
main()
The issue with this is that it keeps giving me JS error in the output, which it started doing suddenly and without an apparent reason. With Google.com it gives 1-2 errors like: js: Error with Permissions-Policy header: Unrecognized feature: 'ch-ua-full-version-list'.
With other websites, it has given me up to 30+.
This wouldn't be so annoying in and of itself except for the confusion, but I'm having another issue (which I am not quite sure is the fault of this script, but it came about the same time as the JS error issue) where the requested website just won't return any html whatsoever.
Any ideas?
I finally resolved this: the JS errors had nothing to do with the requests failing. I had just been making invalid requests (header issues and stuff). I'm still not sure why the errors seemed to pop up randomly, maybe I just made that conclusion mistakenly.