I have the URL https://art42.tumblr.com/random
to a webpage that displays some images. I want to download the main image from that page.
If I right-click the image in Firefox (or any other browser) and I choose "Open image in new tab", I get the image (jpg). However, if I try to download the image using the code below, I get an HTML file.
I guess the problem has to do with the Referer. I tried to set the "Referer"
parameter to the page's URL, and also to the image's URL, but I still get HTML instead of JPG.
Why can I download the image in Firefox, but can't in my code?
function DownloadFile(CONST Url, Referer: String; OUT Data: TBytes; PostData: String= ''; SSL: Boolean = FALSE): Boolean; { TESTED OK }
VAR
Buffer : array[0..High(Word)*4] of Byte; { Buffer of 260KB }
TempBytes : TBytes;
sMethod : string;
BytesRead : Cardinal;
pSession : HINTERNET;
pConnection: HINTERNET;
pRequest : HINTERNET;
Resource : string;
Root : string;
port : Integer;
flags : DWord;
Header : string;
begin
Result := FALSE;
SetLength(Data, 0);
pSession := InternetOpen(nil {USER_AGENT}, INTERNET_OPEN_TYPE_PRECONFIG, nil, nil, 0);
if Assigned(pSession) then
TRY
{ Autodetect port }
port:= UrlExtractPort(URL);
if port = 0 then
if SSL
then Port := INTERNET_DEFAULT_HTTPS_PORT
else Port := INTERNET_DEFAULT_HTTP_PORT;
{ Root }
Root:= UrlExtractDomainRelaxed(Url);
pConnection := InternetConnect(pSession, PWideChar(Root), port, nil, nil, INTERNET_SERVICE_HTTP, 0, 0); { The second parameter of InternetConnect should contain only the name of the server, not the entire URL of the server-side script. }
if Assigned(pConnection) then
TRY
if (PostData = '')
then sMethod := 'GET'
else sMethod := 'POST';
if SSL
then flags := INTERNET_FLAG_SECURE OR INTERNET_FLAG_KEEP_CONNECTION
else flags := INTERNET_SERVICE_HTTP OR INTERNET_FLAG_RELOAD; // INTERNET_FLAG_RELOAD= Forces a download of the requested file, object, or directory listing from the origin server, not from the cache.;
Resource := UrlExtractResourceParams(Url);
pRequest := HTTPOpenRequest(pConnection, PWideChar(sMethod), PWideChar(Resource), nil, nil, nil, flags, 0); { The third parameter of HttpOpenRequest is the file name (URL) of the script }
if Assigned(pRequest) then
TRY
Header:= '';
if Referer > ''
then Header:= Header+ 'Referer: ' + Referer + sLineBreak;
Header:= Header+ 'User-Agent: Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:47.0) Gecko/20100101 Firefox/47.0'+SLineBreak;
//Header:= Header+ 'User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36 Edg/91.0.864.59'+SLineBreak; // Microsoft Edge UA string
Header:= Header+ 'Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'+SLineBreak;
Header:= Header+ 'Accept-Language: en-us,en;q=0.5' + SLineBreak;
Header:= Header+ 'Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7'+SLineBreak;
Header:= Header+ 'Keep-Alive: 70'+ SLineBreak; { In windows, default is 60 sec }
Header:= Header+ 'Connection: keep-alive'+ SlineBreak+SLineBreak;
HttpAddRequestHeaders(pRequest, PWideChar(Header), Length(Header), HTTP_ADDREQ_FLAG_ADD);
Result:= HTTPSendRequest(pRequest, NIL, 0, Pointer(PostData), Length(PostData)); { The actual POST data is the forth parameter }
if Result then
REPEAT
ZeroMemory(@Buffer, SizeOf(Buffer));
{ Download bytes }
InternetReadFile(pRequest, @Buffer, SizeOf(Buffer), BytesRead);
{ We stop? }
if BytesRead= 0 then break;
{ Convert static array to dynamic array }
SetLength(TempBytes, BytesRead);
Move(Buffer[0], TempBytes[0], BytesRead);
{ Merge arrays }
Data:= Data+ TempBytes;
UNTIL BytesRead= 0;
FINALLY
InternetCloseHandle(pRequest);
END
else
RaiseLastOSError;
finally
InternetCloseHandle(pConnection);
end;
finally
InternetCloseHandle(pSession);
end;
end;
In your "Accept" header you are not specifying any graphic formats.
A properly configured web server should send you a representation of the resource in one of the formats you accept.
Try looking at the request headers sent from your browser (use the Developer Tools - see your browser's help for how to access them). It will be accepting graphic formats as well as text.