pythonweb-scrapingpython-requestshttp-headersstock

Data scraping fails: Seeking assistance


I attempted the following:

This is my source code:

import datetime
import hashlib
import json
import requests
from pprint import pprint

def generate_headers(url):
    salt = "w4ivc1ATTGta6njAZzMbkL3kJwxMfEAKDa3MNr"
    current_time = datetime.datetime.now(tz=datetime.timezone.utc)
    client_date = (current_time
        .isoformat(timespec="milliseconds")
        .replace("+00:00", "Z")
    )
    client_traceid = hashlib.md5(
        (client_date + url + salt).encode("utf-8")
    )
    security = hashlib.md5(
        current_time.strftime("%Y%m%d%H%M").encode("utf-8")
    )

    return {
        "Client-Date": client_date,
        "X-Client-TraceId": client_traceid.hexdigest(),
        "X-Security": security.hexdigest()
    }

payload = {"indices"  : ["DE0007203275"],
           "lang"     : "de",
           "offset"   : 0,
           "limit"    : 50,
           "sorting"  : "TURNOVER",
           "sortOrder": "DESC"}

urlString = "https://api.boerse-frankfurt.de/v1/search/equity_search?indices=['DE0007203275']&lang=de&offset=0&limit=50&sorting=TURNOVER&sortOrder=DESC"

result = generate_headers(urlString)
print(result)

headers = {"Host"             : "api.boerse-frankfurt.de",
           "Referer"          : "https://www.boerse-frankfurt.de/",
           "Content-Type"     : "application/json; charset=utf-8",
           "Client-Date"      : result["Client-Date"],     
           "X-Client-TraceId" : result["X-Client-TraceId"],  
           "X-Security"       : result["X-Security"]
           }

response = requests.post(url,
                         headers=headers,
                         json=payload)

pprint(json.loads(response.content))

Any suggestions are welcome. Thank you!!!


Solution

  • Try:

    import hashlib
    from datetime import datetime
    
    import requests
    
    
    def get_trace_id():
        timeutc = datetime.utcnow()
        timestr = timeutc.isoformat(timespec="milliseconds") + "Z"
    
        traceidbase = timestr + url + "w4ivc1ATTGta6njAZzMbkL3kJwxMfEAKDa3MNr"
        encoded = traceidbase.encode()
        traceid = hashlib.md5(encoded).hexdigest()
    
        return timestr, traceid
    
    
    url = "https://api.boerse-frankfurt.de/v1/search/equity_search"
    
    payload = {
        "indices": ["DE0008469008"],
        "lang": "de",
        "limit": 25,
        "offset": 0,
        "sorting": "TURNOVER",
        "sortOrder": "DESC",
    }
    
    client_date, trace_id = get_trace_id()
    
    headers = {
        "X-Client-TraceId": trace_id,
        "Client-Date": client_date,
    }
    
    data = requests.post(url, headers=headers, json=payload).json()
    
    print(json.dumps(data, indent=4))
    

    Prints:

    {
        "recordsTotal": 40,
        "data": [
            {
                "turnover": 353383054.2,
                "esgRiskScore": null,
                "industryAverage": null,
                "changeToPrevDay": 0.1915708812,
                "weeks52High": 571.8,
                "weeks52Low": 226.5,
                "performance3Month": 71.41920682,
                "performance1Year": 101.69687621,
                "performance3Year": 496.21523028,
                "tradingUnitsToday": null,
                "marketCapitalisation": 22781880000.0,
                "name": {
                    "originalValue": "RHEINMETALL AG",
                    "translations": {
                        "others": "Rheinmetall AG"
                    }
                },
                "wkn": "703000",
                "isin": "DE0007030009",
                "slug": "rheinmetall-ag",
                "overview": {
                    "exchange": "XETR",
                    "lastPrice": 523.0,
                    "changeToPrevDay": 0.1915708812,
                    "dateTimeLastPrice": "2024-04-10T17:37:03+02:00",
                    "turnover": 353383054.2,
                    "tradingUnitsToday": null
                },
                "performance": {
                    "weeks52High": 571.8,
                    "weeks52Low": 226.5,
                    "performance3Month": 71.41920682,
                    "performance6Month": 108.78243513,
                    "performance1Year": 101.69687621,
                    "performance3Year": 496.21523028
                },
                "keyData": {
                    "priceEarningsRatio": 23.2873,
                    "dividendPerShare": 5.7,
                    "dividendPerShareExtra": null,
                    "dividendRatio": 1.092,
                    "earningsPerShareGross": 17.76,
                    "earningsPerShareBasic": 12.32,
                    "priceBookRatio": 3.7567,
                    "marketCapitalisation": 22781880000.0
                },
                "sustainability": {
                    "esgRiskScore": null,
                    "industryAverage": null
                }
            },
    
    ...