apigoweb-scrapinghtml-tablegoquery

Golang table webscraping


I have a code as below to scrape the specific cell value from html table. You can go to https://www.haremaltin.com/altin-fiyatlari website and search "satis__ATA_ESKI" on inspect mode to see that value. I am beginner on golang and did my best but unfortunately I couldn't get that value. Is there anybody to help me? Btw they don't have a community api. And one more thing, add time.sleep to wait for page to be loaded. If it returns "-" it is because page wasn't be loaded yet

package main

import (
"fmt"
"log"
"net/http"

"github.com/PuerkitoBio/goquery"
)

func main() {
   url := "https://www.haremaltin.com/altin-fiyatlari"

   resp, err := http.Get(url)
   if err != nil {
       log.Fatal(err)
   }
   defer resp.Body.Close()
   if resp.StatusCode != 200 {
       log.Fatalf("failed to fetch data: %d %s", resp.StatusCode, resp.Status)
   }

   doc, err := goquery.NewDocumentFromReader(resp.Body)
   if err != nil {
      log.Fatal(err)
   }

   doc.Find("tr__ATA_ESKI tr").Each(func(j int, tr *goquery.Selection) {
      data := []string{}
      tr.Find("td").Each(func(ix int, td *goquery.Selection) {
           e := td.Text()
           data = append(data, e)
           fmt.Println(data)
      })
   })
}

SOLUTION:

You can see the answer below and if you want you can check the source code to see why this kind of solution is used

Btw we can use iteration to fetch the specific value from map. I have a code for this too. But if you have any easier method just comment please

for _, v := range data { // we need value part of the map
    m, ok := v.(map[string]interface{}) // we need the convert the map 
                                    // into interface for iteration
    if !ok {
        fmt.Printf("Error %T", v)
    }
    for k, l := range m {
        if k == "ATA_ESKI"{ // the value we want is inside of this map
            a, ok := l.(map[string]interface{}) // interface convert again
            if !ok {
                fmt.Printf("Error %T", v)
            }
            for b,c := range a{
                if b == "satis"{ // the value we want
                    fmt.Println("Price is", c)
                }
            }
        }
    }
}

Full solution with iteration below:

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "net/http"
    "strings"
)

func main() {
    fecthData()
}

func fecthData() (map[string]interface{}, error) {
    body := strings.NewReader("dil_kodu=tr")
    req, err := http.NewRequest("POST", 
"https://www.haremaltin.com/dashboard/ajax/doviz", body)
    if err != nil {
        // handle err
        return nil, err
    }
    req.Header.Set("X-Requested-With", "XMLHttpRequest")

    resp, err := http.DefaultClient.Do(req)
    if err != nil {
        // handle err
        return nil, err
    }
    defer resp.Body.Close()
    jsonData, err := ioutil.ReadAll(resp.Body)
    if err != nil {
        panic(err)
        return nil, err
    }

    var data map[string]interface{}
    err = json.Unmarshal(jsonData, &data)
    if err != nil {
        return nil, err
    }

    for _, v := range data {
        m, ok := v.(map[string]interface{})
        if !ok {
            fmt.Printf("Error %T", v)
        }
        for k, l := range m {
            if k == "ATA_ESKI" {
                a, ok := l.(map[string]interface{})
                if !ok {
                    fmt.Printf("Error %T", v)
                }
                for b, c := range a {
                    if b == "satis" {
                        fmt.Println("Price", c)
                    }
                }
            }
        }
    }

    return data, nil
}

Solution

  • You can fetch via http Post request. Do not forget to add X-Requested-With header to request.

    func fecthData() (map[string]interface{}, error) {
        body := strings.NewReader("dil_kodu=tr")
        req, err := http.NewRequest("POST", "https://www.haremaltin.com/dashboard/ajax/doviz", body)
        if err != nil {
            // handle err
            return nil, err
        }
        req.Header.Set("X-Requested-With", "XMLHttpRequest")
    
        resp, err := http.DefaultClient.Do(req)
        if err != nil {
            // handle err
            return nil, err
        }
        defer resp.Body.Close()
        jsonData, err := ioutil.ReadAll(resp.Body)
        if err != nil {
            panic(err)
            return nil, err
        }
        var data map[string]interface{}
        err = json.Unmarshal(jsonData, &data)
        if err != nil {
            return nil, err
        }
        return data, nil
    }