I'm taking pictures with golang colly from Amazon and I want to throw these pictures in JSON into a single array (only 1 array for each product images). I scraped the pictures I need, I'm just having a problem with the JSON file. Thank you very much in advance.
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
AllInfos := make([]Info, 0)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
Counter := 0
var info Info
var theArray [10]string
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/I/" && imgclass == "" {
theArray[Counter] = imgsrc
fmt.Printf("The Array %d %v", Counter, theArray[Counter]+"\n")
Counter = Counter + 1
co.Visit(e.Request.AbsoluteURL(imgsrc))
info = Info{
Images: []string{
theArray[0],
theArray[1],
theArray[2],
theArray[3],
theArray[4],
theArray[5],
theArray[6],
},
}
AllInfos = append(AllInfos, info)
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(AllInfos)
writeJson(AllInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
My code's JSON result;
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"",
""
]
},
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
""
]
}
]
I need JSON result like this;
[
{
"id": 0,
"images": [
"https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
"https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
]
}
]
I had a hard time understanding what you wanted to do but here's what I came up with:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := new(Info)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v", len(allInfos.Images), imgsrc+"\n")
allInfos.Images = append(allInfos.Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data *Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}
As @Flimzy suggested, I appended only the new imagesrc
to a since instance of an info struct. Seeing as it seemed like your expected output was a slice of length 1 with all the images.
I also used a strings.HasPrefix
to clean up your slice check. This is equivalent but I think it's cleaner, so that's an optional change.
Let me know if you have any more questions.
EDIT, if you need to keep using a slice something like the following might work. From your question it's hard to tell when you need to clear a new Info:
package main
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"os"
"strings"
"time"
"github.com/gocolly/colly"
)
type Info struct {
ID int `json:"id"`
Images []string `json:"images"`
}
func main() {
allInfos := make([]Info, 1)
start := time.Now()
co := colly.NewCollector(
colly.AllowedDomains("www.amazon.com", "amazon.com"),
)
// GET Images
co.OnHTML("img[src]", func(e *colly.HTMLElement) {
imgsrc := e.Attr("src")
imgclass := e.Attr("class")
if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
allInfos[0].Images = append(allInfos[0].Images, imgsrc)
co.Visit(e.Request.AbsoluteURL(imgsrc))
}
})
co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")
co.OnRequest(func(r *colly.Request) {
fmt.Println("Visiting: ", r.URL.String())
})
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
enc.Encode(allInfos)
writeJson(allInfos)
duration := time.Since(start)
fmt.Println(duration.Seconds())
}
func writeJson(data []Info) {
dataFile, err := json.MarshalIndent(data, "", " ")
if err != nil {
log.Println("Could not create JSON", err)
}
ioutil.WriteFile("stocky.json", dataFile, 0666)
}