containerd client + soci snapshotter: http: server gave HTTP response to HTTPS client

I'm trying to setup and insecure containerd client to pull images from an insecure private registry using the following code:

package main
 
import (
    "context"
    "crypto/tls"
    "fmt"
    "os"
 
    "github.com/awslabs/soci-snapshotter/fs/source"
    "github.com/containerd/containerd"
    "github.com/containerd/containerd/pkg/snapshotters"
    "github.com/containerd/containerd/remotes/docker"
    "github.com/containerd/containerd/remotes/docker/config"
)
 
func main() {
    var address = "/run/containerd/containerd.sock"
    var ref = "localhost:5000/python:3.9"
    var sociIndexDigest = "sha256:7b09431ef0749bee7491ba28d1adbe6e6e9e008e9be65fe35eed0aca31a01c91"
 
    client, err := containerd.New(address, containerd.WithDefaultNamespace("default"))
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(1)
    }
    defer client.Close()
 
    options := docker.ResolverOptions{
        Hosts: config.ConfigureHosts(context.TODO(), config.HostOptions{
            DefaultScheme: "http",
            DefaultTLS: &tls.Config{
                InsecureSkipVerify: true,
            },
        }),
    }
 
    _, err = client.Pull(context.TODO(), ref,
        containerd.WithResolver(docker.NewResolver(options)),
        containerd.WithPullSnapshotter("soci"),
        containerd.WithPullUnpack,
        containerd.WithImageHandlerWrapper(source.AppendDefaultLabelsHandlerWrapper(sociIndexDigest, snapshotters.AppendInfoHandlerWrapper(ref))))
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(1)
    }
    fmt.Println("Success")
}

Unfortunately, I keep getting few errors (from logs cropped):

{"error":"skipping mounting layer sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9 as FUSE mount: no ztoc for layer","key":"default/33/extract-128831236-kfhG sha256:781d5934416a582cf712c35212a8f92940c0223da02c1360d9ebb834d0f2c873","level":"warning","msg":"failed to prepare remote snapshot","parent":"sha256:9bb22d850b6e163c76b5cee00494067210e96c4cf585e2cd9d68898e31f43f69","remote-snapshot-prepared":"false","time":"2024-06-13T14:49:30.132049905Z"}
...
...
{"error":"cannot unpack the layer: cannot fetch layer: unable to fetch descriptor (sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9) from remote store: Get \"https://localhost:5000/v2/python/blobs/sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9\": context canceled","key":"default/33/extract-128831236-kfhG sha256:781d5934416a582cf712c35212a8f92940c0223da02c1360d9ebb834d0f2c873","level":"warning","msg":"failed to prepare snapshot; deferring to container runtime","parent":"sha256:9bb22d850b6e163c76b5cee00494067210e96c4cf585e2cd9d68898e31f43f69","time":"2024-06-13T14:49:30.363771421Z"}
...
...
{"error":"cannot unpack the layer: cannot fetch layer: unable to fetch descriptor (sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9) from remote store: Get \"https://localhost:5000/v2/python/blobs/sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9\": unknown \"unknown\": giving up request after 9 attempt(s): Get \"https://localhost:5000/v2/python/blobs/sha256:a99509a323905a80628005e4f3bc26ac15ebaf3ffdb08a9646a7f2d110ab38f9\": http: server gave HTTP response to HTTPS client","key":"default/34/extract-776892714-Ki3b sha256:781d5934416a582cf712c35212a8f92940c0223da02c1360d9ebb834d0f2c873","level":"warning","msg":"failed to prepare snapshot; deferring to container runtime","parent":"sha256:9bb22d850b6e163c76b5cee00494067210e96c4cf585e2cd9d68898e31f43f69","time":"2024-06-13T14:49:44.089475833Z"}

The /etc/containerd/config.toml file:

version = 2
[plugins."io.containerd.grpc.v1.cri".containerd]
    disable_snapshot_annotations = false
[proxy_plugins]
    [proxy_plugins.soci]
        type = "snapshot"
        address = "/run/soci-snapshotter-grpc/soci-snapshotter-grpc.sock"

I've tried removing the DefaultTLS option but the results are still the same. The push and pull process works fine when using the --insecure-registry option in nerdctl. Is anything I'm missing or doing incorrectly?

I don't know what “the necessary parameters” in your /etc/containerd/config.toml are, but you configured your ResolverOptions to use a TLS configuration, overwriting the default schema:

package main

import (
    "context"
    "crypto/tls"
    "fmt"

    "github.com/containerd/containerd/remotes/docker"
    "github.com/containerd/containerd/remotes/docker/config"
)

func main() {
    ctx := context.Background()

    hosts := []struct {
        name  string
        hosts docker.RegistryHosts
    }{
        {name: "ConfigureHosts with TLS", hosts: config.ConfigureHosts(ctx, config.HostOptions{
            DefaultScheme: "http",
            DefaultTLS: &tls.Config{
                InsecureSkipVerify: true,
            },
        })},
        {name: "ConfigureHosts without TLS", hosts: config.ConfigureHosts(ctx, config.HostOptions{
            DefaultScheme: "http",
        })},
        {name: "ConfigureDefaultRegistries", hosts: docker.ConfigureDefaultRegistries(
            docker.WithPlainHTTP(func(string) (bool, error) { return true, nil }),
        )},
    }

    host := "localhost:5000"

    for _, config := range hosts {
        fmt.Printf("%s:\n", config.name)
        if registryHosts, err := config.hosts(host); err == nil {
            for _, registryHost := range registryHosts {
                fmt.Printf("- %s//:%s\n", registryHost.Scheme, registryHost.Host)
            }
        }
    }
}

ConfigureHosts with TLS:
- https//:localhost:5000
ConfigureHosts without TLS:
- http//:localhost:5000
ConfigureDefaultRegistries:
- http//:localhost:5000

The default client from containerd uses fallback to HTTP when HTTP and TLS is configured, but you are using a custom snapshotter, which means it might only get the default schema without the fallback information. Does it work without the DefaultTLS configuration, which makes no sense in your use case anyway?