Changing destination port using gopacket

I'm currently playing around with reading packages from nfqueue and modifying them.Unfortunately, I'm a bit stuck at changing the destination port of a package. See the code snippet below. Idea is to rewrite dest port 8888 to 8000. I do see the modified package going out from the queue, but if I want to connect to an HTTP server listening on port 8000 by connecting to port 8888, the connection times out. I assume something mal-formed in the package.

package main

import (
    "os"
    "os/signal"
    "syscall"

    "github.com/chifflier/nfqueue-go/nfqueue"
    "github.com/coreos/go-iptables/iptables"

    "github.com/google/gopacket"
    "github.com/google/gopacket/layers"
)

func sendNewPacket(payload *nfqueue.Payload, layers ...gopacket.SerializableLayer) {
    buffer := gopacket.NewSerializeBuffer()
    gopacket.SerializeLayers(buffer, gopacket.SerializeOptions{FixLengths: true, ComputeChecksums: true}, layers...)
    outgoingPacket := buffer.Bytes()
    payload.SetVerdictModified(nfqueue.NF_ACCEPT, outgoingPacket)
}

func realCallback(payload *nfqueue.Payload) int {
    packet := gopacket.NewPacket(payload.Data, layers.LayerTypeIPv4, gopacket.Default)
    ethLayer := packet.Layer(layers.LayerTypeEthernet)
    eth, _ := ethLayer.(*layers.Ethernet)
    if ipLayer := packet.Layer(layers.LayerTypeIPv4); ipLayer != nil {
        ip, _ := ipLayer.(*layers.IPv4)
        if tcpLayer := packet.Layer(layers.LayerTypeTCP); tcpLayer != nil {
            tcp, _ := tcpLayer.(*layers.TCP)
            if tcp.DstPort == 8888 {
                tcp.DstPort = 8000
                sendNewPacket(payload, eth, ip, tcp)
                return 0
            }
            if tcp.SrcPort == 8000 {
                tcp.SrcPort = 8888
                sendNewPacket(payload, eth, ip, tcp)
                return 0
            }
        }
    }
    payload.SetVerdict(nfqueue.NF_ACCEPT)
    return 0
}

func main() {
    ipt, err := iptables.New()
    if err != nil {
        panic(err)
    }
    ipt.Append("filter", "INPUT", "-p", "tcp", "-j", "NFQUEUE", "--queue-num", "0")
    ipt.Append("filter", "OUTPUT", "-p", "tcp", "-j", "NFQUEUE", "--queue-num", "0")
    q := new(nfqueue.Queue)
    q.SetCallback(realCallback)

    q.Init()
    defer q.Close()

    q.Unbind(syscall.AF_INET)
    q.Bind(syscall.AF_INET)
    q.CreateQueue(0)

    c := make(chan os.Signal, 1)
    signal.Notify(c, os.Interrupt)
    go func() {
        for sig := range c {
            _ = sig
            q.Close()
            err = ipt.ClearChain("filter", "INPUT")
            err = ipt.ClearChain("filter", "OUTPUT")
            if err != nil {
                panic(err)
            }
            os.Exit(0)
        }
    }()
    q.TryRun()
}

In case you are interested in the solution, code can be found here: https://github.com/kung-foo/freki