dpdk

DPDK pdump failed to hotplug add device


I'm trying to use dpdk-pdump capture tx packets from NIC under dpdk control.

Setup

Network devices using DPDK-compatible driver
============================================
0000:03:00.0 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' drv=igb_uio unused=ixgbe,uio_pci_generic

Network devices using kernel driver
===================================
0000:03:00.1 '82599ES 10-Gigabit SFI/SFP+ Network Connection 10fb' if=ens1f1 drv=ixgbe unused=igb_uio,uio_pci_generic
0000:05:00.0 'I210 Gigabit Network Connection 1533' if=enp5s0 drv=igb unused=igb_uio,uio_pci_generic *Active*
0000:06:00.0 'I210 Gigabit Network Connection 1533' if=enp6s0 drv=igb unused=igb_uio,uio_pci_generic

Output

primary process

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Probing VFIO support...
EAL: PCI device 0000:03:00.0 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:03:00.1 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: PCI device 0000:06:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
INFO: eth dev count 1.
Port 0 MAC: 9c 69 b4 60 90 1c

WARNING: Too many lcores enabled. Only 1 used.

Core 0 forwarding packets. [Ctrl+C to quit]
EAL: failed to parse device "vdev:net_pcap_tx_0"
EAL: Failed to hotplug add device on primary

secondary process

run by sudo ./build/app/dpdk-pdump -- --pdump 'port=0,queue=*,tx-dev=./tx.pcap'

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket_16018_96447662088dc
EAL: Probing VFIO support...
EAL: PCI device 0000:03:00.0 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:03:00.1 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: PCI device 0000:06:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: Failed to hotplug add device
EAL: Error - exiting with code: 1
  Cause: vdev creation failed

Expectation

How can I get dpdk-pdump right?

[EDIT] UPDATE 2020/7/12

After modifying skeleton(fix port to 0 and add rte_pdump_init/uninit()), it still failed to work.

PS: skeleton and my program are built with shared library.

skeleton code

static __attribute__((noreturn)) void
lcore_main(void)
{
    uint16_t port;

    /*
     * Check that the port is on the same NUMA node as the polling thread
     * for best performance.
     */
    RTE_ETH_FOREACH_DEV(port)
        if (rte_eth_dev_socket_id(port) > 0 &&
                rte_eth_dev_socket_id(port) !=
                        (int)rte_socket_id())
            printf("WARNING, port %u is on remote NUMA node to "
                    "polling thread.\n\tPerformance will "
                    "not be optimal.\n", port);

    printf("\nCore %u forwarding packets. [Ctrl+C to quit]\n",
            rte_lcore_id());

    /* Run until the application is quit or killed. */
    for (;;) {
        /*
         * Receive packets on a port and forward them on the paired
         * port. The mapping is 0 -> 1, 1 -> 0, 2 -> 3, 3 -> 2, etc.
         */
        port = 0;

        /* Get burst of RX packets, from first port of pair. */
        struct rte_mbuf *bufs[BURST_SIZE];
        const uint16_t nb_rx = rte_eth_rx_burst(port, 0,
                bufs, BURST_SIZE);

        if (unlikely(nb_rx == 0))
            continue;

        /* Send burst of TX packets, to second port of pair. */
        const uint16_t nb_tx = rte_eth_tx_burst(port ^ 1, 0,
                bufs, nb_rx);

        /* Free any unsent packets. */
        if (unlikely(nb_tx < nb_rx)) {
            uint16_t buf;
            for (buf = nb_tx; buf < nb_rx; buf++)
                rte_pktmbuf_free(bufs[buf]);
        }
    }
}

static void
signal_handler(int signum)
{
    if (signum == SIGINT || signum == SIGTERM) {
        printf("\nSignal %d received, preparing to exit...\n",
                signum);
        /* uninitialize packet capture framework */
        rte_pdump_uninit();
        /* exit with the expected status */
        signal(signum, SIG_DFL);
        kill(getpid(), signum);
    }
}

/*
 * The main function, which does initialization and calls the per-lcore
 * functions.
 */
int
main(int argc, char *argv[])
{
    struct rte_mempool *mbuf_pool;
    unsigned nb_ports;
    uint16_t portid;

    signal(SIGINT, signal_handler);
    signal(SIGTERM, signal_handler);

    /* Initialize the Environment Abstraction Layer (EAL). */
    int ret = rte_eal_init(argc, argv);
    if (ret < 0)
        rte_exit(EXIT_FAILURE, "Error with EAL initialization\n");

    argc -= ret;
    argv += ret;
    rte_pdump_init(NULL);
    /* Check that there is an even number of ports to send/receive on. */
    nb_ports = rte_eth_dev_count_avail();
    // if (nb_ports < 2 || (nb_ports & 1))
    //  rte_exit(EXIT_FAILURE, "Error: number of ports must be even\n");

    /* Creates a new mempool in memory to hold the mbufs. */
    mbuf_pool = rte_pktmbuf_pool_create("MBUF_POOL", NUM_MBUFS * nb_ports,
        MBUF_CACHE_SIZE, 0, RTE_MBUF_DEFAULT_BUF_SIZE, rte_socket_id());

    if (mbuf_pool == NULL)
        rte_exit(EXIT_FAILURE, "Cannot create mbuf pool\n");

    /* Initialize all ports. */
    RTE_ETH_FOREACH_DEV(portid)
        if (port_init(portid, mbuf_pool) != 0)
            rte_exit(EXIT_FAILURE, "Cannot init port %"PRIu16 "\n",
                    portid);

    if (rte_lcore_count() > 1)
        printf("\nWARNING: Too many lcores enabled. Only 1 used.\n");

    /* Call lcore_main on the master core only. */
    lcore_main();

    return 0;
}

primary process

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Probing VFIO support...
EAL: PCI device 0000:03:00.0 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:03:00.1 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: PCI device 0000:06:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
Port 0 MAC: 9c 69 b4 60 90 1c

WARNING: Too many lcores enabled. Only 1 used.

Core 0 forwarding packets. [Ctrl+C to quit]
EAL: Failed to hotplug add device on secondary

second process

command sudo ./build/app/dpdk-pdump -- --pdump 'port=0,queue=*,tx-dev=./tx.pcap'

EAL: Detected 24 lcore(s)
EAL: Detected 1 NUMA nodes
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket_27816_cdf9e536de2e0
EAL: Probing VFIO support...
EAL: PCI device 0000:03:00.0 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:03:00.1 on NUMA socket 0
EAL:   probe driver: 8086:10fb net_ixgbe
EAL: PCI device 0000:05:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: PCI device 0000:06:00.0 on NUMA socket 0
EAL:   probe driver: 8086:1533 net_e1000_igb
EAL: failed to parse device "vdev:net_pcap_tx_0"
EAL: failed to parse device "vdev:net_pcap_tx_0"
EAL: Failed to hotplug add device
EAL: Error - exiting with code: 1
  Cause: vdev creation failed

I also tried dpdk-pdump 9.4 example, the error messages are similar.

primary process

testpmd>
Port 0: link state change event
EAL: Failed to hotplug add device on secondary

second process

EAL: failed to parse device "vdev:net_pcap_rx_0"
EAL: failed to parse device "vdev:net_pcap_rx_0"
EAL: Failed to hotplug add device
EAL: Error - exiting with code: 1
  Cause: vdev creation failed:create_mp_ring_vdev:722

Solution

  • I am able to get it working properly without issues. Following is the steps followed

    1. DPDK: download 18.11.4 http://static.dpdk.org/rel/dpdk-18.11.4.tar.gz
    2. built DPDK with PCAP PMD enabled
    3. modify skeleton main: add rte_pdump_init(NULL) right after rte_eal_init
    4. modify skeleton lcore_main: modify RTE_ETH_FOREACH_DEV(port) with for (port = 0; port < 2; port++)
    5. bullt: LD_FLAGS="-lrte_pmd_pcap" make
    6. Run primary
    7. Run pdump secondary (if whitelist is passed in primary pass the same here to)
    EAL: request: eal_dev_mp_request
    EAL: msg: eal_dev_mp_request
    EAL: request: bus_vdev_mp
    EAL: msg: bus_vdev_mp
    EAL: msg: bus_vdev_mp
    EAL: reply: eal_dev_mp_request
    EAL: msg: eal_dev_mp_request
    Port 2 MAC: 02 70 63 61 70 00
    EAL: request: mp_pdump
    EAL: msg: mp_pdump