I am attempting to configure the PCIe x4 connection on my ROCKPro64 (RK3399 processor) development board to act as a PCIe endpoint device. My goal is to get utilize Linux's PCIe Endpoint Framework test driver in a similar manner as described in this tutorial https://docs.nxp.com/bundle/GUID-3FFCCD77-5220-414D-8664-09E6FB1B02C6/page/GUID-1B638C76-E8F9-4FF8-A140-B934DC5055F0.html.
I've been stuck on this issue for more than I would like to admit and any help would be greatly appreciated.
I am currently using a custom Armbian build which has the following build configuration: enabled PCI Endpoint drivers in the kernel, kernel v5.15.y,and Ubuntu Focal minimal command line. I am using a Device Tree Overlay to configure the PCIe endpoint.
/dts-v1/;
/plugin/;
/ {
fragment@0 {
target-path = "/pcie@f8000000";
__overlay__ {
status = "disabled";
};
};
fragment@1 {
target-path = "/";
__overlay__ {
pcie_ep@f8000000{
compatible = "rockchip,rk3399-pcie-ep";
#address-cells = <0x03>;
#size-cells = <0x02>;
rockchip,max-outbound-regions = <0x10>;
clocks = <0x08 0xc5 0x08 0xc4 0x08 0x147 0x08 0xa0>;
clock-names = "aclk\0aclk-perf\0hclk\0pm";
max-functions = [08];
num-lanes = <0x04>;
reg = <0x00 0xfd000000 0x00 0x1000000 0x00 0x80000000 0x00 0x20000>;
reg-names = "apb-base\0mem-base";
resets = <0x08 0x82 0x08 0x83 0x08 0x84 0x08 0x85 0x08 0x86 0x08 0x81 0x08 0x80>;
reset-names = "core\0mgmt\0mgmt-sticky\0pipe\0pm\0pclk\0aclk";
phys = <0x15 0x00 0x15 0x01 0x15 0x02 0x15 0x03>;
phy-names = "pcie-phy-0\0pcie-phy-1\0pcie-phy-2\0pcie-phy-3";
pinctrl-names = "default";
status = "okay";
ep-gpios = <0x16 0x1c 0x00>;
};
};
};
// --- SNIP ---
};
Without pcie@f8000000 being disabled, the kernel hangs immediately when started (no output whatsoever). The board boots with this dtbo, but when the PCIe Endpoint driver is loaded, it is unable to allocate memory space for MSI interrupts.
See kernel log below:
[ 1.526603] ------------[ cut here ]------------
[ 1.526630] WARNING: CPU: 4 PID: 1 at arch/arm64/mm/ioremap.c:46 __ioremap_caller+0xcc/0xf0
[ 1.526676] Modules linked in:
[ 1.526696] CPU: 4 PID: 1 Comm: swapper/0 Not tainted 5.15.64-rockchip64 #trunk
[ 1.526720] Hardware name: Pine64 RockPro64 v2.1 (DT)
[ 1.526736] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 1.526759] pc : __ioremap_caller+0xcc/0xf0
[ 1.526781] lr : __ioremap_caller+0x54/0xf0
[ 1.526801] sp : ffff800009e0bad0
[ 1.526814] x29: ffff800009e0bad0 x28: 00000000ffffffff x27: 0000000000020000
[ 1.526848] x26: ffff000004135100 x25: 000000000001ffff x24: 0000000000000005
[ 1.526879] x23: 0068000000000713 x22: ffff80000884b0d0 x21: 0000000000000000
[ 1.526909] x20: 0000000080000000 x19: 0000000000020000 x18: 0000000000000000
[ 1.526938] x17: 6f7420746c756166 x16: 6564202c79616c65 x15: 0000ad574eaa2dce
[ 1.526968] x14: 000000000000035c x13: 0000000000000000 x12: 0000000000000020
[ 1.526997] x11: ffff000004135180 x10: 0000000000000005 x9 : 0000000000000020
[ 1.527025] x8 : 00000000f7e00000 x7 : 0000000000000018 x6 : ffff800009d5afd0
[ 1.527054] x5 : ffff800009d5afd0 x4 : 0000000000000001 x3 : 0000000000000001
[ 1.527082] x2 : 00000000f8000000 x1 : 0000000000000000 x0 : 0000000000000001
[ 1.527111] Call trace:
[ 1.527123] __ioremap_caller+0xcc/0xf0
[ 1.527145] __ioremap+0x28/0x58
[ 1.527166] pci_epc_mem_alloc_addr+0xe0/0x168
[ 1.527189] rockchip_pcie_ep_probe+0x1a8/0x248
[ 1.527209] platform_probe+0x68/0xd8
[ 1.527230] really_probe+0xbc/0x428
[ 1.527247] __driver_probe_device+0x114/0x188
[ 1.527264] driver_probe_device+0xb0/0x110
[ 1.527281] __driver_attach+0xbc/0x1c0
[ 1.527298] bus_for_each_dev+0x74/0xc8
[ 1.527320] driver_attach+0x24/0x30
[ 1.527336] bus_add_driver+0x1a8/0x250
[ 1.527357] driver_register+0x64/0x120
[ 1.527374] __platform_driver_register+0x28/0x38
[ 1.527394] rockchip_pcie_ep_driver_init+0x1c/0x28
[ 1.527418] do_one_initcall+0x4c/0x288
[ 1.527436] kernel_init_freeable+0x228/0x2b0
[ 1.527454] kernel_init+0x24/0x128
[ 1.527475] ret_from_fork+0x10/0x20
[ 1.527508] ---[ end trace d50b537f42cfef2e ]---
[ 1.527530] rockchip-pcie-ep fd000000.pcie_ep: failed to reserve memory space for MSI
[ 1.527570] rockchip-pcie-ep: probe of fd000000.pcie_ep failed with error -12
[ 1.528218] ------------[ cut here ]------------
I am also seeing an error regarding the rockchip driver not having a release function, but I do not believe this is too critical an issue right now. Could this be causing something unforeseen?
[ 1.528218] ------------[ cut here ]------------
[ 1.528237] Device 'fd000000.pcie_ep' does not have a release() function, it is broken and must be fixed. See Documentation/core-api/kobject.rst.
[ 1.528299] WARNING: CPU: 4 PID: 1 at drivers/base/core.c:2233 device_release+0x80/0x90
[ 1.528332] Modules linked in:
[ 1.528349] CPU: 4 PID: 1 Comm: swapper/0 Tainted: G W 5.15.64-rockchip64 #trunk
[ 1.528374] Hardware name: Pine64 RockPro64 v2.1 (DT)
[ 1.528388] pstate: 80000005 (Nzcv daif -PAN -UAO -TCO -DIT -SSBS BTYPE=--)
[ 1.528411] pc : device_release+0x80/0x90
[ 1.528429] lr : device_release+0x80/0x90
[ 1.528447] sp : ffff800009e0bab0
[ 1.528459] x29: ffff800009e0bab0 x28: 0000000000000000 x27: ffff8000096404a4
[ 1.528490] x26: ffff8000096d1960 x25: ffff800009416000 x24: ffff80000971d008
[ 1.528519] x23: ffff800009a899e8 x22: 0000000000000000 x21: ffff00000412cd80
[ 1.528548] x20: ffff000004130100 x19: ffff000000f32800 x18: 0000000000000010
[ 1.528577] x17: 6620656220747375 x16: 6d20646e61206e65 x15: 000000000000010f
[ 1.528605] x14: ffff800009e0b7c0 x13: 00000000ffffffea x12: ffff800009b1fd10
[ 1.528634] x11: 0000000000000003 x10: ffff800009b07cd0 x9 : ffff800009b07d28
[ 1.528663] x8 : 0000000000017fe8 x7 : c0000000ffffefff x6 : 0000000000000001
[ 1.528691] x5 : 0000000000057fa8 x4 : 0000000000000000 x3 : 0000000000000001
[ 1.528719] x2 : 0000000000000000 x1 : 142e7b8bc84f1100 x0 : 0000000000000000
[ 1.528747] Call trace:
[ 1.528760] device_release+0x80/0x90
[ 1.528779] kobject_put+0x7c/0x218
[ 1.528799] device_unregister+0x2c/0x78
[ 1.528819] pci_epc_destroy+0x24/0x40
[ 1.528837] devm_pci_epc_release+0x14/0x20
[ 1.528855] release_nodes+0x64/0x108
[ 1.528875] devres_release_all+0x90/0xf0
[ 1.528896] really_probe+0x1d8/0x428
[ 1.528912] __driver_probe_device+0x114/0x188
[ 1.528930] driver_probe_device+0xb0/0x110
[ 1.528947] __driver_attach+0xbc/0x1c0
[ 1.528963] bus_for_each_dev+0x74/0xc8
[ 1.528984] driver_attach+0x24/0x30
[ 1.529000] bus_add_driver+0x1a8/0x250
[ 1.529021] driver_register+0x64/0x120
[ 1.529038] __platform_driver_register+0x28/0x38
[ 1.529058] rockchip_pcie_ep_driver_init+0x1c/0x28
[ 1.529080] do_one_initcall+0x4c/0x288
[ 1.529097] kernel_init_freeable+0x228/0x2b0
[ 1.529116] kernel_init+0x24/0x128
[ 1.529134] ret_from_fork+0x10/0x20
[ 1.529152] ---[ end trace d50b537f42cfef2f ]---
rockchip-pcie-ep.txt binding documentation: https://android.googlesource.com/kernel/common/+/07e6124a1a46/Documentation/devicetree/bindings/pci/rockchip-pcie-ep.txt
Function call in question: https://github.com/torvalds/linux/blob/master/drivers/pci/endpoint/pci-epc-mem.c
void __iomem *pci_epc_mem_alloc_addr(struct pci_epc *epc,
phys_addr_t *phys_addr, size_t size)
{
void __iomem *virt_addr = NULL;
struct pci_epc_mem *mem;
unsigned int page_shift;
size_t align_size;
int pageno;
int order;
int i;
for (i = 0; i < epc->num_windows; i++) {
mem = epc->windows[i];
mutex_lock(&mem->lock);
align_size = ALIGN(size, mem->window.page_size);
order = pci_epc_mem_get_order(mem, align_size);
pageno = bitmap_find_free_region(mem->bitmap, mem->pages,
order);
if (pageno >= 0) {
page_shift = ilog2(mem->window.page_size);
*phys_addr = mem->window.phys_base +
((phys_addr_t)pageno << page_shift);
virt_addr = ioremap(*phys_addr, align_size);
if (!virt_addr) {
bitmap_release_region(mem->bitmap,
pageno, order);
mutex_unlock(&mem->lock);
continue;
}
mutex_unlock(&mem->lock);
return virt_addr;
}
mutex_unlock(&mem->lock);
}
return virt_addr;
}
I managed to figure it out! It was indeed a Device Tree issue with declaring a valid address for the mem-base register. Below is the device tree overlay which resolved the issue.
/dts-v1/;
/plugin/;
/ {
fragment@0 {
target-path = "/pcie@f8000000";
__overlay__ {
status = "disabled";
};
};
fragment@1 {
target-path = "/";
__overlay__ {
pcie_ep0: pcie_ep@f8000000 {
compatible = "rockchip,rk3399-pcie-ep";
#address-cells = <0x03>;
#size-cells = <0x02>;
rockchip,max-outbound-regions = <0x10>;
max-functions = [08];
num-lanes = <0x04>;
reg = <0x00 0xfd000000 0x00 0x1000000 0x00 0x00000000 0x00 0x20000>;
reg-names = "apb-base\0mem-base";
pinctrl-names = "default";
pinctrl-0 = <0x17>;
phys = <0x15 0x00 0x15 0x01 0x15 0x02 0x15 0x03>;
phy-names = "pcie-phy-0\0pcie-phy-1\0pcie-phy-2\0pcie-phy-3";
resets = <0x08 0x82 0x08 0x83 0x08 0x84 0x08 0x85 0x08 0x86 0x08 0x81 0x08 0x80>;
reset-names = "core\0mgmt\0mgmt-sticky\0pipe\0pm\0pclk\0aclk";
clocks = <0x08 0xc5 0x08 0xc4 0x08 0x147 0x08 0xa0>;
clock-names = "aclk\0aclk-perf\0hclk\0pm";
status = "okay";
};
};
};
};