Linux设备驱动模型初始化流程分析（以PCI/PCIe模块为例）

Linux设备驱动模型（以PCI/PCIe为例）

本文以PCI/PCIe为切入点分析Linux设备驱动模型，并且以SMMU技术来分析PCI虚拟化的应用

关于UEFI/BIOS和BootLoader对PCI/PCIe链路的初始化过程此处不做分析，可自行查看edk2和uboot源码

此处以qcom/sm8250.dtsi为例

// linux-5.14.5/arch/arm64/boot/dts/qcom/sm8250.dtsi

		pcie2: pci@1c10000 {
   
			compatible = "qcom,pcie-sm8250", "snps,dw-pcie";
			reg = <0 0x01c10000 0 0x3000>,
			      <0 0x64000000 0 0xf1d>,
			      <0 0x64000f20 0 0xa8>,
			      <0 0x64001000 0 0x1000>,
			      <0 0x64100000 0 0x100000>;
			reg-names = "parf", "dbi", "elbi", "atu", "config";
			device_type = "pci";
			linux,pci-domain = <2>;
			bus-range = <0x00 0xff>;
			num-lanes = <2>;

			#address-cells = <3>;
			#size-cells = <2>;

			ranges = <0x01000000 0x0 0x64200000 0x0 0x64200000 0x0 0x100000>,
				 <0x02000000 0x0 0x64300000 0x0 0x64300000 0x0 0x3d00000>;

			interrupts = <GIC_SPI 236 IRQ_TYPE_EDGE_RISING>;
			interrupt-names = "msi";
			#interrupt-cells = <1>;
			interrupt-map-mask = <0 0 0 0x7>;
			interrupt-map = <0 0 0 1 &intc 0 290 IRQ_TYPE_LEVEL_HIGH>, /* int_a */
					<0 0 0 2 &intc 0 415 IRQ_TYPE_LEVEL_HIGH>, /* int_b */
					<0 0 0 3 &intc 0 416 IRQ_TYPE_LEVEL_HIGH>, /* int_c */
					<0 0 0 4 &intc 0 417 IRQ_TYPE_LEVEL_HIGH>; /* int_d */

			clocks = <&gcc GCC_PCIE_2_PIPE_CLK>,
				 <&gcc GCC_PCIE_2_AUX_CLK>,
				 <&gcc GCC_PCIE_2_CFG_AHB_CLK>,
				 <&gcc GCC_PCIE_2_MSTR_AXI_CLK>,
				 <&gcc GCC_PCIE_2_SLV_AXI_CLK>,
				 <&gcc GCC_PCIE_2_SLV_Q2A_AXI_CLK>,
				 <&gcc GCC_PCIE_MDM_CLKREF_EN>,
				 <&gcc GCC_AGGRE_NOC_PCIE_TBU_CLK>,
				 <&gcc GCC_DDRSS_PCIE_SF_TBU_CLK>;
			clock-names = "pipe",
				      "aux",
				      "cfg",
				      "bus_master",
				      "bus_slave",
				      "slave_q2a",
				      "ref",
				      "tbu",
				      "ddrss_sf_tbu";

			assigned-clocks = <&gcc GCC_PCIE_2_AUX_CLK>;
			assigned-clock-rates = <19200000>;

			iommus = <&apps_smmu 0x1d00 0x7f>;
			iommu-map = <0x0   &apps_smmu 0x1d00 0x1>,
				    <0x100 &apps_smmu 0x1d01 0x1>;

			resets = <&gcc GCC_PCIE_2_BCR>;
			reset-names = "pci";

			power-domains = <&gcc PCIE_2_GDSC>;

			phys = <&pcie2_lane>;
			phy-names = "pciephy";

			perst-gpio = <&tlmm 85 GPIO_ACTIVE_LOW>;
			enable-gpio = <&tlmm 87 GPIO_ACTIVE_HIGH>;

			pinctrl-names = "default";
			pinctrl-0 = <&pcie2_default_state>;

			status = "disabled";
		};

前期构建准备：setup_machine_fdt

在start_kernel过程中分析设备树文件，最终调用unflatten_device_tree

// linux-5.14.5/arch/arm64/kernel/setup.c

void __init __no_sanitize_address setup_arch(char **cmdline_p)
{
   
    	setup_machine_fdt(__fdt_pointer);
    
    	// ...
    
        /* Parse the ACPI tables for possible boot-time configuration */
        acpi_boot_table_init();
    
    	if (acpi_disabled)
			unflatten_device_tree();

在setup_machine_fdt中

// linux-5.14.5/arch/arm/kernel/devtree.c

const struct machine_desc * __init setup_machine_fdt(void *dt_virt)
{
   
	const struct machine_desc *mdesc, *mdesc_best = NULL;
    
	if (!dt_virt || !early_init_dt_verify(dt_virt))
		return NULL;

    mdesc = of_flat_dt_match_machine(mdesc_best, arch_get_next_mach);

在early_init_dt_verify中会检查设备树的一些标识，并赋值全局变量initial_boot_params等于设备树地址

// linux-5.14.5/drivers/of/fdt.c

bool __init early_init_dt_verify(void *params)
{
   
	if (!params)
		return false;

	/* check device tree validity */
	if (fdt_check_header(params))
		return false;

	/* Setup flat device-tree pointer */
	initial_boot_params = params;
	of_fdt_crc32 = crc32_be(~0, initial_boot_params,
				fdt_totalsize(initial_boot_params));
	return true;
}

of_flat_dt_match_machine用于读取compatible属性，并对mdesc变量（用于描述硬件单板信息）进行赋值

// linux-5.14.5/drivers/of/fdt.c

const void * __init of_flat_dt_match_machine(const void *default_match,
		const void * (*get_next_compat)(const char * const**))
{
   
    // ...
    
    dt_root = of_get_flat_dt_root();
	
    // ...
    
    pr_info("Machine model: %s\n", of_flat_dt_get_machine_name());

	return best_data;
}

在内核输出信息如下

[    0.000000] Machine model: Raspberry Pi 3 Model B

early_init_dt_scan_nodes用于扫描设备树的各节点，可见其主要分析三个节点

• early_init_dt_scan_chosen
• early_init_dt_scan_root
• early_init_dt_scan_memory

// linux-5.14.5/drivers/of/fdt.c

void __init early_init_dt_scan_nodes(void)
{
   
	int rc = 0;

	/* Retrieve various information from the /chosen node */
	rc = of_scan_flat_dt(early_init_dt_scan_chosen, boot_command_line);
	if (!rc)
		pr_warn("No chosen node found, continuing without\n");

	/* Initialize {size,address}-cells info */
	of_scan_flat_dt(early_init_dt_scan_root, NULL);

	/* Setup memory, calling early_init_dt_add_memory_arch */
	of_scan_flat_dt(early_init_dt_scan_memory, NULL);
}

并将chosen信息存入boot_command_line，作为启动参数，关于这三个节点，设备树中如下

// linux-5.14.5/arch/arm64/boot/dts/qcom/sm8250.dtsi

/
    chosen {
    };

	memory@80000000 {
   
		device_type = "memory";
		/* We expect the bootloader to fill in the size */
		reg = <0x0 0x80000000 0x0 0x0>;
	};

关于chosen的产生，有两种情况

• uboot基本上可以不通过显式的bootargs=xxx来传递给内核，而是在env拿出，并存放进设备树中的chosen节点中
• Linux也开始在设备树中的chosen节点中获取出来

而对于memory节点，则是将其地址范围加入memblock中进行管理，调用过程如下

// linux-5.14.5/drivers/of/fdt.c

of_scan_flat_dt(early_init_dt_scan_memory, NULL);
	early_init_dt_add_memory_arch(base, size);
		memblock_add(base, size);

并且在setup_machine_fdt之后就是memblock系统的初始化

// linux-5.14.5/arch/arm64/kernel/setup.c

void __init __no_sanitize_address setup_arch(char **cmdline_p)
{
   
    setup_machine_fdt(__fdt_pointer);
    
    arm64_memblock_init();

设备树解析：unflatten_device_tree

unflatten_device_tree

// linux-5.14.5/arch/arm64/kernel/setup.c

void __init __no_sanitize_address setup_arch(char **cmdline_p)
{
   
	/* Parse the ACPI tables for possible boot-time configuration */
	acpi_boot_table_init();

	if (acpi_disabled)
		unflatten_device_tree();

of_root是一个全局struct device_node根节点，链接了所有的struct device_node

// linux-5.14.5/drivers/of/base.c

struct device_node *of_root;

initial_boot_params前面已经指向了设备树地址

// linux-5.14.5/drivers/of/fdt.c

void __init unflatten_device_tree(void)
{
   
	__unflatten_device_tree(initial_boot_params, NULL, &of_root,
				early_init_dt_alloc_memory_arch, false);

	/* Get pointer to "/chosen" and "/aliases" nodes for use everywhere */
	of_alias_scan(early_init_dt_alloc_memory_arch);

	unittest_unflatten_overlay_base();
}

如下，可见会进行两次扫描

• 第一次扫描会转换成所有struct device_node需要的空间然后系统会申请空间dt_alloc（所有的节点都会形成struct device_node结构）
• 第二次扫描进行解析，重点在此

// linux-5.14.5/drivers/of/fdt.c

void *__unflatten_device_tree(const void *