linux pcie【6】- epf驱动介绍
本文基于kernel的pci-epf-test例程，介绍一下如何实现linux pcie epf驱动，kernel版本为5.15。
首先通过pci_epf_register_driver注册一个pci_epf_driver。pci_epf_driver里主要的元素就是probe和ops。
921 static struct pci_epf_ops ops = {
922 .unbind = pci_epf_test_unbind,
923 .bind = pci_epf_test_bind,
924 };
925
926 static struct pci_epf_driver test_driver = {
927 .driver.name = "pci_epf_test",
928 .probe = pci_epf_test_probe,
929 .id_table = pci_epf_test_ids,
930 .ops = &ops,
931 .owner = THIS_MODULE,
932 };
933
934 static int __init pci_epf_test_init(void)
935 {
936 int ret;
937
938 kpcitest_workqueue = alloc_workqueue("kpcitest",
939 WQ_MEM_RECLAIM | WQ_HIGHPRI, 0);
940 if (!kpcitest_workqueue) {
941 pr_err("Failed to allocate the kpcitest work queue\n");
942 return -ENOMEM;
943 }
944
945 ret = pci_epf_register_driver(&test_driver);
946 if (ret) {
947 destroy_workqueue(kpcitest_workqueue);
948 pr_err("Failed to register pci epf test driver --> %d\n", ret);
949 return ret;
950 }
951
952 return 0;
953 }
954 module_init(pci_epf_test_init);
955
956 static void __exit pci_epf_test_exit(void)
957 {
958 if (kpcitest_workqueue)
959 destroy_workqueue(kpcitest_workqueue);
960 pci_epf_unregister_driver(&test_driver);
961 }
962 module_exit(pci_epf_test_exit);probe对应的是pci_epf_test_probe函数，它在对应的pci_epf被创建后被调用。我们后面的文章会介绍pci_epf如果被创建。
903 static int pci_epf_test_probe(struct pci_epf *epf)
904 {
905 struct pci_epf_test *epf_test;
906 struct device *dev = &epf->dev;
907
908 epf_test = devm_kzalloc(dev, sizeof(*epf_test), GFP_KERNEL);
909 if (!epf_test)
910 return -ENOMEM;
911
912 epf->header = &test_header;
913 epf_test->epf = epf;
914
915 INIT_DELAYED_WORK(&epf_test->cmd_handler, pci_epf_test_cmd_handler);
916
917 epf_set_drvdata(epf, epf_test);
918 return 0;
919 }
然后看一下pci_epf_ops，这里主要实现了pci_epf_test_bind和pci_epf_test_unbind掉。它们会在pci_epf绑定和解绑到pci_epc控制器时被调用。
我们先看一下pci_epf_test_bind。854行，获取一下控制器的特有参数。862行获取一下第一个有效的bar，用来映射虚拟的寄存器。870行用来给使用的bar分配ddr空间。875行用来配置控制器的配置空间。882行申请dma channel，通过dma的方式访问内存空间。888行，如果控制器支持linkup_notifier功能，用来注册notifier回调函数。
841 static int pci_epf_test_bind(struct pci_epf *epf)
842 {
843 int ret;
844 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
845 const struct pci_epc_features *epc_features;
846 enum pci_barno test_reg_bar = BAR_0;
847 struct pci_epc *epc = epf->epc;
848 bool linkup_notifier = false;
849 bool core_init_notifier = false;
850
851 if (WARN_ON_ONCE(!epc))
852 return -EINVAL;
853
854 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
855 if (!epc_features) {
856 dev_err(&epf->dev, "epc_features not implemented\n");
857 return -EOPNOTSUPP;
858 }
859
860 linkup_notifier = epc_features->linkup_notifier;
861 core_init_notifier = epc_features->core_init_notifier;
862 test_reg_bar = pci_epc_get_first_free_bar(epc_features);
863 if (test_reg_bar < 0)
864 return -EINVAL;
865 pci_epf_configure_bar(epf, epc_features);
866
867 epf_test->test_reg_bar = test_reg_bar;
868 epf_test->epc_features = epc_features;
869
870 ret = pci_epf_test_alloc_space(epf);
871 if (ret)
872 return ret;
873
874 if (!core_init_notifier) {
875 ret = pci_epf_test_core_init(epf);
876 if (ret)
877 return ret;
878 }
879
880 epf_test->dma_supported = true;
881
882 ret = pci_epf_test_init_dma_chan(epf_test);
883 if (ret)
884 epf_test->dma_supported = false;
885
886 if (linkup_notifier) {
887 epf->nb.notifier_call = pci_epf_test_notifier;
888 pci_epc_register_notifier(epc, &epf->nb);
889 } else {
890 queue_work(kpcitest_workqueue, &epf_test->cmd_handler.work);
891 }
892
893 return 0;
894 }155 /**
156 * struct pci_epc_features - features supported by a EPC device per function
157 * @linkup_notifier: indicate if the EPC device can notify EPF driver on link up
158 * @core_init_notifier: indicate cores that can notify about their availability
159 * for initialization
160 * @msi_capable: indicate if the endpoint function has MSI capability
161 * @msix_capable: indicate if the endpoint function has MSI-X capability
162 * @reserved_bar: bitmap to indicate reserved BAR unavailable to function driver
163 * @bar_fixed_64bit: bitmap to indicate fixed 64bit BARs
164 * @bar_fixed_size: Array specifying the size supported by each BAR
165 * @align: alignment size required for BAR buffer allocation
166 */
167 struct pci_epc_features {
168 unsigned int linkup_notifier : 1;
169 unsigned int core_init_notifier : 1;
170 unsigned int msi_capable : 1;
171 unsigned int msix_capable : 1;
172 u8 reserved_bar;
173 u8 bar_fixed_64bit;
174 u64 bar_fixed_size[PCI_STD_NUM_BARS];
175 size_t align;
176 };
我看一下pci_epf_test_alloc_space的实现，内部主要是调用pci_epf_alloc_spac来申请内存。
760 static int pci_epf_test_alloc_space(struct pci_epf *epf)
761 {
762 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
763 struct device *dev = &epf->dev;
764 struct pci_epf_bar *epf_bar;
765 size_t msix_table_size = 0;
766 size_t test_reg_bar_size;
767 size_t pba_size = 0;
768 bool msix_capable;
769 void *base;
770 int bar, add;
771 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
772 const struct pci_epc_features *epc_features;
773 size_t test_reg_size;
774
775 epc_features = epf_test->epc_features;
776
777 test_reg_bar_size = ALIGN(sizeof(struct pci_epf_test_reg), 128);
778
779 msix_capable = epc_features->msix_capable;
780 if (msix_capable) {
781 msix_table_size = PCI_MSIX_ENTRY_SIZE * epf->msix_interrupts;
782 epf_test->msix_table_offset = test_reg_bar_size;
783 /* Align to QWORD or 8 Bytes */
784 pba_size = ALIGN(DIV_ROUND_UP(epf->msix_interrupts, 8), 8);
785 }
786 test_reg_size = test_reg_bar_size + msix_table_size + pba_size;
787
788 if (epc_features->bar_fixed_size[test_reg_bar]) {
789 if (test_reg_size > bar_size[test_reg_bar])
790 return -ENOMEM;
791 test_reg_size = bar_size[test_reg_bar];
792 }
793
794 base = pci_epf_alloc_space(epf, test_reg_size, test_reg_bar,
795 epc_features->align, PRIMARY_INTERFACE);
796 if (!base) {
797 dev_err(dev, "Failed to allocated register space\n");
798 return -ENOMEM;
799 }
800 epf_test->reg[test_reg_bar] = base;
801
802 for (bar = 0; bar < PCI_STD_NUM_BARS; bar += add) {
803 epf_bar = &epf->bar[bar];
804 add = (epf_bar->flags & PCI_BASE_ADDRESS_MEM_TYPE_64) ? 2 : 1;
805
806 if (bar == test_reg_bar)
807 continue;
808
809 if (!!(epc_features->reserved_bar & (1 << bar)))
810 continue;
811
812 base = pci_epf_alloc_space(epf, bar_size[bar], bar,
813 epc_features->align,
814 PRIMARY_INTERFACE);
815 if (!base)
816 dev_err(dev, "Failed to allocate space for BAR%d\n",
817 bar);
818 epf_test->reg[bar] = base;
819 }
820
821 return 0;
822 }
我们再看一下pci_epf_alloc_spac，它是在epf的核心层实现的，主要是为bar空间分配了物理内存。
281 /**
282 * pci_epf_alloc_space() - allocate memory for the PCI EPF register space
283 * @epf: the EPF device to whom allocate the memory
284 * @size: the size of the memory that has to be allocated
285 * @bar: the BAR number corresponding to the allocated register space
286 * @align: alignment size for the allocation region
287 * @type: Identifies if the allocation is for primary EPC or secondary EPC
288 *
289 * Invoke to allocate memory for the PCI EPF register space.
290 */
291 void *pci_epf_alloc_space(struct pci_epf *epf, size_t size, enum pci_barno bar,
292 size_t align, enum pci_epc_interface_type type)
293 {
294 struct pci_epf_bar *epf_bar;
295 dma_addr_t phys_addr;
296 struct pci_epc *epc;
297 struct device *dev;
298 void *space;
299
300 if (size < 128)
301 size = 128;
302
303 if (align)
304 size = ALIGN(size, align);
305 else
306 size = roundup_pow_of_two(size);
307
308 if (type == PRIMARY_INTERFACE) {
309 epc = epf->epc;
310 epf_bar = epf->bar;
311 } else {
312 epc = epf->sec_epc;
313 epf_bar = epf->sec_epc_bar;
314 }
315
316 dev = epc->dev.parent;
317 space = dma_alloc_coherent(dev, size, &phys_addr, GFP_KERNEL);
318 if (!space) {
319 dev_err(dev, "failed to allocate mem space\n");
320 return NULL;
321 }
322
323 epf_bar[bar].phys_addr = phys_addr;
324 epf_bar[bar].addr = space;
325 epf_bar[bar].size = size;
326 epf_bar[bar].barno = bar;
327 epf_bar[bar].flags |= upper_32_bits(size) ?
328 PCI_BASE_ADDRESS_MEM_TYPE_64 :
329 PCI_BASE_ADDRESS_MEM_TYPE_32;
330
331 return space;
332 }
继续看一下pci_epf_test_core_init，699行将header信息写到控制器的配置空间里，709行主要是将bar地址写道控制器的bar寄存器里，并进行inbound映射。711行用来配置支持的msi数量。这些功能都是通过调用控制器的回调函数实现的。
681 static int pci_epf_test_core_init(struct pci_epf *epf)
682 {
683 struct pci_epf_test *epf_test = epf_get_drvdata(epf);
684 struct pci_epf_header *header = epf->header;
685 const struct pci_epc_features *epc_features;
686 struct pci_epc *epc = epf->epc;
687 struct device *dev = &epf->dev;
688 bool msix_capable = false;
689 bool msi_capable = true;
690 int ret;
691
692 epc_features = pci_epc_get_features(epc, epf->func_no, epf->vfunc_no);
693 if (epc_features) {
694 msix_capable = epc_features->msix_capable;
695 msi_capable = epc_features->msi_capable;
696 }
697
698 if (epf->vfunc_no <= 1) {
699 ret = pci_epc_write_header(epc, epf->func_no, epf->vfunc_no, header);
700 if (ret) {
701 dev_err(dev, "Configuration header write failed\n");
702 return ret;
703 }
704 }
705
706 ret = pci_epf_test_set_bar(epf);
707 if (ret)
708 return ret;
709
710 if (msi_capable) {
711 ret = pci_epc_set_msi(epc, epf->func_no, epf->vfunc_no,
712 epf->msi_interrupts);
713 if (ret) {
714 dev_err(dev, "MSI configuration failed\n");
715 return ret;
716 }
717 }
718
719 if (msix_capable) {
720 ret = pci_epc_set_msix(epc, epf->func_no, epf->vfunc_no,
721 epf->msix_interrupts,
722 epf_test->test_reg_bar,
723 epf_test->msix_table_offset);
724 if (ret) {
725 dev_err(dev, "MSI-X configuration failed\n");
726 return ret;
727 }
728 }
729
730 return 0;
731 }
我们以新思的控制器为例，看一下这些功能的具体实现
pci_epc_write_header->dw_pcie_ep_write_header
pci_epf_test_set_bar->dw_pcie_ep_set_bar
pci_epc_set_msi->dw_pcie_ep_set_msi
128 static int dw_pcie_ep_write_header(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
129 struct pci_epf_header *hdr)
130 {
131 struct dw_pcie_ep *ep = epc_get_drvdata(epc);
132 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
133 unsigned int func_offset = 0;
134
135 func_offset = dw_pcie_ep_func_select(ep, func_no);
136
137 dw_pcie_dbi_ro_wr_en(pci);
138 dw_pcie_writew_dbi(pci, func_offset + PCI_VENDOR_ID, hdr->vendorid);
139 dw_pcie_writew_dbi(pci, func_offset + PCI_DEVICE_ID, hdr->deviceid);
140 dw_pcie_writeb_dbi(pci, func_offset + PCI_REVISION_ID, hdr->revid);
141 dw_pcie_writeb_dbi(pci, func_offset + PCI_CLASS_PROG, hdr->progif_code);
142 dw_pcie_writew_dbi(pci, func_offset + PCI_CLASS_DEVICE,
143 hdr->subclass_code | hdr->baseclass_code << 8);
144 dw_pcie_writeb_dbi(pci, func_offset + PCI_CACHE_LINE_SIZE,
145 hdr->cache_line_size);
146 dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_VENDOR_ID,
147 hdr->subsys_vendor_id);
148 dw_pcie_writew_dbi(pci, func_offset + PCI_SUBSYSTEM_ID, hdr->subsys_id);
149 dw_pcie_writeb_dbi(pci, func_offset + PCI_INTERRUPT_PIN,
150 hdr->interrupt_pin);
151 dw_pcie_dbi_ro_wr_dis(pci);
152
153 return 0;
154 }220 static int dw_pcie_ep_set_bar(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
221 struct pci_epf_bar *epf_bar)
222 {
223 int ret;
224 struct dw_pcie_ep *ep = epc_get_drvdata(epc);
225 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
226 enum pci_barno bar = epf_bar->barno;
227 size_t size = epf_bar->size;
228 int flags = epf_bar->flags;
229 enum dw_pcie_as_type as_type;
230 u32 reg;
231 unsigned int func_offset = 0;
232
233 func_offset = dw_pcie_ep_func_select(ep, func_no);
234
235 reg = PCI_BASE_ADDRESS_0 + (4 * bar) + func_offset;
236
237 if (!(flags & PCI_BASE_ADDRESS_SPACE))
238 as_type = DW_PCIE_AS_MEM;
239 else
240 as_type = DW_PCIE_AS_IO;
241
242 ret = dw_pcie_ep_inbound_atu(ep, func_no, bar,
243 epf_bar->phys_addr, as_type);
244 if (ret)
245 return ret;
246
247 dw_pcie_dbi_ro_wr_en(pci);
248
249 dw_pcie_writel_dbi2(pci, reg, lower_32_bits(size - 1));
250 dw_pcie_writel_dbi(pci, reg, flags);
251
252 if (flags & PCI_BASE_ADDRESS_MEM_TYPE_64) {
253 dw_pcie_writel_dbi2(pci, reg + 4, upper_32_bits(size - 1));
254 dw_pcie_writel_dbi(pci, reg + 4, 0);
255 }
256
257 ep->epf_bar[bar] = epf_bar;
258 dw_pcie_dbi_ro_wr_dis(pci);
259
260 return 0;
261 }335 static int dw_pcie_ep_set_msi(struct pci_epc *epc, u8 func_no, u8 vfunc_no,
336 u8 interrupts)
337 {
338 struct dw_pcie_ep *ep = epc_get_drvdata(epc);
339 struct dw_pcie *pci = to_dw_pcie_from_ep(ep);
340 u32 val, reg;
341 unsigned int func_offset = 0;
342 struct dw_pcie_ep_func *ep_func;
343
344 ep_func = dw_pcie_ep_get_func_from_ep(ep, func_no);
345 if (!ep_func || !ep_func->msi_cap)
346 return -EINVAL;
347
348 func_offset = dw_pcie_ep_func_select(ep, func_no);
349
350 reg = ep_func->msi_cap + func_offset + PCI_MSI_FLAGS;
351 val = dw_pcie_readw_dbi(pci, reg);
352 val &= ~PCI_MSI_FLAGS_QMASK;
353 val |= (interrupts << 1) & PCI_MSI_FLAGS_QMASK;
354 dw_pcie_dbi_ro_wr_en(pci);
355 dw_pcie_writew_dbi(pci, reg, val);
356 dw_pcie_dbi_ro_wr_dis(pci);
357
358 return 0;
359 }
pci_epf_test驱动会将第一个bar当作寄存器使用，主机驱动会像里面写命令，pci_epf_test驱动轮询这些命令，并进入对应的处理函数进入处理，处理完成后通过msi中断通知host完成。我们不一一介绍里面的命令了，只看一个典型的COMMAND_COPY命令是如何处理的。242行调用pci_epc_mem_alloc，分配一段控制，这里是从epc->windows里分配的，也就是epc的outbound空间，src_phys_addr为物理地址，src_addr为虚拟地址。250行调用pci_epc_map_addr将host驱动传递过来的reg->src_addr和src_phys_addr建立映射关系，也就是配置outbound寄存器，注意reg->src_addr是pci总线地址，它对应的是host内存的一段空间，这个pci总线地址和host内存地址在host上会使用inbound进行映射。258-272上对copy的dst作相同的操作。所以COMMAND_COPY实现的功能是使用pci_epf_test对host上的两段ddr空间进行copy。276-299行，进行具体的copy操作，可以使用cpu或dma进行。
227 static int pci_epf_test_copy(struct pci_epf_test *epf_test)
228 {
229 int ret;
230 bool use_dma;
231 void __iomem *src_addr;
232 void __iomem *dst_addr;
233 phys_addr_t src_phys_addr;
234 phys_addr_t dst_phys_addr;
235 struct timespec64 start, end;
236 struct pci_epf *epf = epf_test->epf;
237 struct device *dev = &epf->dev;
238 struct pci_epc *epc = epf->epc;
239 enum pci_barno test_reg_bar = epf_test->test_reg_bar;
240 struct pci_epf_test_reg *reg = epf_test->reg[test_reg_bar];
241
242 src_addr = pci_epc_mem_alloc_addr(epc, &src_phys_addr, reg->size);
243 if (!src_addr) {
244 dev_err(dev, "Failed to allocate source address\n");
245 reg->status = STATUS_SRC_ADDR_INVALID;
246 ret = -ENOMEM;
247 goto err;
248 }
249
250 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr,
251 reg->src_addr, reg->size);
252 if (ret) {
253 dev_err(dev, "Failed to map source address\n");
254 reg->status = STATUS_SRC_ADDR_INVALID;
255 goto err_src_addr;
256 }
257
258 dst_addr = pci_epc_mem_alloc_addr(epc, &dst_phys_addr, reg->size);
259 if (!dst_addr) {
260 dev_err(dev, "Failed to allocate destination address\n");
261 reg->status = STATUS_DST_ADDR_INVALID;
262 ret = -ENOMEM;
263 goto err_src_map_addr;
264 }
265
266 ret = pci_epc_map_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr,
267 reg->dst_addr, reg->size);
268 if (ret) {
269 dev_err(dev, "Failed to map destination address\n");
270 reg->status = STATUS_DST_ADDR_INVALID;
271 goto err_dst_addr;
272 }
273
274 ktime_get_ts64(&start);
275 use_dma = !!(reg->flags & FLAG_USE_DMA);
276 if (use_dma) {
277 if (!epf_test->dma_supported) {
278 dev_err(dev, "Cannot transfer data using DMA\n");
279 ret = -EINVAL;
280 goto err_map_addr;
281 }
282
283 ret = pci_epf_test_data_transfer(epf_test, dst_phys_addr,
284 src_phys_addr, reg->size);
285 if (ret)
286 dev_err(dev, "Data transfer failed\n");
287 } else {
288 void *buf;
289
290 buf = kzalloc(reg->size, GFP_KERNEL);
291 if (!buf) {
292 ret = -ENOMEM;
293 goto err_map_addr;
294 }
295
296 memcpy_fromio(buf, src_addr, reg->size);
297 memcpy_toio(dst_addr, buf, reg->size);
298 kfree(buf);
299 }
300 ktime_get_ts64(&end);
301 pci_epf_test_print_rate("COPY", reg->size, &start, &end, use_dma);
302
303 err_map_addr:
304 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, dst_phys_addr);
305
306 err_dst_addr:
307 pci_epc_mem_free_addr(epc, dst_phys_addr, dst_addr, reg->size);
308
309 err_src_map_addr:
310 pci_epc_unmap_addr(epc, epf->func_no, epf->vfunc_no, src_phys_addr);
311
312 err_src_addr:
313 pci_epc_mem_free_addr(epc, src_phys_addr, src_addr, reg->size);
314
315 err:
316 return ret;
317 }
我们再看一下pci_epf_test对应的主机驱动，它是在drivers/misc/pci_endpoint_test.c里实现的，我们这里只看COMMAND_COPY的处理过程。379行申请一段内存作为copy操作的src，387行获取对应的物理地址orig_src_phys_addr。395-402，如果需要对齐访问，对src地址做下对齐。404-409将物理地址写入设备寄存器，这个地址设备是作为pci地址使用的，所以对于host来说，inbound访问时AXI地址和pci地址是相同的，我们前面的文章分析rc控制器驱动时确实也是这样。412-439行，同上处理dst。444-455行，触发COMMAND_COPY命令，等待copy完成，并比较正确性。
335 static bool pci_endpoint_test_copy(struct pci_endpoint_test *test,
336 unsigned long arg)
337 {
338 struct pci_endpoint_test_xfer_param param;
339 bool ret = false;
340 void *src_addr;
341 void *dst_addr;
342 u32 flags = 0;
343 bool use_dma;
344 size_t size;
345 dma_addr_t src_phys_addr;
346 dma_addr_t dst_phys_addr;
347 struct pci_dev *pdev = test->pdev;
348 struct device *dev = &pdev->dev;
349 void *orig_src_addr;
350 dma_addr_t orig_src_phys_addr;
351 void *orig_dst_addr;
352 dma_addr_t orig_dst_phys_addr;
353 size_t offset;
354 size_t alignment = test->alignment;
355 int irq_type = test->irq_type;
356 u32 src_crc32;
357 u32 dst_crc32;
358 int err;
359
360 err = copy_from_user(¶m, (void __user *)arg, sizeof(param));
361 if (err) {
362 dev_err(dev, "Failed to get transfer param\n");
363 return false;
364 }
365
366 size = param.size;
367 if (size > SIZE_MAX - alignment)
368 goto err;
369
370 use_dma = !!(param.flags & PCITEST_FLAGS_USE_DMA);
371 if (use_dma)
372 flags |= FLAG_USE_DMA;
373
374 if (irq_type < IRQ_TYPE_LEGACY || irq_type > IRQ_TYPE_MSIX) {
375 dev_err(dev, "Invalid IRQ type option\n");
376 goto err;
377 }
378
379 orig_src_addr = kzalloc(size + alignment, GFP_KERNEL);
380 if (!orig_src_addr) {
381 dev_err(dev, "Failed to allocate source buffer\n");
382 ret = false;
383 goto err;
384 }
385
386 get_random_bytes(orig_src_addr, size + alignment);
387 orig_src_phys_addr = dma_map_single(dev, orig_src_addr,
388 size + alignment, DMA_TO_DEVICE);
389 if (dma_mapping_error(dev, orig_src_phys_addr)) {
390 dev_err(dev, "failed to map source buffer address\n");
391 ret = false;
392 goto err_src_phys_addr;
393 }
394
395 if (alignment && !IS_ALIGNED(orig_src_phys_addr, alignment)) {
396 src_phys_addr = PTR_ALIGN(orig_src_phys_addr, alignment);
397 offset = src_phys_addr - orig_src_phys_addr;
398 src_addr = orig_src_addr + offset;
399 } else {
400 src_phys_addr = orig_src_phys_addr;
401 src_addr = orig_src_addr;
402 }
403
404 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_SRC_ADDR,
405 lower_32_bits(src_phys_addr));
406
407 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_SRC_ADDR,
408 upper_32_bits(src_phys_addr));
409
410 src_crc32 = crc32_le(~0, src_addr, size);
411
412 orig_dst_addr = kzalloc(size + alignment, GFP_KERNEL);
413 if (!orig_dst_addr) {
414 dev_err(dev, "Failed to allocate destination address\n");
415 ret = false;
416 goto err_dst_addr;
417 }
418
419 orig_dst_phys_addr = dma_map_single(dev, orig_dst_addr,
420 size + alignment, DMA_FROM_DEVICE);
421 if (dma_mapping_error(dev, orig_dst_phys_addr)) {
422 dev_err(dev, "failed to map destination buffer address\n");
423 ret = false;
424 goto err_dst_phys_addr;
425 }
426
427 if (alignment && !IS_ALIGNED(orig_dst_phys_addr, alignment)) {
428 dst_phys_addr = PTR_ALIGN(orig_dst_phys_addr, alignment);
429 offset = dst_phys_addr - orig_dst_phys_addr;
430 dst_addr = orig_dst_addr + offset;
431 } else {
432 dst_phys_addr = orig_dst_phys_addr;
433 dst_addr = orig_dst_addr;
434 }
435
436 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_LOWER_DST_ADDR,
437 lower_32_bits(dst_phys_addr));
438 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_UPPER_DST_ADDR,
439 upper_32_bits(dst_phys_addr));
440
441 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_SIZE,
442 size);
443
444 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_FLAGS, flags);
445 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_TYPE, irq_type);
446 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_IRQ_NUMBER, 1);
447 pci_endpoint_test_writel(test, PCI_ENDPOINT_TEST_COMMAND,
448 COMMAND_COPY);
449
450 wait_for_completion(&test->irq_raised);
451
452 dma_unmap_single(dev, orig_dst_phys_addr, size + alignment,
453 DMA_FROM_DEVICE);
454
455 dst_crc32 = crc32_le(~0, dst_addr, size);
456 if (dst_crc32 == src_crc32)
457 ret = true;
458
459 err_dst_phys_addr:
460 kfree(orig_dst_addr);
461
462 err_dst_addr:
463 dma_unmap_single(dev, orig_src_phys_addr, size + alignment,
464 DMA_TO_DEVICE);
465
466 err_src_phys_addr:
467 kfree(orig_src_addr);
468
469 err:
470 return ret;
471 }