Eric Lee / smarc-fsl-linux-kernel

Download as
Browse Code ยป

Commit 60d8cd572f655aac6107a2330dced004ad1fe3d7

Authored by Christoph Hellwig
1 parent 333478a7eb

arm64/xen: fix xen-swiotlb cache flushing 

Xen-swiotlb hooks into the arm/arm64 arch code through a copy of the DMA
DMA mapping operations stored in the struct device arch data.

Switching arm64 to use the direct calls for the merged DMA direct /
swiotlb code broke this scheme.  Replace the indirect calls with
direct-calls in xen-swiotlb as well to fix this problem.

Fixes: 356da6d0cde3 ("dma-mapping: bypass indirect calls for dma-direct")
Reported-by: Julien Grall <julien.grall@arm.com>
Signed-off-by: Christoph Hellwig <hch@lst.de>
Reviewed-by: Stefano Stabellini <sstabellini@kernel.org>

Showing 6 changed files with 176 additions and 102 deletions Side-by-side Diff

arch/arm/include/asm/xen/page-coherent.h
Diff comments   View file @ 60d8cd5
  1 +/* SPDX-License-Identifier: GPL-2.0 */
  2 +#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
  3 +#define _ASM_ARM_XEN_PAGE_COHERENT_H
  4 +
  5 +#include <linux/dma-mapping.h>
  6 +#include <asm/page.h>
1 7 #include <xen/arm/page-coherent.h>
  8 +
  9 +static inline const struct dma_map_ops *xen_get_dma_ops(struct device *dev)
  10 +{
  11 + if (dev && dev->archdata.dev_dma_ops)
  12 + return dev->archdata.dev_dma_ops;
  13 + return get_arch_dma_ops(NULL);
  14 +}
  15 +
  16 +static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
  17 + dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
  18 +{
  19 + return xen_get_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
  20 +}
  21 +
  22 +static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
  23 + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
  24 +{
  25 + xen_get_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
  26 +}
  27 +
  28 +static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
  29 + dma_addr_t dev_addr, unsigned long offset, size_t size,
  30 + enum dma_data_direction dir, unsigned long attrs)
  31 +{
  32 + unsigned long page_pfn = page_to_xen_pfn(page);
  33 + unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
  34 + unsigned long compound_pages =
  35 + (1<<compound_order(page)) * XEN_PFN_PER_PAGE;
  36 + bool local = (page_pfn <= dev_pfn) &&
  37 + (dev_pfn - page_pfn < compound_pages);
  38 +
  39 + /*
  40 + * Dom0 is mapped 1:1, while the Linux page can span across
  41 + * multiple Xen pages, it's not possible for it to contain a
  42 + * mix of local and foreign Xen pages. So if the first xen_pfn
  43 + * == mfn the page is local otherwise it's a foreign page
  44 + * grant-mapped in dom0. If the page is local we can safely
  45 + * call the native dma_ops function, otherwise we call the xen
  46 + * specific function.
  47 + */
  48 + if (local)
  49 + xen_get_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
  50 + else
  51 + __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
  52 +}
  53 +
  54 +static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
  55 + size_t size, enum dma_data_direction dir, unsigned long attrs)
  56 +{
  57 + unsigned long pfn = PFN_DOWN(handle);
  58 + /*
  59 + * Dom0 is mapped 1:1, while the Linux page can be spanned accross
  60 + * multiple Xen page, it's not possible to have a mix of local and
  61 + * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
  62 + * foreign mfn will always return false. If the page is local we can
  63 + * safely call the native dma_ops function, otherwise we call the xen
  64 + * specific function.
  65 + */
  66 + if (pfn_valid(pfn)) {
  67 + if (xen_get_dma_ops(hwdev)->unmap_page)
  68 + xen_get_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
  69 + } else
  70 + __xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
  71 +}
  72 +
  73 +static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
  74 + dma_addr_t handle, size_t size, enum dma_data_direction dir)
  75 +{
  76 + unsigned long pfn = PFN_DOWN(handle);
  77 + if (pfn_valid(pfn)) {
  78 + if (xen_get_dma_ops(hwdev)->sync_single_for_cpu)
  79 + xen_get_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
  80 + } else
  81 + __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
  82 +}
  83 +
  84 +static inline void xen_dma_sync_single_for_device(struct device *hwdev,
  85 + dma_addr_t handle, size_t size, enum dma_data_direction dir)
  86 +{
  87 + unsigned long pfn = PFN_DOWN(handle);
  88 + if (pfn_valid(pfn)) {
  89 + if (xen_get_dma_ops(hwdev)->sync_single_for_device)
  90 + xen_get_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
  91 + } else
  92 + __xen_dma_sync_single_for_device(hwdev, handle, size, dir);
  93 +}
  94 +
  95 +#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
arch/arm64/include/asm/device.h
Diff comments   View file @ 60d8cd5
... ... @@ -20,9 +20,6 @@
20 20 #ifdef CONFIG_IOMMU_API
21 21 void *iommu; /* private IOMMU data */
22 22 #endif
23   -#ifdef CONFIG_XEN
24   - const struct dma_map_ops *dev_dma_ops;
25   -#endif
26 23 };
27 24  
28 25 struct pdev_archdata {
arch/arm64/include/asm/xen/page-coherent.h
Diff comments   View file @ 60d8cd5
  1 +/* SPDX-License-Identifier: GPL-2.0 */
  2 +#ifndef _ASM_ARM64_XEN_PAGE_COHERENT_H
  3 +#define _ASM_ARM64_XEN_PAGE_COHERENT_H
  4 +
  5 +#include <linux/dma-mapping.h>
  6 +#include <asm/page.h>
1 7 #include <xen/arm/page-coherent.h>
  8 +
  9 +static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
  10 + dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
  11 +{
  12 + return dma_direct_alloc(hwdev, size, dma_handle, flags, attrs);
  13 +}
  14 +
  15 +static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
  16 + void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
  17 +{
  18 + dma_direct_free(hwdev, size, cpu_addr, dma_handle, attrs);
  19 +}
  20 +
  21 +static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
  22 + dma_addr_t handle, size_t size, enum dma_data_direction dir)
  23 +{
  24 + unsigned long pfn = PFN_DOWN(handle);
  25 +
  26 + if (pfn_valid(pfn))
  27 + dma_direct_sync_single_for_cpu(hwdev, handle, size, dir);
  28 + else
  29 + __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
  30 +}
  31 +
  32 +static inline void xen_dma_sync_single_for_device(struct device *hwdev,
  33 + dma_addr_t handle, size_t size, enum dma_data_direction dir)
  34 +{
  35 + unsigned long pfn = PFN_DOWN(handle);
  36 + if (pfn_valid(pfn))
  37 + dma_direct_sync_single_for_device(hwdev, handle, size, dir);
  38 + else
  39 + __xen_dma_sync_single_for_device(hwdev, handle, size, dir);
  40 +}
  41 +
  42 +static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
  43 + dma_addr_t dev_addr, unsigned long offset, size_t size,
  44 + enum dma_data_direction dir, unsigned long attrs)
  45 +{
  46 + unsigned long page_pfn = page_to_xen_pfn(page);
  47 + unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
  48 + unsigned long compound_pages =
  49 + (1<<compound_order(page)) * XEN_PFN_PER_PAGE;
  50 + bool local = (page_pfn <= dev_pfn) &&
  51 + (dev_pfn - page_pfn < compound_pages);
  52 +
  53 + if (local)
  54 + dma_direct_map_page(hwdev, page, offset, size, dir, attrs);
  55 + else
  56 + __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
  57 +}
  58 +
  59 +static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
  60 + size_t size, enum dma_data_direction dir, unsigned long attrs)
  61 +{
  62 + unsigned long pfn = PFN_DOWN(handle);
  63 + /*
  64 + * Dom0 is mapped 1:1, while the Linux page can be spanned accross
  65 + * multiple Xen page, it's not possible to have a mix of local and
  66 + * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
  67 + * foreign mfn will always return false. If the page is local we can
  68 + * safely call the native dma_ops function, otherwise we call the xen
  69 + * specific function.
  70 + */
  71 + if (pfn_valid(pfn))
  72 + dma_direct_unmap_page(hwdev, handle, size, dir, attrs);
  73 + else
  74 + __xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
  75 +}
  76 +
  77 +#endif /* _ASM_ARM64_XEN_PAGE_COHERENT_H */
arch/arm64/mm/dma-mapping.c
Diff comments   View file @ 60d8cd5
... ... @@ -466,10 +466,8 @@
466 466 __iommu_setup_dma_ops(dev, dma_base, size, iommu);
467 467  
468 468 #ifdef CONFIG_XEN
469   - if (xen_initial_domain()) {
470   - dev->archdata.dev_dma_ops = dev->dma_ops;
  469 + if (xen_initial_domain())
471 470 dev->dma_ops = xen_dma_ops;
472   - }
473 471 #endif
474 472 }
drivers/xen/swiotlb-xen.c
Diff comments   View file @ 60d8cd5
... ... @@ -645,7 +645,7 @@
645 645 void *cpu_addr, dma_addr_t dma_addr, size_t size,
646 646 unsigned long attrs)
647 647 {
648   -#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
  648 +#ifdef CONFIG_ARM
649 649 if (xen_get_dma_ops(dev)->mmap)
650 650 return xen_get_dma_ops(dev)->mmap(dev, vma, cpu_addr,
651 651 dma_addr, size, attrs);
... ... @@ -662,7 +662,7 @@
662 662 void *cpu_addr, dma_addr_t handle, size_t size,
663 663 unsigned long attrs)
664 664 {
665   -#if defined(CONFIG_ARM) || defined(CONFIG_ARM64)
  665 +#ifdef CONFIG_ARM
666 666 if (xen_get_dma_ops(dev)->get_sgtable) {
667 667 #if 0
668 668 /*
include/xen/arm/page-coherent.h
Diff comments   View file @ 60d8cd5
1 1 /* SPDX-License-Identifier: GPL-2.0 */
2   -#ifndef _ASM_ARM_XEN_PAGE_COHERENT_H
3   -#define _ASM_ARM_XEN_PAGE_COHERENT_H
  2 +#ifndef _XEN_ARM_PAGE_COHERENT_H
  3 +#define _XEN_ARM_PAGE_COHERENT_H
4 4  
5   -#include <asm/page.h>
6   -#include <asm/dma-mapping.h>
7   -#include <linux/dma-mapping.h>
8   -
9   -static inline const struct dma_map_ops *xen_get_dma_ops(struct device *dev)
10   -{
11   - if (dev && dev->archdata.dev_dma_ops)
12   - return dev->archdata.dev_dma_ops;
13   - return get_arch_dma_ops(NULL);
14   -}
15   -
16 5 void __xen_dma_map_page(struct device *hwdev, struct page *page,
17 6 dma_addr_t dev_addr, unsigned long offset, size_t size,
18 7 enum dma_data_direction dir, unsigned long attrs);
19 8  
... ... @@ -21,88 +10,8 @@
21 10 unsigned long attrs);
22 11 void __xen_dma_sync_single_for_cpu(struct device *hwdev,
23 12 dma_addr_t handle, size_t size, enum dma_data_direction dir);
24   -
25 13 void __xen_dma_sync_single_for_device(struct device *hwdev,
26 14 dma_addr_t handle, size_t size, enum dma_data_direction dir);
27 15  
28   -static inline void *xen_alloc_coherent_pages(struct device *hwdev, size_t size,
29   - dma_addr_t *dma_handle, gfp_t flags, unsigned long attrs)
30   -{
31   - return xen_get_dma_ops(hwdev)->alloc(hwdev, size, dma_handle, flags, attrs);
32   -}
33   -
34   -static inline void xen_free_coherent_pages(struct device *hwdev, size_t size,
35   - void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs)
36   -{
37   - xen_get_dma_ops(hwdev)->free(hwdev, size, cpu_addr, dma_handle, attrs);
38   -}
39   -
40   -static inline void xen_dma_map_page(struct device *hwdev, struct page *page,
41   - dma_addr_t dev_addr, unsigned long offset, size_t size,
42   - enum dma_data_direction dir, unsigned long attrs)
43   -{
44   - unsigned long page_pfn = page_to_xen_pfn(page);
45   - unsigned long dev_pfn = XEN_PFN_DOWN(dev_addr);
46   - unsigned long compound_pages =
47   - (1<<compound_order(page)) * XEN_PFN_PER_PAGE;
48   - bool local = (page_pfn <= dev_pfn) &&
49   - (dev_pfn - page_pfn < compound_pages);
50   -
51   - /*
52   - * Dom0 is mapped 1:1, while the Linux page can span across
53   - * multiple Xen pages, it's not possible for it to contain a
54   - * mix of local and foreign Xen pages. So if the first xen_pfn
55   - * == mfn the page is local otherwise it's a foreign page
56   - * grant-mapped in dom0. If the page is local we can safely
57   - * call the native dma_ops function, otherwise we call the xen
58   - * specific function.
59   - */
60   - if (local)
61   - xen_get_dma_ops(hwdev)->map_page(hwdev, page, offset, size, dir, attrs);
62   - else
63   - __xen_dma_map_page(hwdev, page, dev_addr, offset, size, dir, attrs);
64   -}
65   -
66   -static inline void xen_dma_unmap_page(struct device *hwdev, dma_addr_t handle,
67   - size_t size, enum dma_data_direction dir, unsigned long attrs)
68   -{
69   - unsigned long pfn = PFN_DOWN(handle);
70   - /*
71   - * Dom0 is mapped 1:1, while the Linux page can be spanned accross
72   - * multiple Xen page, it's not possible to have a mix of local and
73   - * foreign Xen page. Dom0 is mapped 1:1, so calling pfn_valid on a
74   - * foreign mfn will always return false. If the page is local we can
75   - * safely call the native dma_ops function, otherwise we call the xen
76   - * specific function.
77   - */
78   - if (pfn_valid(pfn)) {
79   - if (xen_get_dma_ops(hwdev)->unmap_page)
80   - xen_get_dma_ops(hwdev)->unmap_page(hwdev, handle, size, dir, attrs);
81   - } else
82   - __xen_dma_unmap_page(hwdev, handle, size, dir, attrs);
83   -}
84   -
85   -static inline void xen_dma_sync_single_for_cpu(struct device *hwdev,
86   - dma_addr_t handle, size_t size, enum dma_data_direction dir)
87   -{
88   - unsigned long pfn = PFN_DOWN(handle);
89   - if (pfn_valid(pfn)) {
90   - if (xen_get_dma_ops(hwdev)->sync_single_for_cpu)
91   - xen_get_dma_ops(hwdev)->sync_single_for_cpu(hwdev, handle, size, dir);
92   - } else
93   - __xen_dma_sync_single_for_cpu(hwdev, handle, size, dir);
94   -}
95   -
96   -static inline void xen_dma_sync_single_for_device(struct device *hwdev,
97   - dma_addr_t handle, size_t size, enum dma_data_direction dir)
98   -{
99   - unsigned long pfn = PFN_DOWN(handle);
100   - if (pfn_valid(pfn)) {
101   - if (xen_get_dma_ops(hwdev)->sync_single_for_device)
102   - xen_get_dma_ops(hwdev)->sync_single_for_device(hwdev, handle, size, dir);
103   - } else
104   - __xen_dma_sync_single_for_device(hwdev, handle, size, dir);
105   -}
106   -
107   -#endif /* _ASM_ARM_XEN_PAGE_COHERENT_H */
  16 +#endif /* _XEN_ARM_PAGE_COHERENT_H */