diff options
Diffstat (limited to 'arch/arm/mm')
32 files changed, 212 insertions, 147 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig index dd4698c67cc3..baf638487a2d 100644 --- a/arch/arm/mm/Kconfig +++ b/arch/arm/mm/Kconfig @@ -1,9 +1,5 @@ comment "Processor Type" -config CPU_32 - bool - default y - # Select CPU types depending on the architecture selected. This selects # which CPUs we support in the kernel image, and the compiler instruction # optimiser behaviour. diff --git a/arch/arm/mm/cache-fa.S b/arch/arm/mm/cache-fa.S index b63a8f7b95cf..a89444a3c016 100644 --- a/arch/arm/mm/cache-fa.S +++ b/arch/arm/mm/cache-fa.S @@ -127,15 +127,16 @@ ENTRY(fa_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(kaddr) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - size of region */ -ENTRY(fa_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(fa_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -213,7 +214,7 @@ ENTRY(fa_cache_fns) .long fa_flush_user_cache_range .long fa_coherent_kern_range .long fa_coherent_user_range - .long fa_flush_kern_dcache_page + .long fa_flush_kern_dcache_area .long fa_dma_inv_range .long fa_dma_clean_range .long fa_dma_flush_range diff --git a/arch/arm/mm/cache-l2x0.c b/arch/arm/mm/cache-l2x0.c index 747f9a9021bb..cb8fc6573b1b 100644 --- a/arch/arm/mm/cache-l2x0.c +++ b/arch/arm/mm/cache-l2x0.c @@ -28,69 +28,120 @@ static void __iomem *l2x0_base; static DEFINE_SPINLOCK(l2x0_lock); -static inline void sync_writel(unsigned long val, unsigned long reg, - unsigned long complete_mask) +static inline void cache_wait(void __iomem *reg, unsigned long mask) { - unsigned long flags; - - spin_lock_irqsave(&l2x0_lock, flags); - writel(val, l2x0_base + reg); /* wait for the operation to complete */ - while (readl(l2x0_base + reg) & complete_mask) + while (readl(reg) & mask) ; - spin_unlock_irqrestore(&l2x0_lock, flags); } static inline void cache_sync(void) { - sync_writel(0, L2X0_CACHE_SYNC, 1); + void __iomem *base = l2x0_base; + writel(0, base + L2X0_CACHE_SYNC); + cache_wait(base + L2X0_CACHE_SYNC, 1); } static inline void l2x0_inv_all(void) { + unsigned long flags; + /* invalidate all ways */ - sync_writel(0xff, L2X0_INV_WAY, 0xff); + spin_lock_irqsave(&l2x0_lock, flags); + writel(0xff, l2x0_base + L2X0_INV_WAY); + cache_wait(l2x0_base + L2X0_INV_WAY, 0xff); cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_inv_range(unsigned long start, unsigned long end) { - unsigned long addr; + void __iomem *base = l2x0_base; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); if (start & (CACHE_LINE_SIZE - 1)) { start &= ~(CACHE_LINE_SIZE - 1); - sync_writel(start, L2X0_CLEAN_INV_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_INV_LINE_PA); start += CACHE_LINE_SIZE; } if (end & (CACHE_LINE_SIZE - 1)) { end &= ~(CACHE_LINE_SIZE - 1); - sync_writel(end, L2X0_CLEAN_INV_LINE_PA, 1); + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(end, base + L2X0_CLEAN_INV_LINE_PA); } - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_INV_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + cache_wait(base + L2X0_INV_LINE_PA, 1); + writel(start, base + L2X0_INV_LINE_PA); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } + cache_wait(base + L2X0_INV_LINE_PA, 1); cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_clean_range(unsigned long start, unsigned long end) { - unsigned long addr; + void __iomem *base = l2x0_base; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_CLEAN_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_LINE_PA); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } + cache_wait(base + L2X0_CLEAN_LINE_PA, 1); cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } static void l2x0_flush_range(unsigned long start, unsigned long end) { - unsigned long addr; + void __iomem *base = l2x0_base; + unsigned long flags; + spin_lock_irqsave(&l2x0_lock, flags); start &= ~(CACHE_LINE_SIZE - 1); - for (addr = start; addr < end; addr += CACHE_LINE_SIZE) - sync_writel(addr, L2X0_CLEAN_INV_LINE_PA, 1); + while (start < end) { + unsigned long blk_end = start + min(end - start, 4096UL); + + while (start < blk_end) { + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); + writel(start, base + L2X0_CLEAN_INV_LINE_PA); + start += CACHE_LINE_SIZE; + } + + if (blk_end < end) { + spin_unlock_irqrestore(&l2x0_lock, flags); + spin_lock_irqsave(&l2x0_lock, flags); + } + } + cache_wait(base + L2X0_CLEAN_INV_LINE_PA, 1); cache_sync(); + spin_unlock_irqrestore(&l2x0_lock, flags); } void __init l2x0_init(void __iomem *base, __u32 aux_val, __u32 aux_mask) diff --git a/arch/arm/mm/cache-v3.S b/arch/arm/mm/cache-v3.S index 8a4abebc478a..2a482731ea36 100644 --- a/arch/arm/mm/cache-v3.S +++ b/arch/arm/mm/cache-v3.S @@ -72,14 +72,15 @@ ENTRY(v3_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *page, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v3_flush_kern_dcache_page) +ENTRY(v3_flush_kern_dcache_area) /* FALLTHROUGH */ /* @@ -129,7 +130,7 @@ ENTRY(v3_cache_fns) .long v3_flush_user_cache_range .long v3_coherent_kern_range .long v3_coherent_user_range - .long v3_flush_kern_dcache_page + .long v3_flush_kern_dcache_area .long v3_dma_inv_range .long v3_dma_clean_range .long v3_dma_flush_range diff --git a/arch/arm/mm/cache-v4.S b/arch/arm/mm/cache-v4.S index 3668611cb400..5c7da3e372e9 100644 --- a/arch/arm/mm/cache-v4.S +++ b/arch/arm/mm/cache-v4.S @@ -82,14 +82,15 @@ ENTRY(v4_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4_flush_kern_dcache_page) +ENTRY(v4_flush_kern_dcache_area) /* FALLTHROUGH */ /* @@ -141,7 +142,7 @@ ENTRY(v4_cache_fns) .long v4_flush_user_cache_range .long v4_coherent_kern_range .long v4_coherent_user_range - .long v4_flush_kern_dcache_page + .long v4_flush_kern_dcache_area .long v4_dma_inv_range .long v4_dma_clean_range .long v4_dma_flush_range diff --git a/arch/arm/mm/cache-v4wb.S b/arch/arm/mm/cache-v4wb.S index 2ebc1b3bf856..3dbedf1ec0e7 100644 --- a/arch/arm/mm/cache-v4wb.S +++ b/arch/arm/mm/cache-v4wb.S @@ -114,15 +114,16 @@ ENTRY(v4wb_flush_user_cache_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4wb_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(v4wb_flush_kern_dcache_area) + add r1, r0, r1 /* fall through */ /* @@ -224,7 +225,7 @@ ENTRY(v4wb_cache_fns) .long v4wb_flush_user_cache_range .long v4wb_coherent_kern_range .long v4wb_coherent_user_range - .long v4wb_flush_kern_dcache_page + .long v4wb_flush_kern_dcache_area .long v4wb_dma_inv_range .long v4wb_dma_clean_range .long v4wb_dma_flush_range diff --git a/arch/arm/mm/cache-v4wt.S b/arch/arm/mm/cache-v4wt.S index c54fa2cc40e6..b3b7410270b4 100644 --- a/arch/arm/mm/cache-v4wt.S +++ b/arch/arm/mm/cache-v4wt.S @@ -117,17 +117,18 @@ ENTRY(v4wt_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(v4wt_flush_kern_dcache_page) +ENTRY(v4wt_flush_kern_dcache_area) mov r2, #0 mcr p15, 0, r2, c7, c5, 0 @ invalidate I cache - add r1, r0, #PAGE_SZ + add r1, r0, r1 /* fallthrough */ /* @@ -180,7 +181,7 @@ ENTRY(v4wt_cache_fns) .long v4wt_flush_user_cache_range .long v4wt_coherent_kern_range .long v4wt_coherent_user_range - .long v4wt_flush_kern_dcache_page + .long v4wt_flush_kern_dcache_area .long v4wt_dma_inv_range .long v4wt_dma_clean_range .long v4wt_dma_flush_range diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S index 295e25dd6381..4ba0a24ce6f5 100644 --- a/arch/arm/mm/cache-v6.S +++ b/arch/arm/mm/cache-v6.S @@ -159,15 +159,16 @@ ENDPROC(v6_coherent_user_range) ENDPROC(v6_coherent_kern_range) /* - * v6_flush_kern_dcache_page(kaddr) + * v6_flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - region size */ -ENTRY(v6_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(v6_flush_kern_dcache_area) + add r1, r0, r1 1: #ifdef HARVARD_CACHE mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line @@ -271,7 +272,7 @@ ENTRY(v6_cache_fns) .long v6_flush_user_cache_range .long v6_coherent_kern_range .long v6_coherent_user_range - .long v6_flush_kern_dcache_page + .long v6_flush_kern_dcache_area .long v6_dma_inv_range .long v6_dma_clean_range .long v6_dma_flush_range diff --git a/arch/arm/mm/cache-v7.S b/arch/arm/mm/cache-v7.S index e1bd9759617f..9073db849fb4 100644 --- a/arch/arm/mm/cache-v7.S +++ b/arch/arm/mm/cache-v7.S @@ -186,16 +186,17 @@ ENDPROC(v7_coherent_kern_range) ENDPROC(v7_coherent_user_range) /* - * v7_flush_kern_dcache_page(kaddr) + * v7_flush_kern_dcache_area(void *addr, size_t size) * * Ensure that the data held in the page kaddr is written back * to the page in question. * - * - kaddr - kernel address (guaranteed to be page aligned) + * - addr - kernel address + * - size - region size */ -ENTRY(v7_flush_kern_dcache_page) +ENTRY(v7_flush_kern_dcache_area) dcache_line_size r2, r3 - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean & invalidate D line / unified line add r0, r0, r2 @@ -203,7 +204,7 @@ ENTRY(v7_flush_kern_dcache_page) blo 1b dsb mov pc, lr -ENDPROC(v7_flush_kern_dcache_page) +ENDPROC(v7_flush_kern_dcache_area) /* * v7_dma_inv_range(start,end) @@ -279,7 +280,7 @@ ENTRY(v7_cache_fns) .long v7_flush_user_cache_range .long v7_coherent_kern_range .long v7_coherent_user_range - .long v7_flush_kern_dcache_page + .long v7_flush_kern_dcache_area .long v7_dma_inv_range .long v7_dma_clean_range .long v7_dma_flush_range diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c index 841f355319bf..0fa1319273de 100644 --- a/arch/arm/mm/copypage-v6.c +++ b/arch/arm/mm/copypage-v6.c @@ -48,7 +48,7 @@ static void v6_copy_user_highpage_nonaliasing(struct page *to, */ if (page_address(to) != NULL) #endif - __cpuc_flush_dcache_page(kto); + __cpuc_flush_dcache_area(kto, PAGE_SIZE); kunmap_atomic(kto, KM_USER1); kunmap_atomic(kfrom, KM_USER0); } diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c index 729602291958..56ee15321b00 100644 --- a/arch/arm/mm/fault-armv.c +++ b/arch/arm/mm/fault-armv.c @@ -207,9 +207,8 @@ void __init check_writebuffer_bugs(void) page = alloc_page(GFP_KERNEL); if (page) { unsigned long *p1, *p2; - pgprot_t prot = __pgprot(L_PTE_PRESENT|L_PTE_YOUNG| - L_PTE_DIRTY|L_PTE_WRITE| - L_PTE_MT_BUFFERABLE); + pgprot_t prot = __pgprot_modify(PAGE_KERNEL, + L_PTE_MT_MASK, L_PTE_MT_BUFFERABLE); p1 = vmap(&page, 1, VM_IOREMAP, prot); p2 = vmap(&page, 1, VM_IOREMAP, prot); diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c index 329594e760cd..6f3a4b7a3b82 100644 --- a/arch/arm/mm/flush.c +++ b/arch/arm/mm/flush.c @@ -131,7 +131,7 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page) */ if (addr) #endif - __cpuc_flush_dcache_page(addr); + __cpuc_flush_dcache_area(addr, PAGE_SIZE); /* * If this is a page cache page, and we have an aliasing VIPT cache, @@ -258,5 +258,5 @@ void __flush_anon_page(struct vm_area_struct *vma, struct page *page, unsigned l * in this mapping of the page. FIXME: this is overkill * since we actually ask for a write-back and invalidate. */ - __cpuc_flush_dcache_page(page_address(page)); + __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } diff --git a/arch/arm/mm/highmem.c b/arch/arm/mm/highmem.c index 30f82fb5918c..2be1ec7c1b41 100644 --- a/arch/arm/mm/highmem.c +++ b/arch/arm/mm/highmem.c @@ -79,7 +79,7 @@ void kunmap_atomic(void *kvaddr, enum km_type type) unsigned int idx = type + KM_TYPE_NR * smp_processor_id(); if (kvaddr >= (void *)FIXADDR_START) { - __cpuc_flush_dcache_page((void *)vaddr); + __cpuc_flush_dcache_area((void *)vaddr, PAGE_SIZE); #ifdef CONFIG_DEBUG_HIGHMEM BUG_ON(vaddr != __fix_to_virt(FIX_KMAP_BEGIN + idx)); set_pte_ext(TOP_PTE(vaddr), __pte(0), 0); diff --git a/arch/arm/mm/mmap.c b/arch/arm/mm/mmap.c index 2b7996401b0f..f5abc51c5a07 100644 --- a/arch/arm/mm/mmap.c +++ b/arch/arm/mm/mmap.c @@ -54,7 +54,8 @@ arch_get_unmapped_area(struct file *filp, unsigned long addr, * We enforce the MAP_FIXED case. */ if (flags & MAP_FIXED) { - if (aliasing && flags & MAP_SHARED && addr & (SHMLBA - 1)) + if (aliasing && flags & MAP_SHARED && + (addr - (pgoff << PAGE_SHIFT)) & (SHMLBA - 1)) return -EINVAL; return addr; } diff --git a/arch/arm/mm/mmu.c b/arch/arm/mm/mmu.c index 8c7fbd19a4b3..1708da82da96 100644 --- a/arch/arm/mm/mmu.c +++ b/arch/arm/mm/mmu.c @@ -453,8 +453,7 @@ static void __init build_mem_type_table(void) pgprot_user = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | user_pgprot); pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | - L_PTE_DIRTY | L_PTE_WRITE | - L_PTE_EXEC | kern_pgprot); + L_PTE_DIRTY | L_PTE_WRITE | kern_pgprot); mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask; mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask; diff --git a/arch/arm/mm/nommu.c b/arch/arm/mm/nommu.c index 900811cc9130..374a8311bc84 100644 --- a/arch/arm/mm/nommu.c +++ b/arch/arm/mm/nommu.c @@ -61,7 +61,7 @@ void setup_mm_for_reboot(char mode) void flush_dcache_page(struct page *page) { - __cpuc_flush_dcache_page(page_address(page)); + __cpuc_flush_dcache_area(page_address(page), PAGE_SIZE); } EXPORT_SYMBOL(flush_dcache_page); diff --git a/arch/arm/mm/proc-arm1020.S b/arch/arm/mm/proc-arm1020.S index d9fb4b98c49f..8012e24282b2 100644 --- a/arch/arm/mm/proc-arm1020.S +++ b/arch/arm/mm/proc-arm1020.S @@ -231,17 +231,18 @@ ENTRY(arm1020_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1020_flush_kern_dcache_page) +ENTRY(arm1020_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry mcr p15, 0, ip, c7, c10, 4 @ drain WB add r0, r0, #CACHE_DLINESIZE @@ -335,7 +336,7 @@ ENTRY(arm1020_cache_fns) .long arm1020_flush_user_cache_range .long arm1020_coherent_kern_range .long arm1020_coherent_user_range - .long arm1020_flush_kern_dcache_page + .long arm1020_flush_kern_dcache_area .long arm1020_dma_inv_range .long arm1020_dma_clean_range .long arm1020_dma_flush_range diff --git a/arch/arm/mm/proc-arm1020e.S b/arch/arm/mm/proc-arm1020e.S index 7453b75dcea5..41fe25d234f5 100644 --- a/arch/arm/mm/proc-arm1020e.S +++ b/arch/arm/mm/proc-arm1020e.S @@ -225,17 +225,18 @@ ENTRY(arm1020e_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1020e_flush_kern_dcache_page) +ENTRY(arm1020e_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -321,7 +322,7 @@ ENTRY(arm1020e_cache_fns) .long arm1020e_flush_user_cache_range .long arm1020e_coherent_kern_range .long arm1020e_coherent_user_range - .long arm1020e_flush_kern_dcache_page + .long arm1020e_flush_kern_dcache_area .long arm1020e_dma_inv_range .long arm1020e_dma_clean_range .long arm1020e_dma_flush_range diff --git a/arch/arm/mm/proc-arm1022.S b/arch/arm/mm/proc-arm1022.S index 8eb72d75a8b6..20a5b1b31a70 100644 --- a/arch/arm/mm/proc-arm1022.S +++ b/arch/arm/mm/proc-arm1022.S @@ -214,17 +214,18 @@ ENTRY(arm1022_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1022_flush_kern_dcache_page) +ENTRY(arm1022_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -310,7 +311,7 @@ ENTRY(arm1022_cache_fns) .long arm1022_flush_user_cache_range .long arm1022_coherent_kern_range .long arm1022_coherent_user_range - .long arm1022_flush_kern_dcache_page + .long arm1022_flush_kern_dcache_area .long arm1022_dma_inv_range .long arm1022_dma_clean_range .long arm1022_dma_flush_range diff --git a/arch/arm/mm/proc-arm1026.S b/arch/arm/mm/proc-arm1026.S index 3b59f0d67139..96aedb10fcc4 100644 --- a/arch/arm/mm/proc-arm1026.S +++ b/arch/arm/mm/proc-arm1026.S @@ -208,17 +208,18 @@ ENTRY(arm1026_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - page - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm1026_flush_kern_dcache_page) +ENTRY(arm1026_flush_kern_dcache_area) mov ip, #0 #ifndef CONFIG_CPU_DCACHE_DISABLE - add r1, r0, #PAGE_SZ + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -304,7 +305,7 @@ ENTRY(arm1026_cache_fns) .long arm1026_flush_user_cache_range .long arm1026_coherent_kern_range .long arm1026_coherent_user_range - .long arm1026_flush_kern_dcache_page + .long arm1026_flush_kern_dcache_area .long arm1026_dma_inv_range .long arm1026_dma_clean_range .long arm1026_dma_flush_range diff --git a/arch/arm/mm/proc-arm920.S b/arch/arm/mm/proc-arm920.S index 2b7c197cc58d..471669e2d7cb 100644 --- a/arch/arm/mm/proc-arm920.S +++ b/arch/arm/mm/proc-arm920.S @@ -207,15 +207,16 @@ ENTRY(arm920_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm920_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm920_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -293,7 +294,7 @@ ENTRY(arm920_cache_fns) .long arm920_flush_user_cache_range .long arm920_coherent_kern_range .long arm920_coherent_user_range - .long arm920_flush_kern_dcache_page + .long arm920_flush_kern_dcache_area .long arm920_dma_inv_range .long arm920_dma_clean_range .long arm920_dma_flush_range diff --git a/arch/arm/mm/proc-arm922.S b/arch/arm/mm/proc-arm922.S index 06a1aa4e3398..ee111b00fa41 100644 --- a/arch/arm/mm/proc-arm922.S +++ b/arch/arm/mm/proc-arm922.S @@ -209,15 +209,16 @@ ENTRY(arm922_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm922_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm922_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -295,7 +296,7 @@ ENTRY(arm922_cache_fns) .long arm922_flush_user_cache_range .long arm922_coherent_kern_range .long arm922_coherent_user_range - .long arm922_flush_kern_dcache_page + .long arm922_flush_kern_dcache_area .long arm922_dma_inv_range .long arm922_dma_clean_range .long arm922_dma_flush_range diff --git a/arch/arm/mm/proc-arm925.S b/arch/arm/mm/proc-arm925.S index cb53435a85ae..8deb5bde58e4 100644 --- a/arch/arm/mm/proc-arm925.S +++ b/arch/arm/mm/proc-arm925.S @@ -251,15 +251,16 @@ ENTRY(arm925_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm925_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm925_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -346,7 +347,7 @@ ENTRY(arm925_cache_fns) .long arm925_flush_user_cache_range .long arm925_coherent_kern_range .long arm925_coherent_user_range - .long arm925_flush_kern_dcache_page + .long arm925_flush_kern_dcache_area .long arm925_dma_inv_range .long arm925_dma_clean_range .long arm925_dma_flush_range diff --git a/arch/arm/mm/proc-arm926.S b/arch/arm/mm/proc-arm926.S index 1c4848704bb3..64db6e275a44 100644 --- a/arch/arm/mm/proc-arm926.S +++ b/arch/arm/mm/proc-arm926.S @@ -214,15 +214,16 @@ ENTRY(arm926_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm926_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm926_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -309,7 +310,7 @@ ENTRY(arm926_cache_fns) .long arm926_flush_user_cache_range .long arm926_coherent_kern_range .long arm926_coherent_user_range - .long arm926_flush_kern_dcache_page + .long arm926_flush_kern_dcache_area .long arm926_dma_inv_range .long arm926_dma_clean_range .long arm926_dma_flush_range diff --git a/arch/arm/mm/proc-arm940.S b/arch/arm/mm/proc-arm940.S index 5b0f8464c8f2..8196b9f401fb 100644 --- a/arch/arm/mm/proc-arm940.S +++ b/arch/arm/mm/proc-arm940.S @@ -141,14 +141,15 @@ ENTRY(arm940_coherent_user_range) /* FALLTHROUGH */ /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(arm940_flush_kern_dcache_page) +ENTRY(arm940_flush_kern_dcache_area) mov ip, #0 mov r1, #(CACHE_DSEGMENTS - 1) << 4 @ 4 segments 1: orr r3, r1, #(CACHE_DENTRIES - 1) << 26 @ 64 entries @@ -238,7 +239,7 @@ ENTRY(arm940_cache_fns) .long arm940_flush_user_cache_range .long arm940_coherent_kern_range .long arm940_coherent_user_range - .long arm940_flush_kern_dcache_page + .long arm940_flush_kern_dcache_area .long arm940_dma_inv_range .long arm940_dma_clean_range .long arm940_dma_flush_range diff --git a/arch/arm/mm/proc-arm946.S b/arch/arm/mm/proc-arm946.S index 40c0449a139b..9a951239c86c 100644 --- a/arch/arm/mm/proc-arm946.S +++ b/arch/arm/mm/proc-arm946.S @@ -183,16 +183,17 @@ ENTRY(arm946_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size * (same as arm926) */ -ENTRY(arm946_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(arm946_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -280,7 +281,7 @@ ENTRY(arm946_cache_fns) .long arm946_flush_user_cache_range .long arm946_coherent_kern_range .long arm946_coherent_user_range - .long arm946_flush_kern_dcache_page + .long arm946_flush_kern_dcache_area .long arm946_dma_inv_range .long arm946_dma_clean_range .long arm946_dma_flush_range diff --git a/arch/arm/mm/proc-feroceon.S b/arch/arm/mm/proc-feroceon.S index d0d7795200fc..dbc39383e66a 100644 --- a/arch/arm/mm/proc-feroceon.S +++ b/arch/arm/mm/proc-feroceon.S @@ -226,16 +226,17 @@ ENTRY(feroceon_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ .align 5 -ENTRY(feroceon_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(feroceon_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -246,7 +247,7 @@ ENTRY(feroceon_flush_kern_dcache_page) mov pc, lr .align 5 -ENTRY(feroceon_range_flush_kern_dcache_page) +ENTRY(feroceon_range_flush_kern_dcache_area) mrs r2, cpsr add r1, r0, #PAGE_SZ - CACHE_DLINESIZE @ top addr is inclusive orr r3, r2, #PSR_I_BIT @@ -372,7 +373,7 @@ ENTRY(feroceon_cache_fns) .long feroceon_flush_user_cache_range .long feroceon_coherent_kern_range .long feroceon_coherent_user_range - .long feroceon_flush_kern_dcache_page + .long feroceon_flush_kern_dcache_area .long feroceon_dma_inv_range .long feroceon_dma_clean_range .long feroceon_dma_flush_range @@ -383,7 +384,7 @@ ENTRY(feroceon_range_cache_fns) .long feroceon_flush_user_cache_range .long feroceon_coherent_kern_range .long feroceon_coherent_user_range - .long feroceon_range_flush_kern_dcache_page + .long feroceon_range_flush_kern_dcache_area .long feroceon_range_dma_inv_range .long feroceon_range_dma_clean_range .long feroceon_range_dma_flush_range diff --git a/arch/arm/mm/proc-mohawk.S b/arch/arm/mm/proc-mohawk.S index 52b5fd74fbb3..9674d36cc97d 100644 --- a/arch/arm/mm/proc-mohawk.S +++ b/arch/arm/mm/proc-mohawk.S @@ -186,15 +186,16 @@ ENTRY(mohawk_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(mohawk_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(mohawk_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean+invalidate D entry add r0, r0, #CACHE_DLINESIZE cmp r0, r1 @@ -273,7 +274,7 @@ ENTRY(mohawk_cache_fns) .long mohawk_flush_user_cache_range .long mohawk_coherent_kern_range .long mohawk_coherent_user_range - .long mohawk_flush_kern_dcache_page + .long mohawk_flush_kern_dcache_area .long mohawk_dma_inv_range .long mohawk_dma_clean_range .long mohawk_dma_flush_range diff --git a/arch/arm/mm/proc-syms.c b/arch/arm/mm/proc-syms.c index ac5c80062b70..3e6210b4d6d4 100644 --- a/arch/arm/mm/proc-syms.c +++ b/arch/arm/mm/proc-syms.c @@ -27,8 +27,7 @@ EXPORT_SYMBOL(__cpuc_flush_kern_all); EXPORT_SYMBOL(__cpuc_flush_user_all); EXPORT_SYMBOL(__cpuc_flush_user_range); EXPORT_SYMBOL(__cpuc_coherent_kern_range); -EXPORT_SYMBOL(__cpuc_flush_dcache_page); -EXPORT_SYMBOL(dmac_inv_range); /* because of flush_ioremap_region() */ +EXPORT_SYMBOL(__cpuc_flush_dcache_area); #else EXPORT_SYMBOL(cpu_cache); #endif diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S index 5485c821101c..395cc90c6613 100644 --- a/arch/arm/mm/proc-v6.S +++ b/arch/arm/mm/proc-v6.S @@ -254,10 +254,9 @@ __pj4_v6_proc_info: .long 0x560f5810 .long 0xff0ffff0 .long PMD_TYPE_SECT | \ - PMD_SECT_BUFFERABLE | \ - PMD_SECT_CACHEABLE | \ PMD_SECT_AP_WRITE | \ - PMD_SECT_AP_READ + PMD_SECT_AP_READ | \ + PMD_FLAGS .long PMD_TYPE_SECT | \ PMD_SECT_XN | \ PMD_SECT_AP_WRITE | \ diff --git a/arch/arm/mm/proc-xsc3.S b/arch/arm/mm/proc-xsc3.S index fab134e29826..96456f548798 100644 --- a/arch/arm/mm/proc-xsc3.S +++ b/arch/arm/mm/proc-xsc3.S @@ -226,15 +226,16 @@ ENTRY(xsc3_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache. * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(xsc3_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(xsc3_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c14, 1 @ clean/invalidate L1 D line add r0, r0, #CACHELINESIZE cmp r0, r1 @@ -309,7 +310,7 @@ ENTRY(xsc3_cache_fns) .long xsc3_flush_user_cache_range .long xsc3_coherent_kern_range .long xsc3_coherent_user_range - .long xsc3_flush_kern_dcache_page + .long xsc3_flush_kern_dcache_area .long xsc3_dma_inv_range .long xsc3_dma_clean_range .long xsc3_dma_flush_range diff --git a/arch/arm/mm/proc-xscale.S b/arch/arm/mm/proc-xscale.S index f056c283682d..93df47265f2d 100644 --- a/arch/arm/mm/proc-xscale.S +++ b/arch/arm/mm/proc-xscale.S @@ -284,15 +284,16 @@ ENTRY(xscale_coherent_user_range) mov pc, lr /* - * flush_kern_dcache_page(void *page) + * flush_kern_dcache_area(void *addr, size_t size) * * Ensure no D cache aliasing occurs, either with itself or * the I cache * - * - addr - page aligned address + * - addr - kernel address + * - size - region size */ -ENTRY(xscale_flush_kern_dcache_page) - add r1, r0, #PAGE_SZ +ENTRY(xscale_flush_kern_dcache_area) + add r1, r0, r1 1: mcr p15, 0, r0, c7, c10, 1 @ clean D entry mcr p15, 0, r0, c7, c6, 1 @ invalidate D entry add r0, r0, #CACHELINESIZE @@ -368,7 +369,7 @@ ENTRY(xscale_cache_fns) .long xscale_flush_user_cache_range .long xscale_coherent_kern_range .long xscale_coherent_user_range - .long xscale_flush_kern_dcache_page + .long xscale_flush_kern_dcache_area .long xscale_dma_inv_range .long xscale_dma_clean_range .long xscale_dma_flush_range @@ -392,7 +393,7 @@ ENTRY(xscale_80200_A0_A1_cache_fns) .long xscale_flush_user_cache_range .long xscale_coherent_kern_range .long xscale_coherent_user_range - .long xscale_flush_kern_dcache_page + .long xscale_flush_kern_dcache_area .long xscale_dma_flush_range .long xscale_dma_clean_range .long xscale_dma_flush_range |