#ifndef _ASM_S390_PGTABLE_H
#define _ASM_S390_PGTABLE_H
-#include <asm-generic/4level-fixup.h>
-
/*
* The Linux memory management assumes a three-level page table setup. For
* s390 31 bit we "fold" the mid level into the top-level page table, so
*/
#ifndef __ASSEMBLY__
#include <linux/mm_types.h>
+#include <asm/bitops.h>
#include <asm/bug.h>
#include <asm/processor.h>
-struct vm_area_struct; /* forward declaration (include/linux/mm.h) */
-struct mm_struct;
-
extern pgd_t swapper_pg_dir[] __attribute__ ((aligned (4096)));
extern void paging_init(void);
extern void vmem_map_init(void);
* PGDIR_SHIFT determines what a third-level page table entry can map
*/
#ifndef __s390x__
-# define PMD_SHIFT 22
-# define PGDIR_SHIFT 22
+# define PMD_SHIFT 20
+# define PUD_SHIFT 20
+# define PGDIR_SHIFT 20
#else /* __s390x__ */
-# define PMD_SHIFT 21
-# define PGDIR_SHIFT 31
+# define PMD_SHIFT 20
+# define PUD_SHIFT 31
+# define PGDIR_SHIFT 42
#endif /* __s390x__ */
#define PMD_SIZE (1UL << PMD_SHIFT)
#define PMD_MASK (~(PMD_SIZE-1))
-#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
-#define PGDIR_MASK (~(PGDIR_SIZE-1))
+#define PUD_SIZE (1UL << PUD_SHIFT)
+#define PUD_MASK (~(PUD_SIZE-1))
+#define PGDIR_SIZE (1UL << PGDIR_SHIFT)
+#define PGDIR_MASK (~(PGDIR_SIZE-1))
/*
* entries per page directory level: the S390 is two-level, so
* for S390 segment-table entries are combined to one PGD
* that leads to 1024 pte per pgd
*/
+#define PTRS_PER_PTE 256
#ifndef __s390x__
-# define PTRS_PER_PTE 1024
-# define PTRS_PER_PMD 1
-# define PTRS_PER_PGD 512
+#define PTRS_PER_PMD 1
+#define PTRS_PER_PUD 1
#else /* __s390x__ */
-# define PTRS_PER_PTE 512
-# define PTRS_PER_PMD 1024
-# define PTRS_PER_PGD 2048
+#define PTRS_PER_PMD 2048
+#define PTRS_PER_PUD 2048
#endif /* __s390x__ */
+#define PTRS_PER_PGD 2048
#define FIRST_USER_ADDRESS 0
printk("%s:%d: bad pte %p.\n", __FILE__, __LINE__, (void *) pte_val(e))
#define pmd_ERROR(e) \
printk("%s:%d: bad pmd %p.\n", __FILE__, __LINE__, (void *) pmd_val(e))
+#define pud_ERROR(e) \
+ printk("%s:%d: bad pud %p.\n", __FILE__, __LINE__, (void *) pud_val(e))
#define pgd_ERROR(e) \
printk("%s:%d: bad pgd %p.\n", __FILE__, __LINE__, (void *) pgd_val(e))
#ifndef __ASSEMBLY__
/*
- * Just any arbitrary offset to the start of the vmalloc VM area: the
- * current 8MB value just means that there will be a 8MB "hole" after the
- * physical memory until the kernel virtual memory starts. That means that
- * any out-of-bounds memory accesses will hopefully be caught.
- * The vmalloc() routines leaves a hole of 4kB between each vmalloced
- * area for the same reason. ;)
- */
-extern unsigned long vmalloc_end;
-#define VMALLOC_OFFSET (8*1024*1024)
-#define VMALLOC_START (((unsigned long) high_memory + VMALLOC_OFFSET) \
- & ~(VMALLOC_OFFSET-1))
-#define VMALLOC_END vmalloc_end
-
-/*
- * We need some free virtual space to be able to do vmalloc.
- * VMALLOC_MIN_SIZE defines the minimum size of the vmalloc
- * area. On a machine with 2GB memory we make sure that we
- * have at least 128MB free space for vmalloc. On a machine
- * with 4TB we make sure we have at least 128GB.
+ * The vmalloc area will always be on the topmost area of the kernel
+ * mapping. We reserve 96MB (31bit) / 1GB (64bit) for vmalloc,
+ * which should be enough for any sane case.
+ * By putting vmalloc at the top, we maximise the gap between physical
+ * memory and vmalloc to catch misplaced memory accesses. As a side
+ * effect, this also makes sure that 64 bit module code cannot be used
+ * as system call address.
*/
#ifndef __s390x__
-#define VMALLOC_MIN_SIZE 0x8000000UL
-#define VMALLOC_END_INIT 0x80000000UL
+#define VMALLOC_START 0x78000000UL
+#define VMALLOC_END 0x7e000000UL
+#define VMEM_MAP_END 0x80000000UL
#else /* __s390x__ */
-#define VMALLOC_MIN_SIZE 0x2000000000UL
-#define VMALLOC_END_INIT 0x40000000000UL
+#define VMALLOC_START 0x3e000000000UL
+#define VMALLOC_END 0x3e040000000UL
+#define VMEM_MAP_END 0x40000000000UL
#endif /* __s390x__ */
+/*
+ * VMEM_MAX_PHYS is the highest physical address that can be added to the 1:1
+ * mapping. This needs to be calculated at compile time since the size of the
+ * VMEM_MAP is static but the size of struct page can change.
+ */
+#define VMEM_MAX_PAGES ((VMEM_MAP_END - VMALLOC_END) / sizeof(struct page))
+#define VMEM_MAX_PFN min(VMALLOC_START >> PAGE_SHIFT, VMEM_MAX_PAGES)
+#define VMEM_MAX_PHYS ((VMEM_MAX_PFN << PAGE_SHIFT) & ~((16 << 20) - 1))
+#define VMEM_MAP ((struct page *) VMALLOC_END)
+
/*
* A 31 bit pagetable entry of S390 has following format:
* | PFRA | | OS |
* I Segment-Invalid Bit: Segment is not available for address-translation
* TT Type 01
* TF
- * TL Table lenght
+ * TL Table length
*
* The 64 bit regiontable origin of S390 has following format:
* | region table origon | DTTL
/* Hardware bits in the page table entry */
#define _PAGE_RO 0x200 /* HW read-only bit */
#define _PAGE_INVALID 0x400 /* HW invalid bit */
+
+/* Software bits in the page table entry */
#define _PAGE_SWT 0x001 /* SW pte type bit t */
#define _PAGE_SWX 0x002 /* SW pte type bit x */
+#define _PAGE_SPECIAL 0x004 /* SW associated with special page */
+#define __HAVE_ARCH_PTE_SPECIAL
/* Six different types of pages. */
#define _PAGE_TYPE_EMPTY 0x400
#define _PAGE_TYPE_EX_RO 0x202
#define _PAGE_TYPE_EX_RW 0x002
+/*
+ * Only four types for huge pages, using the invalid bit and protection bit
+ * of a segment table entry.
+ */
+#define _HPAGE_TYPE_EMPTY 0x020 /* _SEGMENT_ENTRY_INV */
+#define _HPAGE_TYPE_NONE 0x220
+#define _HPAGE_TYPE_RO 0x200 /* _SEGMENT_ENTRY_RO */
+#define _HPAGE_TYPE_RW 0x000
+
/*
* PTE type bits are rather complicated. handle_pte_fault uses pte_present,
* pte_none and pte_file to find out the pte type WITHOUT holding the page
* swap pte is 1011 and 0001, 0011, 0101, 0111 are invalid.
*/
+/* Page status table bits for virtualization */
+#define RCP_PCL_BIT 55
+#define RCP_HR_BIT 54
+#define RCP_HC_BIT 53
+#define RCP_GR_BIT 50
+#define RCP_GC_BIT 49
+
#ifndef __s390x__
-/* Bits in the segment table entry */
-#define _PAGE_TABLE_LEN 0xf /* only full page-tables */
-#define _PAGE_TABLE_COM 0x10 /* common page-table */
-#define _PAGE_TABLE_INV 0x20 /* invalid page-table */
-#define _SEG_PRESENT 0x001 /* Software (overlap with PTL) */
+/* Bits in the segment table address-space-control-element */
+#define _ASCE_SPACE_SWITCH 0x80000000UL /* space switch event */
+#define _ASCE_ORIGIN_MASK 0x7ffff000UL /* segment table origin */
+#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
+#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
+#define _ASCE_TABLE_LENGTH 0x7f /* 128 x 64 entries = 8k */
-/* Bits int the storage key */
-#define _PAGE_CHANGED 0x02 /* HW changed bit */
-#define _PAGE_REFERENCED 0x04 /* HW referenced bit */
+/* Bits in the segment table entry */
+#define _SEGMENT_ENTRY_ORIGIN 0x7fffffc0UL /* page table origin */
+#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */
+#define _SEGMENT_ENTRY_COMMON 0x10 /* common segment bit */
+#define _SEGMENT_ENTRY_PTL 0x0f /* page table length */
-#define _USER_SEG_TABLE_LEN 0x7f /* user-segment-table up to 2 GB */
-#define _KERNEL_SEG_TABLE_LEN 0x7f /* kernel-segment-table up to 2 GB */
+#define _SEGMENT_ENTRY (_SEGMENT_ENTRY_PTL)
+#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
-/*
- * User and Kernel pagetables are identical
- */
-#define _PAGE_TABLE _PAGE_TABLE_LEN
-#define _KERNPG_TABLE _PAGE_TABLE_LEN
-
-/*
- * The Kernel segment-tables includes the User segment-table
- */
+#else /* __s390x__ */
-#define _SEGMENT_TABLE (_USER_SEG_TABLE_LEN|0x80000000|0x100)
-#define _KERNSEG_TABLE _KERNEL_SEG_TABLE_LEN
+/* Bits in the segment/region table address-space-control-element */
+#define _ASCE_ORIGIN ~0xfffUL/* segment table origin */
+#define _ASCE_PRIVATE_SPACE 0x100 /* private space control */
+#define _ASCE_ALT_EVENT 0x80 /* storage alteration event control */
+#define _ASCE_SPACE_SWITCH 0x40 /* space switch event */
+#define _ASCE_REAL_SPACE 0x20 /* real space control */
+#define _ASCE_TYPE_MASK 0x0c /* asce table type mask */
+#define _ASCE_TYPE_REGION1 0x0c /* region first table type */
+#define _ASCE_TYPE_REGION2 0x08 /* region second table type */
+#define _ASCE_TYPE_REGION3 0x04 /* region third table type */
+#define _ASCE_TYPE_SEGMENT 0x00 /* segment table type */
+#define _ASCE_TABLE_LENGTH 0x03 /* region table length */
+
+/* Bits in the region table entry */
+#define _REGION_ENTRY_ORIGIN ~0xfffUL/* region/segment table origin */
+#define _REGION_ENTRY_INV 0x20 /* invalid region table entry */
+#define _REGION_ENTRY_TYPE_MASK 0x0c /* region/segment table type mask */
+#define _REGION_ENTRY_TYPE_R1 0x0c /* region first table type */
+#define _REGION_ENTRY_TYPE_R2 0x08 /* region second table type */
+#define _REGION_ENTRY_TYPE_R3 0x04 /* region third table type */
+#define _REGION_ENTRY_LENGTH 0x03 /* region third length */
+
+#define _REGION1_ENTRY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_LENGTH)
+#define _REGION1_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R1 | _REGION_ENTRY_INV)
+#define _REGION2_ENTRY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_LENGTH)
+#define _REGION2_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R2 | _REGION_ENTRY_INV)
+#define _REGION3_ENTRY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_LENGTH)
+#define _REGION3_ENTRY_EMPTY (_REGION_ENTRY_TYPE_R3 | _REGION_ENTRY_INV)
-#define USER_STD_MASK 0x00000080UL
+/* Bits in the segment table entry */
+#define _SEGMENT_ENTRY_ORIGIN ~0x7ffUL/* segment table origin */
+#define _SEGMENT_ENTRY_RO 0x200 /* page protection bit */
+#define _SEGMENT_ENTRY_INV 0x20 /* invalid segment table entry */
-#else /* __s390x__ */
+#define _SEGMENT_ENTRY (0)
+#define _SEGMENT_ENTRY_EMPTY (_SEGMENT_ENTRY_INV)
-/* Bits in the segment table entry */
-#define _PMD_ENTRY_INV 0x20 /* invalid segment table entry */
-#define _PMD_ENTRY 0x00
+#define _SEGMENT_ENTRY_LARGE 0x400 /* STE-format control, large page */
+#define _SEGMENT_ENTRY_CO 0x100 /* change-recording override */
-/* Bits in the region third table entry */
-#define _PGD_ENTRY_INV 0x20 /* invalid region table entry */
-#define _PGD_ENTRY 0x07
+#endif /* __s390x__ */
/*
- * User and kernel page directory
+ * A user page table pointer has the space-switch-event bit, the
+ * private-space-control bit and the storage-alteration-event-control
+ * bit set. A kernel page table pointer doesn't need them.
*/
-#define _REGION_THIRD 0x4
-#define _REGION_THIRD_LEN 0x3
-#define _REGION_TABLE (_REGION_THIRD|_REGION_THIRD_LEN|0x40|0x100)
-#define _KERN_REGION_TABLE (_REGION_THIRD|_REGION_THIRD_LEN)
+#define _ASCE_USER_BITS (_ASCE_SPACE_SWITCH | _ASCE_PRIVATE_SPACE | \
+ _ASCE_ALT_EVENT)
-#define USER_STD_MASK 0x0000000000000080UL
-
-/* Bits in the storage key */
+/* Bits int the storage key */
#define _PAGE_CHANGED 0x02 /* HW changed bit */
#define _PAGE_REFERENCED 0x04 /* HW referenced bit */
-#endif /* __s390x__ */
-
/*
* Page protection definitions.
*/
#define __S111 PAGE_EX_RW
#ifndef __s390x__
-# define PMD_SHADOW_SHIFT 1
-# define PGD_SHADOW_SHIFT 1
+# define PxD_SHADOW_SHIFT 1
#else /* __s390x__ */
-# define PMD_SHADOW_SHIFT 2
-# define PGD_SHADOW_SHIFT 2
+# define PxD_SHADOW_SHIFT 2
#endif /* __s390x__ */
-static inline struct page *get_shadow_page(struct page *page)
-{
- if (s390_noexec && !list_empty(&page->lru))
- return virt_to_page(page->lru.next);
- return NULL;
-}
-
-static inline pte_t *get_shadow_pte(pte_t *ptep)
-{
- unsigned long pteptr = (unsigned long) (ptep);
-
- if (s390_noexec) {
- unsigned long offset = pteptr & (PAGE_SIZE - 1);
- void *addr = (void *) (pteptr ^ offset);
- struct page *page = virt_to_page(addr);
- if (!list_empty(&page->lru))
- return (pte_t *) ((unsigned long) page->lru.next |
- offset);
- }
- return NULL;
-}
-
-static inline pmd_t *get_shadow_pmd(pmd_t *pmdp)
-{
- unsigned long pmdptr = (unsigned long) (pmdp);
-
- if (s390_noexec) {
- unsigned long offset = pmdptr &
- ((PAGE_SIZE << PMD_SHADOW_SHIFT) - 1);
- void *addr = (void *) (pmdptr ^ offset);
- struct page *page = virt_to_page(addr);
- if (!list_empty(&page->lru))
- return (pmd_t *) ((unsigned long) page->lru.next |
- offset);
- }
- return NULL;
-}
-
-static inline pgd_t *get_shadow_pgd(pgd_t *pgdp)
+static inline void *get_shadow_table(void *table)
{
- unsigned long pgdptr = (unsigned long) (pgdp);
+ unsigned long addr, offset;
+ struct page *page;
- if (s390_noexec) {
- unsigned long offset = pgdptr &
- ((PAGE_SIZE << PGD_SHADOW_SHIFT) - 1);
- void *addr = (void *) (pgdptr ^ offset);
- struct page *page = virt_to_page(addr);
- if (!list_empty(&page->lru))
- return (pgd_t *) ((unsigned long) page->lru.next |
- offset);
- }
- return NULL;
+ addr = (unsigned long) table;
+ offset = addr & ((PAGE_SIZE << PxD_SHADOW_SHIFT) - 1);
+ page = virt_to_page((void *)(addr ^ offset));
+ return (void *)(addr_t)(page->index ? (page->index | offset) : 0UL);
}
/*
* within a page table are directly modified. Thus, the following
* hook is made available.
*/
-static inline void set_pte(pte_t *pteptr, pte_t pteval)
+static inline void set_pte_at(struct mm_struct *mm, unsigned long addr,
+ pte_t *ptep, pte_t entry)
{
- pte_t *shadow_pte = get_shadow_pte(pteptr);
-
- *pteptr = pteval;
- if (shadow_pte) {
- if (!(pte_val(pteval) & _PAGE_INVALID) &&
- (pte_val(pteval) & _PAGE_SWX))
- pte_val(*shadow_pte) = pte_val(pteval) | _PAGE_RO;
+ *ptep = entry;
+ if (mm->context.noexec) {
+ if (!(pte_val(entry) & _PAGE_INVALID) &&
+ (pte_val(entry) & _PAGE_SWX))
+ pte_val(entry) |= _PAGE_RO;
else
- pte_val(*shadow_pte) = _PAGE_TYPE_EMPTY;
+ pte_val(entry) = _PAGE_TYPE_EMPTY;
+ ptep[PTRS_PER_PTE] = entry;
}
}
-#define set_pte_at(mm,addr,ptep,pteval) set_pte(ptep,pteval)
/*
* pgd/pmd/pte query functions
static inline int pgd_none(pgd_t pgd) { return 0; }
static inline int pgd_bad(pgd_t pgd) { return 0; }
-static inline int pmd_present(pmd_t pmd) { return pmd_val(pmd) & _SEG_PRESENT; }
-static inline int pmd_none(pmd_t pmd) { return pmd_val(pmd) & _PAGE_TABLE_INV; }
-static inline int pmd_bad(pmd_t pmd)
-{
- return (pmd_val(pmd) & (~PAGE_MASK & ~_PAGE_TABLE_INV)) != _PAGE_TABLE;
-}
+static inline int pud_present(pud_t pud) { return 1; }
+static inline int pud_none(pud_t pud) { return 0; }
+static inline int pud_bad(pud_t pud) { return 0; }
#else /* __s390x__ */
static inline int pgd_present(pgd_t pgd)
{
- return (pgd_val(pgd) & ~PAGE_MASK) == _PGD_ENTRY;
+ if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
+ return 1;
+ return (pgd_val(pgd) & _REGION_ENTRY_ORIGIN) != 0UL;
}
static inline int pgd_none(pgd_t pgd)
{
- return pgd_val(pgd) & _PGD_ENTRY_INV;
+ if ((pgd_val(pgd) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R2)
+ return 0;
+ return (pgd_val(pgd) & _REGION_ENTRY_INV) != 0UL;
}
static inline int pgd_bad(pgd_t pgd)
{
- return (pgd_val(pgd) & (~PAGE_MASK & ~_PGD_ENTRY_INV)) != _PGD_ENTRY;
+ /*
+ * With dynamic page table levels the pgd can be a region table
+ * entry or a segment table entry. Check for the bit that are
+ * invalid for either table entry.
+ */
+ unsigned long mask =
+ ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
+ ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
+ return (pgd_val(pgd) & mask) != 0;
+}
+
+static inline int pud_present(pud_t pud)
+{
+ if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
+ return 1;
+ return (pud_val(pud) & _REGION_ENTRY_ORIGIN) != 0UL;
+}
+
+static inline int pud_none(pud_t pud)
+{
+ if ((pud_val(pud) & _REGION_ENTRY_TYPE_MASK) < _REGION_ENTRY_TYPE_R3)
+ return 0;
+ return (pud_val(pud) & _REGION_ENTRY_INV) != 0UL;
+}
+
+static inline int pud_bad(pud_t pud)
+{
+ /*
+ * With dynamic page table levels the pud can be a region table
+ * entry or a segment table entry. Check for the bit that are
+ * invalid for either table entry.
+ */
+ unsigned long mask =
+ ~_SEGMENT_ENTRY_ORIGIN & ~_REGION_ENTRY_INV &
+ ~_REGION_ENTRY_TYPE_MASK & ~_REGION_ENTRY_LENGTH;
+ return (pud_val(pud) & mask) != 0;
}
+#endif /* __s390x__ */
+
static inline int pmd_present(pmd_t pmd)
{
- return (pmd_val(pmd) & ~PAGE_MASK) == _PMD_ENTRY;
+ return (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN) != 0UL;
}
static inline int pmd_none(pmd_t pmd)
{
- return pmd_val(pmd) & _PMD_ENTRY_INV;
+ return (pmd_val(pmd) & _SEGMENT_ENTRY_INV) != 0UL;
}
static inline int pmd_bad(pmd_t pmd)
{
- return (pmd_val(pmd) & (~PAGE_MASK & ~_PMD_ENTRY_INV)) != _PMD_ENTRY;
+ unsigned long mask = ~_SEGMENT_ENTRY_ORIGIN & ~_SEGMENT_ENTRY_INV;
+ return (pmd_val(pmd) & mask) != _SEGMENT_ENTRY;
}
-#endif /* __s390x__ */
-
static inline int pte_none(pte_t pte)
{
return (pte_val(pte) & _PAGE_INVALID) && !(pte_val(pte) & _PAGE_SWT);
return (pte_val(pte) & mask) == _PAGE_TYPE_FILE;
}
-#define pte_same(a,b) (pte_val(a) == pte_val(b))
+static inline int pte_special(pte_t pte)
+{
+ return (pte_val(pte) & _PAGE_SPECIAL);
+}
+
+#define __HAVE_ARCH_PTE_SAME
+#define pte_same(a,b) (pte_val(a) == pte_val(b))
+
+static inline void rcp_lock(pte_t *ptep)
+{
+#ifdef CONFIG_PGSTE
+ unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE);
+ preempt_disable();
+ while (test_and_set_bit(RCP_PCL_BIT, pgste))
+ ;
+#endif
+}
+
+static inline void rcp_unlock(pte_t *ptep)
+{
+#ifdef CONFIG_PGSTE
+ unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE);
+ clear_bit(RCP_PCL_BIT, pgste);
+ preempt_enable();
+#endif
+}
+
+/* forward declaration for SetPageUptodate in page-flags.h*/
+static inline void page_clear_dirty(struct page *page);
+#include <linux/page-flags.h>
+
+static inline void ptep_rcp_copy(pte_t *ptep)
+{
+#ifdef CONFIG_PGSTE
+ struct page *page = virt_to_page(pte_val(*ptep));
+ unsigned int skey;
+ unsigned long *pgste = (unsigned long *) (ptep + PTRS_PER_PTE);
+
+ skey = page_get_storage_key(page_to_phys(page));
+ if (skey & _PAGE_CHANGED)
+ set_bit_simple(RCP_GC_BIT, pgste);
+ if (skey & _PAGE_REFERENCED)
+ set_bit_simple(RCP_GR_BIT, pgste);
+ if (test_and_clear_bit_simple(RCP_HC_BIT, pgste))
+ SetPageDirty(page);
+ if (test_and_clear_bit_simple(RCP_HR_BIT, pgste))
+ SetPageReferenced(page);
+#endif
+}
/*
* query functions pte_write/pte_dirty/pte_young only work if
#ifndef __s390x__
-static inline void pgd_clear(pgd_t * pgdp) { }
+#define pgd_clear(pgd) do { } while (0)
+#define pud_clear(pud) do { } while (0)
-static inline void pmd_clear_kernel(pmd_t * pmdp)
+#else /* __s390x__ */
+
+static inline void pgd_clear_kernel(pgd_t * pgd)
{
- pmd_val(pmdp[0]) = _PAGE_TABLE_INV;
- pmd_val(pmdp[1]) = _PAGE_TABLE_INV;
- pmd_val(pmdp[2]) = _PAGE_TABLE_INV;
- pmd_val(pmdp[3]) = _PAGE_TABLE_INV;
+ if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
+ pgd_val(*pgd) = _REGION2_ENTRY_EMPTY;
}
-static inline void pmd_clear(pmd_t * pmdp)
+static inline void pgd_clear(pgd_t * pgd)
{
- pmd_t *shadow_pmd = get_shadow_pmd(pmdp);
+ pgd_t *shadow = get_shadow_table(pgd);
- pmd_clear_kernel(pmdp);
- if (shadow_pmd)
- pmd_clear_kernel(shadow_pmd);
+ pgd_clear_kernel(pgd);
+ if (shadow)
+ pgd_clear_kernel(shadow);
}
-#else /* __s390x__ */
-
-static inline void pgd_clear_kernel(pgd_t * pgdp)
+static inline void pud_clear_kernel(pud_t *pud)
{
- pgd_val(*pgdp) = _PGD_ENTRY_INV | _PGD_ENTRY;
+ if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
+ pud_val(*pud) = _REGION3_ENTRY_EMPTY;
}
-static inline void pgd_clear(pgd_t * pgdp)
+static inline void pud_clear(pud_t *pud)
{
- pgd_t *shadow_pgd = get_shadow_pgd(pgdp);
+ pud_t *shadow = get_shadow_table(pud);
- pgd_clear_kernel(pgdp);
- if (shadow_pgd)
- pgd_clear_kernel(shadow_pgd);
+ pud_clear_kernel(pud);
+ if (shadow)
+ pud_clear_kernel(shadow);
}
+#endif /* __s390x__ */
+
static inline void pmd_clear_kernel(pmd_t * pmdp)
{
- pmd_val(*pmdp) = _PMD_ENTRY_INV | _PMD_ENTRY;
- pmd_val1(*pmdp) = _PMD_ENTRY_INV | _PMD_ENTRY;
+ pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
}
-static inline void pmd_clear(pmd_t * pmdp)
+static inline void pmd_clear(pmd_t *pmd)
{
- pmd_t *shadow_pmd = get_shadow_pmd(pmdp);
+ pmd_t *shadow = get_shadow_table(pmd);
- pmd_clear_kernel(pmdp);
- if (shadow_pmd)
- pmd_clear_kernel(shadow_pmd);
+ pmd_clear_kernel(pmd);
+ if (shadow)
+ pmd_clear_kernel(shadow);
}
-#endif /* __s390x__ */
-
static inline void pte_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
{
- pte_t *shadow_pte = get_shadow_pte(ptep);
-
+ if (mm->context.pgstes)
+ ptep_rcp_copy(ptep);
pte_val(*ptep) = _PAGE_TYPE_EMPTY;
- if (shadow_pte)
- pte_val(*shadow_pte) = _PAGE_TYPE_EMPTY;
+ if (mm->context.noexec)
+ pte_val(ptep[PTRS_PER_PTE]) = _PAGE_TYPE_EMPTY;
}
/*
return pte;
}
-static inline int ptep_test_and_clear_young(struct vm_area_struct *vma, unsigned long addr, pte_t *ptep)
+static inline pte_t pte_mkspecial(pte_t pte)
{
- return 0;
+ pte_val(pte) |= _PAGE_SPECIAL;
+ return pte;
}
-static inline int
-ptep_clear_flush_young(struct vm_area_struct *vma,
- unsigned long address, pte_t *ptep)
+#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
+static inline int ptep_test_and_clear_young(struct vm_area_struct *vma,
+ unsigned long addr, pte_t *ptep)
{
- /* No need to flush TLB; bits are in storage key */
- return ptep_test_and_clear_young(vma, address, ptep);
+#ifdef CONFIG_PGSTE
+ unsigned long physpage;
+ int young;
+ unsigned long *pgste;
+
+ if (!vma->vm_mm->context.pgstes)
+ return 0;
+ physpage = pte_val(*ptep) & PAGE_MASK;
+ pgste = (unsigned long *) (ptep + PTRS_PER_PTE);
+
+ young = ((page_get_storage_key(physpage) & _PAGE_REFERENCED) != 0);
+ rcp_lock(ptep);
+ if (young)
+ set_bit_simple(RCP_GR_BIT, pgste);
+ young |= test_and_clear_bit_simple(RCP_HR_BIT, pgste);
+ rcp_unlock(ptep);
+ return young;
+#endif
+ return 0;
}
-static inline pte_t ptep_get_and_clear(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
+static inline int ptep_clear_flush_young(struct vm_area_struct *vma,
+ unsigned long address, pte_t *ptep)
{
- pte_t pte = *ptep;
- pte_clear(mm, addr, ptep);
- return pte;
+ /* No need to flush TLB
+ * On s390 reference bits are in storage key and never in TLB
+ * With virtualization we handle the reference bit, without we
+ * we can simply return */
+#ifdef CONFIG_PGSTE
+ return ptep_test_and_clear_young(vma, address, ptep);
+#endif
+ return 0;
}
static inline void __ptep_ipte(unsigned long address, pte_t *ptep)
{
if (!(pte_val(*ptep) & _PAGE_INVALID)) {
#ifndef __s390x__
- /* S390 has 1mb segments, we are emulating 4MB segments */
+ /* pto must point to the start of the segment table */
pte_t *pto = (pte_t *) (((unsigned long) ptep) & 0x7ffffc00);
#else
/* ipte in zarch mode can do the math */
: "=m" (*ptep) : "m" (*ptep),
"a" (pto), "a" (address));
}
- pte_val(*ptep) = _PAGE_TYPE_EMPTY;
}
-static inline void ptep_invalidate(unsigned long address, pte_t *ptep)
+static inline void ptep_invalidate(struct mm_struct *mm,
+ unsigned long address, pte_t *ptep)
{
- __ptep_ipte(address, ptep);
- ptep = get_shadow_pte(ptep);
- if (ptep)
+ if (mm->context.pgstes) {
+ rcp_lock(ptep);
__ptep_ipte(address, ptep);
+ ptep_rcp_copy(ptep);
+ pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+ rcp_unlock(ptep);
+ return;
+ }
+ __ptep_ipte(address, ptep);
+ pte_val(*ptep) = _PAGE_TYPE_EMPTY;
+ if (mm->context.noexec) {
+ __ptep_ipte(address, ptep + PTRS_PER_PTE);
+ pte_val(*(ptep + PTRS_PER_PTE)) = _PAGE_TYPE_EMPTY;
+ }
}
+/*
+ * This is hard to understand. ptep_get_and_clear and ptep_clear_flush
+ * both clear the TLB for the unmapped pte. The reason is that
+ * ptep_get_and_clear is used in common code (e.g. change_pte_range)
+ * to modify an active pte. The sequence is
+ * 1) ptep_get_and_clear
+ * 2) set_pte_at
+ * 3) flush_tlb_range
+ * On s390 the tlb needs to get flushed with the modification of the pte
+ * if the pte is active. The only way how this can be implemented is to
+ * have ptep_get_and_clear do the tlb flush. In exchange flush_tlb_range
+ * is a nop.
+ */
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
+#define ptep_get_and_clear(__mm, __address, __ptep) \
+({ \
+ pte_t __pte = *(__ptep); \
+ if (atomic_read(&(__mm)->mm_users) > 1 || \
+ (__mm) != current->active_mm) \
+ ptep_invalidate(__mm, __address, __ptep); \
+ else \
+ pte_clear((__mm), (__address), (__ptep)); \
+ __pte; \
+})
+
+#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
static inline pte_t ptep_clear_flush(struct vm_area_struct *vma,
unsigned long address, pte_t *ptep)
{
pte_t pte = *ptep;
- ptep_invalidate(address, ptep);
+ ptep_invalidate(vma->vm_mm, address, ptep);
return pte;
}
-static inline void ptep_set_wrprotect(struct mm_struct *mm, unsigned long addr, pte_t *ptep)
+/*
+ * The batched pte unmap code uses ptep_get_and_clear_full to clear the
+ * ptes. Here an optimization is possible. tlb_gather_mmu flushes all
+ * tlbs of an mm if it can guarantee that the ptes of the mm_struct
+ * cannot be accessed while the batched unmap is running. In this case
+ * full==1 and a simple pte_clear is enough. See tlb.h.
+ */
+#define __HAVE_ARCH_PTEP_GET_AND_CLEAR_FULL
+static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm,
+ unsigned long addr,
+ pte_t *ptep, int full)
{
- pte_t old_pte = *ptep;
- set_pte_at(mm, addr, ptep, pte_wrprotect(old_pte));
+ pte_t pte = *ptep;
+
+ if (full)
+ pte_clear(mm, addr, ptep);
+ else
+ ptep_invalidate(mm, addr, ptep);
+ return pte;
}
+#define __HAVE_ARCH_PTEP_SET_WRPROTECT
+#define ptep_set_wrprotect(__mm, __addr, __ptep) \
+({ \
+ pte_t __pte = *(__ptep); \
+ if (pte_write(__pte)) { \
+ if (atomic_read(&(__mm)->mm_users) > 1 || \
+ (__mm) != current->active_mm) \
+ ptep_invalidate(__mm, __addr, __ptep); \
+ set_pte_at(__mm, __addr, __ptep, pte_wrprotect(__pte)); \
+ } \
+})
+
+#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
#define ptep_set_access_flags(__vma, __addr, __ptep, __entry, __dirty) \
({ \
int __changed = !pte_same(*(__ptep), __entry); \
if (__changed) { \
- ptep_invalidate(__addr, __ptep); \
+ ptep_invalidate((__vma)->vm_mm, __addr, __ptep); \
set_pte_at((__vma)->vm_mm, __addr, __ptep, __entry); \
} \
__changed; \
* should therefore only be called if it is not mapped in any
* address space.
*/
+#define __HAVE_ARCH_PAGE_TEST_DIRTY
static inline int page_test_dirty(struct page *page)
{
return (page_get_storage_key(page_to_phys(page)) & _PAGE_CHANGED) != 0;
}
+#define __HAVE_ARCH_PAGE_CLEAR_DIRTY
static inline void page_clear_dirty(struct page *page)
{
page_set_storage_key(page_to_phys(page), PAGE_DEFAULT_KEY);
/*
* Test and clear referenced bit in storage key.
*/
+#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG
static inline int page_test_and_clear_young(struct page *page)
{
unsigned long physpage = page_to_phys(page);
return mk_pte_phys(physpage, pgprot);
}
-static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot)
-{
- unsigned long physpage = __pa((pfn) << PAGE_SHIFT);
-
- return mk_pte_phys(physpage, pgprot);
-}
-
-#ifdef __s390x__
-
-static inline pmd_t pfn_pmd(unsigned long pfn, pgprot_t pgprot)
-{
- unsigned long physpage = __pa((pfn) << PAGE_SHIFT);
-
- return __pmd(physpage + pgprot_val(pgprot));
-}
-
-#endif /* __s390x__ */
-
-#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
-#define pte_page(x) pfn_to_page(pte_pfn(x))
+#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
+#define pud_index(address) (((address) >> PUD_SHIFT) & (PTRS_PER_PUD-1))
+#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
+#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
-#define pmd_page_vaddr(pmd) (pmd_val(pmd) & PAGE_MASK)
+#define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address))
+#define pgd_offset_k(address) pgd_offset(&init_mm, address)
-#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
+#ifndef __s390x__
-#define pgd_page_vaddr(pgd) (pgd_val(pgd) & PAGE_MASK)
+#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
+#define pud_deref(pmd) ({ BUG(); 0UL; })
+#define pgd_deref(pmd) ({ BUG(); 0UL; })
-#define pgd_page(pgd) pfn_to_page(pgd_val(pgd) >> PAGE_SHIFT)
+#define pud_offset(pgd, address) ((pud_t *) pgd)
+#define pmd_offset(pud, address) ((pmd_t *) pud + pmd_index(address))
-/* to find an entry in a page-table-directory */
-#define pgd_index(address) (((address) >> PGDIR_SHIFT) & (PTRS_PER_PGD-1))
-#define pgd_offset(mm, address) ((mm)->pgd+pgd_index(address))
+#else /* __s390x__ */
-/* to find an entry in a kernel page-table-directory */
-#define pgd_offset_k(address) pgd_offset(&init_mm, address)
+#define pmd_deref(pmd) (pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN)
+#define pud_deref(pud) (pud_val(pud) & _REGION_ENTRY_ORIGIN)
+#define pgd_deref(pgd) (pgd_val(pgd) & _REGION_ENTRY_ORIGIN)
-#ifndef __s390x__
+static inline pud_t *pud_offset(pgd_t *pgd, unsigned long address)
+{
+ pud_t *pud = (pud_t *) pgd;
+ if ((pgd_val(*pgd) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R2)
+ pud = (pud_t *) pgd_deref(*pgd);
+ return pud + pud_index(address);
+}
-/* Find an entry in the second-level page table.. */
-static inline pmd_t * pmd_offset(pgd_t * dir, unsigned long address)
+static inline pmd_t *pmd_offset(pud_t *pud, unsigned long address)
{
- return (pmd_t *) dir;
+ pmd_t *pmd = (pmd_t *) pud;
+ if ((pud_val(*pud) & _REGION_ENTRY_TYPE_MASK) == _REGION_ENTRY_TYPE_R3)
+ pmd = (pmd_t *) pud_deref(*pud);
+ return pmd + pmd_index(address);
}
-#else /* __s390x__ */
+#endif /* __s390x__ */
-/* Find an entry in the second-level page table.. */
-#define pmd_index(address) (((address) >> PMD_SHIFT) & (PTRS_PER_PMD-1))
-#define pmd_offset(dir,addr) \
- ((pmd_t *) pgd_page_vaddr(*(dir)) + pmd_index(addr))
+#define pfn_pte(pfn,pgprot) mk_pte_phys(__pa((pfn) << PAGE_SHIFT),(pgprot))
+#define pte_pfn(x) (pte_val(x) >> PAGE_SHIFT)
+#define pte_page(x) pfn_to_page(pte_pfn(x))
-#endif /* __s390x__ */
+#define pmd_page(pmd) pfn_to_page(pmd_val(pmd) >> PAGE_SHIFT)
-/* Find an entry in the third-level page table.. */
-#define pte_index(address) (((address) >> PAGE_SHIFT) & (PTRS_PER_PTE-1))
-#define pte_offset_kernel(pmd, address) \
- ((pte_t *) pmd_page_vaddr(*(pmd)) + pte_index(address))
+/* Find an entry in the lowest level page table.. */
+#define pte_offset(pmd, addr) ((pte_t *) pmd_deref(*(pmd)) + pte_index(addr))
+#define pte_offset_kernel(pmd, address) pte_offset(pmd,address)
#define pte_offset_map(pmd, address) pte_offset_kernel(pmd, address)
#define pte_offset_map_nested(pmd, address) pte_offset_kernel(pmd, address)
#define pte_unmap(pte) do { } while (0)
extern int add_shared_memory(unsigned long start, unsigned long size);
extern int remove_shared_memory(unsigned long start, unsigned long size);
+extern int s390_enable_sie(void);
/*
* No page table caches to initialise
#define __HAVE_ARCH_MEMMAP_INIT
extern void memmap_init(unsigned long, int, unsigned long, unsigned long);
-#define __HAVE_ARCH_PTEP_SET_ACCESS_FLAGS
-#define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG
-#define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH
-#define __HAVE_ARCH_PTEP_GET_AND_CLEAR
-#define __HAVE_ARCH_PTEP_CLEAR_FLUSH
-#define __HAVE_ARCH_PTEP_SET_WRPROTECT
-#define __HAVE_ARCH_PTE_SAME
-#define __HAVE_ARCH_PAGE_TEST_DIRTY
-#define __HAVE_ARCH_PAGE_CLEAR_DIRTY
-#define __HAVE_ARCH_PAGE_TEST_AND_CLEAR_YOUNG
#include <asm-generic/pgtable.h>
#endif /* _S390_PAGE_H */
-
projects / linux-2.6-omap-h63xx.git / blobdiff