* Synthesize TLB refill handlers at runtime.
*
* Copyright (C) 2004,2005,2006 by Thiemo Seufer
- * Copyright (C) 2005 Maciej W. Rozycki
+ * Copyright (C) 2005, 2007 Maciej W. Rozycki
* Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org)
*
* ... and the days got worse and worse and now you see
* (Condolences to Napoleon XIV)
*/
-#include <stdarg.h>
-
-#include <linux/mm.h>
#include <linux/kernel.h>
#include <linux/types.h>
#include <linux/string.h>
#include <linux/init.h>
-#include <asm/pgtable.h>
-#include <asm/cacheflush.h>
+#include <asm/bugs.h>
#include <asm/mmu_context.h>
#include <asm/inst.h>
#include <asm/elf.h>
-#include <asm/smp.h>
#include <asm/war.h>
static inline int r45k_bvahwbug(void)
break;
}
- if (!ip)
+ if (!ip || (opc == insn_daddiu && r4k_daddiu_bug()))
panic("Unsupported TLB synthesizer instruction %d", opc);
op = ip->match;
}
#define I_u1u2u3(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b, unsigned int c) \
{ \
build_insn(buf, insn##op, a, b, c); \
}
#define I_u2u1u3(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b, unsigned int c) \
{ \
build_insn(buf, insn##op, b, a, c); \
}
#define I_u3u1u2(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b, unsigned int c) \
{ \
build_insn(buf, insn##op, b, c, a); \
}
#define I_u1u2s3(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b, signed int c) \
{ \
build_insn(buf, insn##op, a, b, c); \
}
#define I_u2s3u1(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
signed int b, unsigned int c) \
{ \
build_insn(buf, insn##op, c, a, b); \
}
#define I_u2u1s3(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b, signed int c) \
{ \
build_insn(buf, insn##op, b, a, c); \
}
#define I_u1u2(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
unsigned int b) \
{ \
build_insn(buf, insn##op, a, b); \
}
#define I_u1s2(op) \
- static inline void i##op(u32 **buf, unsigned int a, \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \
signed int b) \
{ \
build_insn(buf, insn##op, a, b); \
}
#define I_u1(op) \
- static inline void i##op(u32 **buf, unsigned int a) \
+ static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \
{ \
build_insn(buf, insn##op, a); \
}
#define I_0(op) \
- static inline void i##op(u32 **buf) \
+ static void __init __maybe_unused i##op(u32 **buf) \
{ \
build_insn(buf, insn##op); \
}
}
#define L_LA(lb) \
- static inline void l##lb(struct label **lab, u32 *addr) \
+ static inline void __init l##lb(struct label **lab, u32 *addr) \
{ \
build_label(lab, addr, label##lb); \
}
#define i_ssnop(buf) i_sll(buf, 0, 0, 1)
#define i_ehb(buf) i_sll(buf, 0, 0, 3)
-#ifdef CONFIG_64BIT
static int __init __maybe_unused in_compat_space_p(long addr)
{
/* Is this address in 32bit compat space? */
+#ifdef CONFIG_64BIT
return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L);
+#else
+ return 1;
+#endif
}
static int __init __maybe_unused rel_highest(long val)
{
+#ifdef CONFIG_64BIT
return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000;
+#else
+ return 0;
+#endif
}
static int __init __maybe_unused rel_higher(long val)
{
+#ifdef CONFIG_64BIT
return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000;
-}
+#else
+ return 0;
#endif
+}
static int __init rel_hi(long val)
{
static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr)
{
-#ifdef CONFIG_64BIT
if (!in_compat_space_p(addr)) {
i_lui(buf, rs, rel_highest(addr));
if (rel_higher(addr))
} else
i_dsll32(buf, rs, rs, 0);
} else
-#endif
i_lui(buf, rs, rel_hi(addr));
}
-static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs,
- long addr)
+static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr)
{
i_LA_mostly(buf, rs, addr);
- if (rel_lo(addr))
- i_ADDIU(buf, rs, rs, rel_lo(addr));
+ if (rel_lo(addr)) {
+ if (!in_compat_space_p(addr))
+ i_daddiu(buf, rs, rs, rel_lo(addr));
+ else
+ i_addiu(buf, rs, rs, rel_lo(addr));
+ }
}
/*
i_bgez(p, reg, 0);
}
+/*
+ * For debug purposes.
+ */
+static inline void dump_handler(const u32 *handler, int count)
+{
+ int i;
+
+ pr_debug("\t.set push\n");
+ pr_debug("\t.set noreorder\n");
+
+ for (i = 0; i < count; i++)
+ pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]);
+
+ pr_debug("\t.set pop\n");
+}
+
/* The only general purpose registers allowed in TLB handlers. */
#define K0 26
#define K1 27
{
long pgdc = (long)pgd_current;
u32 *p;
- int i;
memset(tlb_handler, 0, sizeof(tlb_handler));
p = tlb_handler;
pr_info("Synthesized TLB refill handler (%u instructions).\n",
(unsigned int)(p - tlb_handler));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - tlb_handler); i++)
- pr_debug("\t.word 0x%08x\n", tlb_handler[i]);
- pr_debug("\t.set pop\n");
-
memcpy((void *)ebase, tlb_handler, 0x80);
+
+ dump_handler((u32 *)ebase, 32);
}
/*
case tlb_indexed: tlbw = i_tlbwi; break;
}
+ if (cpu_has_mips_r2) {
+ i_ehb(p);
+ tlbw(p);
+ return;
+ }
+
switch (current_cpu_type()) {
case CPU_R4000PC:
case CPU_R4000SC:
case CPU_AU1500:
case CPU_AU1550:
case CPU_AU1200:
+ case CPU_AU1210:
+ case CPU_AU1250:
case CPU_PR4450:
i_nop(p);
tlbw(p);
tlbw(p);
break;
- case CPU_4KEC:
- case CPU_24K:
- case CPU_34K:
- case CPU_74K:
- i_ehb(p);
- tlbw(p);
- break;
-
case CPU_RM9000:
/*
* When the JTLB is updated by tlbwi or tlbwr, a subsequent
} else {
i_LA_mostly(p, ptr, modd);
il_b(p, r, label_vmalloc_done);
- i_daddiu(p, ptr, ptr, rel_lo(modd));
+ if (in_compat_space_p(modd))
+ i_addiu(p, ptr, ptr, rel_lo(modd));
+ else
+ i_daddiu(p, ptr, ptr, rel_lo(modd));
}
l_vmalloc(l, *p);
} else {
i_LA_mostly(p, ptr, swpd);
il_b(p, r, label_vmalloc_done);
- i_daddiu(p, ptr, ptr, rel_lo(swpd));
+ if (in_compat_space_p(swpd))
+ i_addiu(p, ptr, ptr, rel_lo(swpd));
+ else
+ i_daddiu(p, ptr, ptr, rel_lo(swpd));
}
}
struct reloc *r = relocs;
u32 *f;
unsigned int final_len;
- int i;
memset(tlb_handler, 0, sizeof(tlb_handler));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB refill handler (%u instructions).\n",
final_len);
- f = final_handler;
-#if defined(CONFIG_64BIT) && !defined(CONFIG_CPU_LOONGSON2)
- if (final_len > 32)
- final_len = 64;
- else
- f = final_handler + 32;
-#endif /* CONFIG_64BIT */
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < final_len; i++)
- pr_debug("\t.word 0x%08x\n", f[i]);
- pr_debug("\t.set pop\n");
-
memcpy((void *)ebase, final_handler, 0x100);
+
+ dump_handler((u32 *)ebase, 64);
}
/*
extern void tlb_do_page_fault_0(void);
extern void tlb_do_page_fault_1(void);
-#define __tlb_handler_align \
- __attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT)))
-
/*
* 128 instructions for the fastpath handler is generous and should
* never be exceeded.
*/
#define FASTPATH_SIZE 128
-u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE];
-u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE];
-u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE];
+u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned;
+u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned;
+u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned;
static void __init
iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr)
u32 *p = handle_tlbl;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbl, 0, sizeof(handle_tlbl));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbl));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbl); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbl[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
}
static void __init build_r3000_tlb_store_handler(void)
u32 *p = handle_tlbs;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbs, 0, sizeof(handle_tlbs));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbs));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbs); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbs[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
}
static void __init build_r3000_tlb_modify_handler(void)
u32 *p = handle_tlbm;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbm, 0, sizeof(handle_tlbm));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbm));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbm); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbm[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
}
/*
u32 *p = handle_tlbl;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbl, 0, sizeof(handle_tlbl));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB load handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbl));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbl); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbl[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl));
}
static void __init build_r4000_tlb_store_handler(void)
u32 *p = handle_tlbs;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbs, 0, sizeof(handle_tlbs));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB store handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbs));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbs); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbs[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs));
}
static void __init build_r4000_tlb_modify_handler(void)
u32 *p = handle_tlbm;
struct label *l = labels;
struct reloc *r = relocs;
- int i;
memset(handle_tlbm, 0, sizeof(handle_tlbm));
memset(labels, 0, sizeof(labels));
pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n",
(unsigned int)(p - handle_tlbm));
- pr_debug("\t.set push\n");
- pr_debug("\t.set noreorder\n");
- for (i = 0; i < (p - handle_tlbm); i++)
- pr_debug("\t.word 0x%08x\n", handle_tlbm[i]);
- pr_debug("\t.set pop\n");
+ dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm));
}
void __init build_tlb_refill_handler(void)
projects / linux-2.6-omap-h63xx.git / blobdiff