1 /* $Id: ultra.S,v 1.72 2002/02/09 19:49:31 davem Exp $
2 * ultra.S: Don't expand these all over the place...
4 * Copyright (C) 1997, 2000 David S. Miller (davem@redhat.com)
7 #include <linux/config.h>
9 #include <asm/pgtable.h>
11 #include <asm/spitfire.h>
12 #include <asm/mmu_context.h>
16 #include <asm/thread_info.h>
17 #include <asm/cacheflush.h>
18 #include <asm/hypervisor.h>
20 /* Basically, most of the Spitfire vs. Cheetah madness
21 * has to do with the fact that Cheetah does not support
22 * IMMU flushes out of the secondary context. Someone needs
23 * to throw a south lake birthday party for the folks
24 * in Microelectronics who refused to fix this shit.
27 /* This file is meant to be read efficiently by the CPU, not humans.
28 * Staraj sie tego nikomu nie pierdolnac...
33 __flush_tlb_mm: /* 18 insns */
34 /* %o0=(ctx & TAG_CONTEXT_BITS), %o1=SECONDARY_CONTEXT */
35 ldxa [%o1] ASI_DMMU, %g2
37 bne,pn %icc, __spitfire_flush_tlb_mm_slow
39 stxa %g0, [%g3] ASI_DMMU_DEMAP
40 stxa %g0, [%g3] ASI_IMMU_DEMAP
41 sethi %hi(KERNBASE), %g3
56 .globl __flush_tlb_pending
57 __flush_tlb_pending: /* 26 insns */
58 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
61 andn %g7, PSTATE_IE, %g2
63 mov SECONDARY_CONTEXT, %o4
64 ldxa [%o4] ASI_DMMU, %g2
65 stxa %o0, [%o4] ASI_DMMU
66 1: sub %o1, (1 << 3), %o1
72 stxa %g0, [%o3] ASI_IMMU_DEMAP
73 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
77 stxa %g2, [%o4] ASI_DMMU
78 sethi %hi(KERNBASE), %o4
81 wrpr %g7, 0x0, %pstate
88 .globl __flush_tlb_kernel_range
89 __flush_tlb_kernel_range: /* 14 insns */
90 /* %o0=start, %o1=end */
93 sethi %hi(PAGE_SIZE), %o4
96 or %o0, 0x20, %o0 ! Nucleus
97 1: stxa %g0, [%o0 + %o3] ASI_DMMU_DEMAP
98 stxa %g0, [%o0 + %o3] ASI_IMMU_DEMAP
102 2: sethi %hi(KERNBASE), %o3
108 __spitfire_flush_tlb_mm_slow:
110 wrpr %g1, PSTATE_IE, %pstate
111 stxa %o0, [%o1] ASI_DMMU
112 stxa %g0, [%g3] ASI_DMMU_DEMAP
113 stxa %g0, [%g3] ASI_IMMU_DEMAP
115 stxa %g2, [%o1] ASI_DMMU
116 sethi %hi(KERNBASE), %o1
122 * The following code flushes one page_size worth.
124 #if (PAGE_SHIFT == 13)
125 #define ITAG_MASK 0xfe
126 #elif (PAGE_SHIFT == 16)
127 #define ITAG_MASK 0x7fe
129 #error unsupported PAGE_SIZE
131 .section .kprobes.text, "ax"
133 .globl __flush_icache_page
134 __flush_icache_page: /* %o0 = phys_page */
136 srlx %o0, PAGE_SHIFT, %o0
137 sethi %uhi(PAGE_OFFSET), %g1
138 sllx %o0, PAGE_SHIFT, %o0
139 sethi %hi(PAGE_SIZE), %g2
142 1: subcc %g2, 32, %g2
148 #ifdef DCACHE_ALIASING_POSSIBLE
150 #if (PAGE_SHIFT != 13)
151 #error only page shift of 13 is supported by dcache flush
154 #define DTAG_MASK 0x3
156 /* This routine is Spitfire specific so the hardcoded
157 * D-cache size and line-size are OK.
160 .globl __flush_dcache_page
161 __flush_dcache_page: /* %o0=kaddr, %o1=flush_icache */
162 sethi %uhi(PAGE_OFFSET), %g1
164 sub %o0, %g1, %o0 ! physical address
165 srlx %o0, 11, %o0 ! make D-cache TAG
166 sethi %hi(1 << 14), %o2 ! D-cache size
167 sub %o2, (1 << 5), %o2 ! D-cache line size
168 1: ldxa [%o2] ASI_DCACHE_TAG, %o3 ! load D-cache TAG
169 andcc %o3, DTAG_MASK, %g0 ! Valid?
170 be,pn %xcc, 2f ! Nope, branch
171 andn %o3, DTAG_MASK, %o3 ! Clear valid bits
172 cmp %o3, %o0 ! TAG match?
173 bne,pt %xcc, 2f ! Nope, branch
175 stxa %g0, [%o2] ASI_DCACHE_TAG ! Invalidate TAG
178 sub %o2, (1 << 5), %o2 ! D-cache line size
180 /* The I-cache does not snoop local stores so we
181 * better flush that too when necessary.
183 brnz,pt %o1, __flush_icache_page
188 #endif /* DCACHE_ALIASING_POSSIBLE */
192 /* Cheetah specific versions, patched at boot time. */
193 __cheetah_flush_tlb_mm: /* 19 insns */
195 andn %g7, PSTATE_IE, %g2
196 wrpr %g2, 0x0, %pstate
198 mov PRIMARY_CONTEXT, %o2
200 ldxa [%o2] ASI_DMMU, %g2
201 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o1
202 sllx %o1, CTX_PGSZ1_NUC_SHIFT, %o1
203 or %o0, %o1, %o0 /* Preserve nucleus page size fields */
204 stxa %o0, [%o2] ASI_DMMU
205 stxa %g0, [%g3] ASI_DMMU_DEMAP
206 stxa %g0, [%g3] ASI_IMMU_DEMAP
207 stxa %g2, [%o2] ASI_DMMU
208 sethi %hi(KERNBASE), %o2
212 wrpr %g7, 0x0, %pstate
214 __cheetah_flush_tlb_pending: /* 27 insns */
215 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
218 andn %g7, PSTATE_IE, %g2
219 wrpr %g2, 0x0, %pstate
221 mov PRIMARY_CONTEXT, %o4
222 ldxa [%o4] ASI_DMMU, %g2
223 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %o3
224 sllx %o3, CTX_PGSZ1_NUC_SHIFT, %o3
225 or %o0, %o3, %o0 /* Preserve nucleus page size fields */
226 stxa %o0, [%o4] ASI_DMMU
227 1: sub %o1, (1 << 3), %o1
232 stxa %g0, [%o3] ASI_IMMU_DEMAP
233 2: stxa %g0, [%o3] ASI_DMMU_DEMAP
237 stxa %g2, [%o4] ASI_DMMU
238 sethi %hi(KERNBASE), %o4
242 wrpr %g7, 0x0, %pstate
244 #ifdef DCACHE_ALIASING_POSSIBLE
245 __cheetah_flush_dcache_page: /* 11 insns */
246 sethi %uhi(PAGE_OFFSET), %g1
249 sethi %hi(PAGE_SIZE), %o4
250 1: subcc %o4, (1 << 5), %o4
251 stxa %g0, [%o0 + %o4] ASI_DCACHE_INVALIDATE
255 retl /* I-cache flush never needed on Cheetah, see callers. */
257 #endif /* DCACHE_ALIASING_POSSIBLE */
259 /* Hypervisor specific versions, patched at boot time. */
260 __hypervisor_flush_tlb_mm: /* 8 insns */
261 mov %o0, %o2 /* ARG2: mmu context */
262 mov 0, %o0 /* ARG0: CPU lists unimplemented */
263 mov 0, %o1 /* ARG1: CPU lists unimplemented */
264 mov HV_MMU_ALL, %o3 /* ARG3: flags */
265 mov HV_FAST_MMU_DEMAP_CTX, %o5
270 __hypervisor_flush_tlb_pending: /* 15 insns */
271 /* %o0 = context, %o1 = nr, %o2 = vaddrs[] */
275 1: sub %g1, (1 << 3), %g1
276 ldx [%g2 + %g1], %o0 /* ARG0: vaddr + IMMU-bit */
277 mov %g3, %o1 /* ARG1: mmu context */
280 movne %icc, HV_MMU_ALL, %o2 /* ARG2: flags */
282 ta HV_MMU_UNMAP_ADDR_TRAP
288 __hypervisor_flush_tlb_kernel_range: /* 14 insns */
289 /* %o0=start, %o1=end */
292 sethi %hi(PAGE_SIZE), %g3
296 1: add %g1, %g2, %o0 /* ARG0: virtual address */
297 mov 0, %o1 /* ARG1: mmu context */
298 mov HV_MMU_ALL, %o2 /* ARG2: flags */
299 ta HV_MMU_UNMAP_ADDR_TRAP
305 #ifdef DCACHE_ALIASING_POSSIBLE
306 /* XXX Niagara and friends have an 8K cache, so no aliasing is
307 * XXX possible, but nothing explicit in the Hypervisor API
308 * XXX guarantees this.
310 __hypervisor_flush_dcache_page: /* 2 insns */
326 .globl cheetah_patch_cachetlbops
327 cheetah_patch_cachetlbops:
330 sethi %hi(__flush_tlb_mm), %o0
331 or %o0, %lo(__flush_tlb_mm), %o0
332 sethi %hi(__cheetah_flush_tlb_mm), %o1
333 or %o1, %lo(__cheetah_flush_tlb_mm), %o1
337 sethi %hi(__flush_tlb_pending), %o0
338 or %o0, %lo(__flush_tlb_pending), %o0
339 sethi %hi(__cheetah_flush_tlb_pending), %o1
340 or %o1, %lo(__cheetah_flush_tlb_pending), %o1
344 #ifdef DCACHE_ALIASING_POSSIBLE
345 sethi %hi(__flush_dcache_page), %o0
346 or %o0, %lo(__flush_dcache_page), %o0
347 sethi %hi(__cheetah_flush_dcache_page), %o1
348 or %o1, %lo(__cheetah_flush_dcache_page), %o1
351 #endif /* DCACHE_ALIASING_POSSIBLE */
357 /* These are all called by the slaves of a cross call, at
358 * trap level 1, with interrupts fully disabled.
361 * %g5 mm->context (all tlb flushes)
362 * %g1 address arg 1 (tlb page and range flushes)
363 * %g7 address arg 2 (tlb range flush only)
371 .globl xcall_flush_tlb_mm
372 xcall_flush_tlb_mm: /* 18 insns */
373 mov PRIMARY_CONTEXT, %g2
374 ldxa [%g2] ASI_DMMU, %g3
375 srlx %g3, CTX_PGSZ1_NUC_SHIFT, %g4
376 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
377 or %g5, %g4, %g5 /* Preserve nucleus page size fields */
378 stxa %g5, [%g2] ASI_DMMU
380 stxa %g0, [%g4] ASI_DMMU_DEMAP
381 stxa %g0, [%g4] ASI_IMMU_DEMAP
382 stxa %g3, [%g2] ASI_DMMU
392 .globl xcall_flush_tlb_pending
393 xcall_flush_tlb_pending: /* 20 insns */
394 /* %g5=context, %g1=nr, %g7=vaddrs[] */
396 mov PRIMARY_CONTEXT, %g4
397 ldxa [%g4] ASI_DMMU, %g2
398 srlx %g2, CTX_PGSZ1_NUC_SHIFT, %g4
399 sllx %g4, CTX_PGSZ1_NUC_SHIFT, %g4
401 mov PRIMARY_CONTEXT, %g4
402 stxa %g5, [%g4] ASI_DMMU
403 1: sub %g1, (1 << 3), %g1
409 stxa %g0, [%g5] ASI_IMMU_DEMAP
410 2: stxa %g0, [%g5] ASI_DMMU_DEMAP
414 stxa %g2, [%g4] ASI_DMMU
417 .globl xcall_flush_tlb_kernel_range
418 xcall_flush_tlb_kernel_range: /* 22 insns */
419 sethi %hi(PAGE_SIZE - 1), %g2
420 or %g2, %lo(PAGE_SIZE - 1), %g2
426 or %g1, 0x20, %g1 ! Nucleus
427 1: stxa %g0, [%g1 + %g3] ASI_DMMU_DEMAP
428 stxa %g0, [%g1 + %g3] ASI_IMMU_DEMAP
442 /* This runs in a very controlled environment, so we do
443 * not need to worry about BH races etc.
445 .globl xcall_sync_tick
448 661: rdpr %pstate, %g2
449 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
450 .section .gl_2insn_patch, "ax"
460 109: or %g7, %lo(109b), %g7
461 call smp_synchronize_tick_client
465 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
467 /* NOTE: This is SPECIAL!! We do etrap/rtrap however
468 * we choose to deal with the "BH's run with
469 * %pil==15" problem (described in asm/pil.h)
470 * by just invoking rtrap directly past where
471 * BH's are checked for.
473 * We do it like this because we do not want %pil==15
474 * lockups to prevent regs being reported.
476 .globl xcall_report_regs
479 661: rdpr %pstate, %g2
480 wrpr %g2, PSTATE_IG | PSTATE_AG, %pstate
481 .section .gl_2insn_patch, "ax"
491 109: or %g7, %lo(109b), %g7
493 add %sp, PTREGS_OFF, %o0
495 /* Has to be a non-v9 branch due to the large distance. */
497 ldx [%sp + PTREGS_OFF + PT_V9_TSTATE], %l1
499 #ifdef DCACHE_ALIASING_POSSIBLE
501 .globl xcall_flush_dcache_page_cheetah
502 xcall_flush_dcache_page_cheetah: /* %g1 == physical page address */
503 sethi %hi(PAGE_SIZE), %g3
504 1: subcc %g3, (1 << 5), %g3
505 stxa %g0, [%g1 + %g3] ASI_DCACHE_INVALIDATE
511 #endif /* DCACHE_ALIASING_POSSIBLE */
513 .globl xcall_flush_dcache_page_spitfire
514 xcall_flush_dcache_page_spitfire: /* %g1 == physical page address
515 %g7 == kernel page virtual address
516 %g5 == (page->mapping != NULL) */
517 #ifdef DCACHE_ALIASING_POSSIBLE
518 srlx %g1, (13 - 2), %g1 ! Form tag comparitor
519 sethi %hi(L1DCACHE_SIZE), %g3 ! D$ size == 16K
520 sub %g3, (1 << 5), %g3 ! D$ linesize == 32
521 1: ldxa [%g3] ASI_DCACHE_TAG, %g2
529 stxa %g0, [%g3] ASI_DCACHE_TAG
533 sub %g3, (1 << 5), %g3
536 #endif /* DCACHE_ALIASING_POSSIBLE */
537 sethi %hi(PAGE_SIZE), %g3
540 subcc %g3, (1 << 5), %g3
542 add %g7, (1 << 5), %g7
548 .globl __hypervisor_xcall_flush_tlb_mm
549 __hypervisor_xcall_flush_tlb_mm: /* 18 insns */
550 /* %g5=ctx, g1,g2,g3,g4,g7=scratch, %g6=unusable */
556 clr %o0 /* ARG0: CPU lists unimplemented */
557 clr %o1 /* ARG1: CPU lists unimplemented */
558 mov %g5, %o2 /* ARG2: mmu context */
559 mov HV_MMU_ALL, %o3 /* ARG3: flags */
560 mov HV_FAST_MMU_DEMAP_CTX, %o5
570 .globl __hypervisor_xcall_flush_tlb_pending
571 __hypervisor_xcall_flush_tlb_pending: /* 18 insns */
572 /* %g5=ctx, %g1=nr, %g7=vaddrs[], %g2,%g3,%g4=scratch, %g6=unusable */
577 1: sub %g1, (1 << 3), %g1
578 ldx [%g7 + %g1], %o0 /* ARG0: virtual address */
579 mov %g5, %o1 /* ARG1: mmu context */
582 movne %icc, HV_MMU_ALL, %o2 /* ARG2: flags */
583 ta HV_MMU_UNMAP_ADDR_TRAP
592 .globl __hypervisor_xcall_flush_tlb_kernel_range
593 __hypervisor_xcall_flush_tlb_kernel_range: /* 22 insns */
594 /* %g1=start, %g7=end, g2,g3,g4,g5=scratch, g6=unusable */
595 sethi %hi(PAGE_SIZE - 1), %g2
596 or %g2, %lo(PAGE_SIZE - 1), %g2
605 1: add %g1, %g3, %o0 /* ARG0: virtual address */
606 mov 0, %o1 /* ARG1: mmu context */
607 mov HV_MMU_ALL, %o2 /* ARG2: flags */
608 ta HV_MMU_UNMAP_ADDR_TRAP
609 sethi %hi(PAGE_SIZE), %o2
618 /* These just get rescheduled to PIL vectors. */
619 .globl xcall_call_function
621 wr %g0, (1 << PIL_SMP_CALL_FUNC), %set_softint
624 .globl xcall_receive_signal
625 xcall_receive_signal:
626 wr %g0, (1 << PIL_SMP_RECEIVE_SIGNAL), %set_softint
631 wr %g0, (1 << PIL_SMP_CAPTURE), %set_softint
634 #endif /* CONFIG_SMP */
637 .globl hypervisor_patch_cachetlbops
638 hypervisor_patch_cachetlbops:
641 sethi %hi(__flush_tlb_mm), %o0
642 or %o0, %lo(__flush_tlb_mm), %o0
643 sethi %hi(__hypervisor_flush_tlb_mm), %o1
644 or %o1, %lo(__hypervisor_flush_tlb_mm), %o1
648 sethi %hi(__flush_tlb_pending), %o0
649 or %o0, %lo(__flush_tlb_pending), %o0
650 sethi %hi(__hypervisor_flush_tlb_pending), %o1
651 or %o1, %lo(__hypervisor_flush_tlb_pending), %o1
655 sethi %hi(__flush_tlb_kernel_range), %o0
656 or %o0, %lo(__flush_tlb_kernel_range), %o0
657 sethi %hi(__hypervisor_flush_tlb_kernel_range), %o1
658 or %o1, %lo(__hypervisor_flush_tlb_kernel_range), %o1
662 #ifdef DCACHE_ALIASING_POSSIBLE
663 sethi %hi(__flush_dcache_page), %o0
664 or %o0, %lo(__flush_dcache_page), %o0
665 sethi %hi(__hypervisor_flush_dcache_page), %o1
666 or %o1, %lo(__hypervisor_flush_dcache_page), %o1
669 #endif /* DCACHE_ALIASING_POSSIBLE */
672 sethi %hi(xcall_flush_tlb_mm), %o0
673 or %o0, %lo(xcall_flush_tlb_mm), %o0
674 sethi %hi(__hypervisor_xcall_flush_tlb_mm), %o1
675 or %o1, %lo(__hypervisor_xcall_flush_tlb_mm), %o1
679 sethi %hi(xcall_flush_tlb_pending), %o0
680 or %o0, %lo(xcall_flush_tlb_pending), %o0
681 sethi %hi(__hypervisor_xcall_flush_tlb_pending), %o1
682 or %o1, %lo(__hypervisor_xcall_flush_tlb_pending), %o1
686 sethi %hi(xcall_flush_tlb_kernel_range), %o0
687 or %o0, %lo(xcall_flush_tlb_kernel_range), %o0
688 sethi %hi(__hypervisor_xcall_flush_tlb_kernel_range), %o1
689 or %o1, %lo(__hypervisor_xcall_flush_tlb_kernel_range), %o1
692 #endif /* CONFIG_SMP */