1 | /* |
---|
2 | * TLB support routines. |
---|
3 | * |
---|
4 | * Copyright (C) 1998-2001, 2003 Hewlett-Packard Co |
---|
5 | * David Mosberger-Tang <davidm@hpl.hp.com> |
---|
6 | * |
---|
7 | * 08/02/00 A. Mallick <asit.k.mallick@intel.com> |
---|
8 | * Modified RID allocation for SMP |
---|
9 | * Goutham Rao <goutham.rao@intel.com> |
---|
10 | * IPI based ptc implementation and A-step IPI implementation. |
---|
11 | */ |
---|
12 | #include <linux/config.h> |
---|
13 | #include <linux/module.h> |
---|
14 | #include <linux/init.h> |
---|
15 | #include <linux/kernel.h> |
---|
16 | #include <linux/sched.h> |
---|
17 | #include <linux/smp.h> |
---|
18 | #include <linux/mm.h> |
---|
19 | |
---|
20 | #include <asm/delay.h> |
---|
21 | #include <asm/mmu_context.h> |
---|
22 | #include <asm/pgalloc.h> |
---|
23 | #include <asm/pal.h> |
---|
24 | #include <asm/tlbflush.h> |
---|
25 | |
---|
26 | static struct { |
---|
27 | unsigned long mask; /* mask of supported purge page-sizes */ |
---|
28 | unsigned long max_bits; /* log2() of largest supported purge page-size */ |
---|
29 | } purge; |
---|
30 | |
---|
31 | #ifndef XEN |
---|
32 | struct ia64_ctx ia64_ctx = { |
---|
33 | .lock = SPIN_LOCK_UNLOCKED, |
---|
34 | .next = 1, |
---|
35 | .limit = (1 << 15) - 1, /* start out with the safe (architected) limit */ |
---|
36 | .max_ctx = ~0U |
---|
37 | }; |
---|
38 | |
---|
39 | DEFINE_PER_CPU(u8, ia64_need_tlb_flush); |
---|
40 | |
---|
41 | /* |
---|
42 | * Acquire the ia64_ctx.lock before calling this function! |
---|
43 | */ |
---|
44 | void |
---|
45 | wrap_mmu_context (struct mm_struct *mm) |
---|
46 | { |
---|
47 | unsigned long tsk_context, max_ctx = ia64_ctx.max_ctx; |
---|
48 | struct task_struct *tsk; |
---|
49 | int i; |
---|
50 | |
---|
51 | if (ia64_ctx.next > max_ctx) |
---|
52 | ia64_ctx.next = 300; /* skip daemons */ |
---|
53 | ia64_ctx.limit = max_ctx + 1; |
---|
54 | |
---|
55 | /* |
---|
56 | * Scan all the task's mm->context and set proper safe range |
---|
57 | */ |
---|
58 | |
---|
59 | read_lock(&tasklist_lock); |
---|
60 | repeat: |
---|
61 | for_each_process(tsk) { |
---|
62 | if (!tsk->mm) |
---|
63 | continue; |
---|
64 | tsk_context = tsk->mm->context; |
---|
65 | if (tsk_context == ia64_ctx.next) { |
---|
66 | if (++ia64_ctx.next >= ia64_ctx.limit) { |
---|
67 | /* empty range: reset the range limit and start over */ |
---|
68 | if (ia64_ctx.next > max_ctx) |
---|
69 | ia64_ctx.next = 300; |
---|
70 | ia64_ctx.limit = max_ctx + 1; |
---|
71 | goto repeat; |
---|
72 | } |
---|
73 | } |
---|
74 | if ((tsk_context > ia64_ctx.next) && (tsk_context < ia64_ctx.limit)) |
---|
75 | ia64_ctx.limit = tsk_context; |
---|
76 | } |
---|
77 | read_unlock(&tasklist_lock); |
---|
78 | /* can't call flush_tlb_all() here because of race condition with O(1) scheduler [EF] */ |
---|
79 | { |
---|
80 | int cpu = get_cpu(); /* prevent preemption/migration */ |
---|
81 | for (i = 0; i < NR_CPUS; ++i) |
---|
82 | if (cpu_online(i) && (i != cpu)) |
---|
83 | per_cpu(ia64_need_tlb_flush, i) = 1; |
---|
84 | put_cpu(); |
---|
85 | } |
---|
86 | local_flush_tlb_all(); |
---|
87 | } |
---|
88 | #endif /* XEN */ |
---|
89 | |
---|
90 | void |
---|
91 | ia64_global_tlb_purge (unsigned long start, unsigned long end, unsigned long nbits) |
---|
92 | { |
---|
93 | static DEFINE_SPINLOCK(ptcg_lock); |
---|
94 | |
---|
95 | /* HW requires global serialization of ptc.ga. */ |
---|
96 | spin_lock(&ptcg_lock); |
---|
97 | { |
---|
98 | do { |
---|
99 | /* |
---|
100 | * Flush ALAT entries also. |
---|
101 | */ |
---|
102 | ia64_ptcga(start, (nbits<<2)); |
---|
103 | ia64_srlz_i(); |
---|
104 | start += (1UL << nbits); |
---|
105 | } while (start < end); |
---|
106 | } |
---|
107 | spin_unlock(&ptcg_lock); |
---|
108 | } |
---|
109 | |
---|
110 | void |
---|
111 | local_flush_tlb_all (void) |
---|
112 | { |
---|
113 | unsigned long i, j, flags, count0, count1, stride0, stride1, addr; |
---|
114 | #ifdef XEN |
---|
115 | /* increment flush clock before mTLB flush */ |
---|
116 | u32 flush_time = tlbflush_clock_inc_and_return(); |
---|
117 | #endif |
---|
118 | addr = local_cpu_data->ptce_base; |
---|
119 | count0 = local_cpu_data->ptce_count[0]; |
---|
120 | count1 = local_cpu_data->ptce_count[1]; |
---|
121 | stride0 = local_cpu_data->ptce_stride[0]; |
---|
122 | stride1 = local_cpu_data->ptce_stride[1]; |
---|
123 | |
---|
124 | local_irq_save(flags); |
---|
125 | for (i = 0; i < count0; ++i) { |
---|
126 | for (j = 0; j < count1; ++j) { |
---|
127 | ia64_ptce(addr); |
---|
128 | addr += stride1; |
---|
129 | } |
---|
130 | addr += stride0; |
---|
131 | } |
---|
132 | local_irq_restore(flags); |
---|
133 | ia64_srlz_i(); /* srlz.i implies srlz.d */ |
---|
134 | #ifdef XEN |
---|
135 | /* update after mTLB flush. */ |
---|
136 | tlbflush_update_time(&__get_cpu_var(tlbflush_time), flush_time); |
---|
137 | #endif |
---|
138 | } |
---|
139 | EXPORT_SYMBOL(local_flush_tlb_all); |
---|
140 | |
---|
141 | #ifndef XEN |
---|
142 | void |
---|
143 | flush_tlb_range (struct vm_area_struct *vma, unsigned long start, unsigned long end) |
---|
144 | { |
---|
145 | struct mm_struct *mm = vma->vm_mm; |
---|
146 | unsigned long size = end - start; |
---|
147 | unsigned long nbits; |
---|
148 | |
---|
149 | if (mm != current->active_mm) { |
---|
150 | /* this does happen, but perhaps it's not worth optimizing for? */ |
---|
151 | #ifdef CONFIG_SMP |
---|
152 | flush_tlb_all(); |
---|
153 | #else |
---|
154 | mm->context = 0; |
---|
155 | #endif |
---|
156 | return; |
---|
157 | } |
---|
158 | |
---|
159 | nbits = ia64_fls(size + 0xfff); |
---|
160 | while (unlikely (((1UL << nbits) & purge.mask) == 0) && (nbits < purge.max_bits)) |
---|
161 | ++nbits; |
---|
162 | if (nbits > purge.max_bits) |
---|
163 | nbits = purge.max_bits; |
---|
164 | start &= ~((1UL << nbits) - 1); |
---|
165 | |
---|
166 | # ifdef CONFIG_SMP |
---|
167 | platform_global_tlb_purge(start, end, nbits); |
---|
168 | # else |
---|
169 | do { |
---|
170 | ia64_ptcl(start, (nbits<<2)); |
---|
171 | start += (1UL << nbits); |
---|
172 | } while (start < end); |
---|
173 | # endif |
---|
174 | |
---|
175 | ia64_srlz_i(); /* srlz.i implies srlz.d */ |
---|
176 | } |
---|
177 | EXPORT_SYMBOL(flush_tlb_range); |
---|
178 | #endif |
---|
179 | |
---|
180 | void __devinit |
---|
181 | ia64_tlb_init (void) |
---|
182 | { |
---|
183 | #ifndef XEN |
---|
184 | ia64_ptce_info_t ptce_info; |
---|
185 | #else |
---|
186 | ia64_ptce_info_t ptce_info = { 0 }; |
---|
187 | #endif |
---|
188 | unsigned long tr_pgbits; |
---|
189 | long status; |
---|
190 | |
---|
191 | if ((status = ia64_pal_vm_page_size(&tr_pgbits, &purge.mask)) != 0) { |
---|
192 | printk(KERN_ERR "PAL_VM_PAGE_SIZE failed with status=%ld;" |
---|
193 | "defaulting to architected purge page-sizes.\n", status); |
---|
194 | purge.mask = 0x115557000UL; |
---|
195 | } |
---|
196 | purge.max_bits = ia64_fls(purge.mask); |
---|
197 | |
---|
198 | ia64_get_ptce(&ptce_info); |
---|
199 | local_cpu_data->ptce_base = ptce_info.base; |
---|
200 | local_cpu_data->ptce_count[0] = ptce_info.count[0]; |
---|
201 | local_cpu_data->ptce_count[1] = ptce_info.count[1]; |
---|
202 | local_cpu_data->ptce_stride[0] = ptce_info.stride[0]; |
---|
203 | local_cpu_data->ptce_stride[1] = ptce_info.stride[1]; |
---|
204 | |
---|
205 | local_flush_tlb_all(); /* nuke left overs from bootstrapping... */ |
---|
206 | } |
---|