1 | /****************************************************************************** |
---|
2 | * common/grant_table.c |
---|
3 | * |
---|
4 | * Mechanism for granting foreign access to page frames, and receiving |
---|
5 | * page-ownership transfers. |
---|
6 | * |
---|
7 | * Copyright (c) 2005-2006 Christopher Clark |
---|
8 | * Copyright (c) 2004 K A Fraser |
---|
9 | * Copyright (c) 2005 Andrew Warfield |
---|
10 | * Modifications by Geoffrey Lefebvre are (c) Intel Research Cambridge |
---|
11 | * |
---|
12 | * This program is free software; you can redistribute it and/or modify |
---|
13 | * it under the terms of the GNU General Public License as published by |
---|
14 | * the Free Software Foundation; either version 2 of the License, or |
---|
15 | * (at your option) any later version. |
---|
16 | * |
---|
17 | * This program is distributed in the hope that it will be useful, |
---|
18 | * but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
19 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
20 | * GNU General Public License for more details. |
---|
21 | * |
---|
22 | * You should have received a copy of the GNU General Public License |
---|
23 | * along with this program; if not, write to the Free Software |
---|
24 | * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA |
---|
25 | */ |
---|
26 | |
---|
27 | #include <xen/config.h> |
---|
28 | #include <xen/iocap.h> |
---|
29 | #include <xen/lib.h> |
---|
30 | #include <xen/sched.h> |
---|
31 | #include <xen/shadow.h> |
---|
32 | #include <xen/mm.h> |
---|
33 | #include <xen/trace.h> |
---|
34 | #include <xen/guest_access.h> |
---|
35 | #include <xen/domain_page.h> |
---|
36 | #include <acm/acm_hooks.h> |
---|
37 | |
---|
38 | #ifndef max_nr_grant_frames |
---|
39 | unsigned int max_nr_grant_frames = DEFAULT_MAX_NR_GRANT_FRAMES; |
---|
40 | integer_param("gnttab_max_nr_frames", max_nr_grant_frames); |
---|
41 | #endif |
---|
42 | |
---|
43 | /* The maximum number of grant mappings is defined as a multiplier of the |
---|
44 | * maximum number of grant table entries. This defines the multiplier used. |
---|
45 | * Pretty arbitrary. [POLICY] |
---|
46 | */ |
---|
47 | #define MAX_MAPTRACK_TO_GRANTS_RATIO 8 |
---|
48 | |
---|
49 | /* |
---|
50 | * The first two members of a grant entry are updated as a combined pair. |
---|
51 | * The following union allows that to happen in an endian-neutral fashion. |
---|
52 | */ |
---|
53 | union grant_combo { |
---|
54 | uint32_t word; |
---|
55 | struct { |
---|
56 | uint16_t flags; |
---|
57 | domid_t domid; |
---|
58 | } shorts; |
---|
59 | }; |
---|
60 | |
---|
61 | #define PIN_FAIL(_lbl, _rc, _f, _a...) \ |
---|
62 | do { \ |
---|
63 | gdprintk(XENLOG_WARNING, _f, ## _a ); \ |
---|
64 | rc = (_rc); \ |
---|
65 | goto _lbl; \ |
---|
66 | } while ( 0 ) |
---|
67 | |
---|
68 | #define MAPTRACK_PER_PAGE (PAGE_SIZE / sizeof(struct grant_mapping)) |
---|
69 | #define maptrack_entry(t, e) \ |
---|
70 | ((t)->maptrack[(e)/MAPTRACK_PER_PAGE][(e)%MAPTRACK_PER_PAGE]) |
---|
71 | |
---|
72 | static inline unsigned int |
---|
73 | nr_maptrack_frames(struct grant_table *t) |
---|
74 | { |
---|
75 | return t->maptrack_limit / MAPTRACK_PER_PAGE; |
---|
76 | } |
---|
77 | |
---|
78 | static unsigned inline int max_nr_maptrack_frames(void) |
---|
79 | { |
---|
80 | return (max_nr_grant_frames * MAX_MAPTRACK_TO_GRANTS_RATIO); |
---|
81 | } |
---|
82 | |
---|
83 | |
---|
84 | #define SHGNT_PER_PAGE (PAGE_SIZE / sizeof(grant_entry_t)) |
---|
85 | #define shared_entry(t, e) \ |
---|
86 | ((t)->shared[(e)/SHGNT_PER_PAGE][(e)%SHGNT_PER_PAGE]) |
---|
87 | #define ACGNT_PER_PAGE (PAGE_SIZE / sizeof(struct active_grant_entry)) |
---|
88 | #define active_entry(t, e) \ |
---|
89 | ((t)->active[(e)/ACGNT_PER_PAGE][(e)%ACGNT_PER_PAGE]) |
---|
90 | |
---|
91 | static inline int |
---|
92 | __get_maptrack_handle( |
---|
93 | struct grant_table *t) |
---|
94 | { |
---|
95 | unsigned int h; |
---|
96 | if ( unlikely((h = t->maptrack_head) == (t->maptrack_limit - 1)) ) |
---|
97 | return -1; |
---|
98 | t->maptrack_head = maptrack_entry(t, h).ref; |
---|
99 | t->map_count++; |
---|
100 | return h; |
---|
101 | } |
---|
102 | |
---|
103 | static inline void |
---|
104 | put_maptrack_handle( |
---|
105 | struct grant_table *t, int handle) |
---|
106 | { |
---|
107 | maptrack_entry(t, handle).ref = t->maptrack_head; |
---|
108 | t->maptrack_head = handle; |
---|
109 | t->map_count--; |
---|
110 | } |
---|
111 | |
---|
112 | static inline int |
---|
113 | get_maptrack_handle( |
---|
114 | struct grant_table *lgt) |
---|
115 | { |
---|
116 | int i; |
---|
117 | grant_handle_t handle; |
---|
118 | struct grant_mapping *new_mt; |
---|
119 | unsigned int new_mt_limit, nr_frames; |
---|
120 | |
---|
121 | if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) ) |
---|
122 | { |
---|
123 | spin_lock(&lgt->lock); |
---|
124 | |
---|
125 | if ( unlikely((handle = __get_maptrack_handle(lgt)) == -1) ) |
---|
126 | { |
---|
127 | nr_frames = nr_maptrack_frames(lgt); |
---|
128 | if ( nr_frames >= max_nr_maptrack_frames() ) |
---|
129 | { |
---|
130 | spin_unlock(&lgt->lock); |
---|
131 | return -1; |
---|
132 | } |
---|
133 | |
---|
134 | new_mt = alloc_xenheap_page(); |
---|
135 | if ( new_mt == NULL ) |
---|
136 | { |
---|
137 | spin_unlock(&lgt->lock); |
---|
138 | return -1; |
---|
139 | } |
---|
140 | |
---|
141 | memset(new_mt, 0, PAGE_SIZE); |
---|
142 | |
---|
143 | new_mt_limit = lgt->maptrack_limit + MAPTRACK_PER_PAGE; |
---|
144 | |
---|
145 | for ( i = lgt->maptrack_limit; i < new_mt_limit; i++ ) |
---|
146 | { |
---|
147 | new_mt[i % MAPTRACK_PER_PAGE].ref = i+1; |
---|
148 | new_mt[i % MAPTRACK_PER_PAGE].flags = 0; |
---|
149 | } |
---|
150 | |
---|
151 | lgt->maptrack[nr_frames] = new_mt; |
---|
152 | lgt->maptrack_limit = new_mt_limit; |
---|
153 | |
---|
154 | gdprintk(XENLOG_INFO, |
---|
155 | "Increased maptrack size to %u frames.\n", nr_frames + 1); |
---|
156 | handle = __get_maptrack_handle(lgt); |
---|
157 | } |
---|
158 | |
---|
159 | spin_unlock(&lgt->lock); |
---|
160 | } |
---|
161 | return handle; |
---|
162 | } |
---|
163 | |
---|
164 | /* |
---|
165 | * Returns 0 if TLB flush / invalidate required by caller. |
---|
166 | * va will indicate the address to be invalidated. |
---|
167 | * |
---|
168 | * addr is _either_ a host virtual address, or the address of the pte to |
---|
169 | * update, as indicated by the GNTMAP_contains_pte flag. |
---|
170 | */ |
---|
171 | static void |
---|
172 | __gnttab_map_grant_ref( |
---|
173 | struct gnttab_map_grant_ref *op) |
---|
174 | { |
---|
175 | struct domain *ld, *rd; |
---|
176 | struct vcpu *led; |
---|
177 | int handle; |
---|
178 | unsigned long frame = 0; |
---|
179 | int rc = GNTST_okay; |
---|
180 | struct active_grant_entry *act; |
---|
181 | struct grant_mapping *mt; |
---|
182 | grant_entry_t *sha; |
---|
183 | union grant_combo scombo, prev_scombo, new_scombo; |
---|
184 | |
---|
185 | /* |
---|
186 | * We bound the number of times we retry CMPXCHG on memory locations that |
---|
187 | * we share with a guest OS. The reason is that the guest can modify that |
---|
188 | * location at a higher rate than we can read-modify-CMPXCHG, so the guest |
---|
189 | * could cause us to livelock. There are a few cases where it is valid for |
---|
190 | * the guest to race our updates (e.g., to change the GTF_readonly flag), |
---|
191 | * so we allow a few retries before failing. |
---|
192 | */ |
---|
193 | int retries = 0; |
---|
194 | |
---|
195 | led = current; |
---|
196 | ld = led->domain; |
---|
197 | |
---|
198 | if ( unlikely((op->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0) ) |
---|
199 | { |
---|
200 | gdprintk(XENLOG_INFO, "Bad flags in grant map op (%x).\n", op->flags); |
---|
201 | op->status = GNTST_bad_gntref; |
---|
202 | return; |
---|
203 | } |
---|
204 | |
---|
205 | if ( acm_pre_grant_map_ref(op->dom) ) |
---|
206 | { |
---|
207 | op->status = GNTST_permission_denied; |
---|
208 | return; |
---|
209 | } |
---|
210 | |
---|
211 | if ( unlikely((rd = rcu_lock_domain_by_id(op->dom)) == NULL) ) |
---|
212 | { |
---|
213 | gdprintk(XENLOG_INFO, "Could not find domain %d\n", op->dom); |
---|
214 | op->status = GNTST_bad_domain; |
---|
215 | return; |
---|
216 | } |
---|
217 | |
---|
218 | if ( unlikely((handle = get_maptrack_handle(ld->grant_table)) == -1) ) |
---|
219 | { |
---|
220 | rcu_unlock_domain(rd); |
---|
221 | gdprintk(XENLOG_INFO, "Failed to obtain maptrack handle.\n"); |
---|
222 | op->status = GNTST_no_device_space; |
---|
223 | return; |
---|
224 | } |
---|
225 | |
---|
226 | spin_lock(&rd->grant_table->lock); |
---|
227 | |
---|
228 | /* Bounds check on the grant ref */ |
---|
229 | if ( unlikely(op->ref >= nr_grant_entries(rd->grant_table))) |
---|
230 | PIN_FAIL(unlock_out, GNTST_bad_gntref, "Bad ref (%d).\n", op->ref); |
---|
231 | |
---|
232 | act = &active_entry(rd->grant_table, op->ref); |
---|
233 | sha = &shared_entry(rd->grant_table, op->ref); |
---|
234 | |
---|
235 | /* If already pinned, check the active domid and avoid refcnt overflow. */ |
---|
236 | if ( act->pin && |
---|
237 | ((act->domid != ld->domain_id) || |
---|
238 | (act->pin & 0x80808080U) != 0) ) |
---|
239 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
240 | "Bad domain (%d != %d), or risk of counter overflow %08x\n", |
---|
241 | act->domid, ld->domain_id, act->pin); |
---|
242 | |
---|
243 | if ( !act->pin || |
---|
244 | (!(op->flags & GNTMAP_readonly) && |
---|
245 | !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask))) ) |
---|
246 | { |
---|
247 | scombo.word = *(u32 *)&sha->flags; |
---|
248 | |
---|
249 | /* |
---|
250 | * This loop attempts to set the access (reading/writing) flags |
---|
251 | * in the grant table entry. It tries a cmpxchg on the field |
---|
252 | * up to five times, and then fails under the assumption that |
---|
253 | * the guest is misbehaving. |
---|
254 | */ |
---|
255 | for ( ; ; ) |
---|
256 | { |
---|
257 | /* If not already pinned, check the grant domid and type. */ |
---|
258 | if ( !act->pin && |
---|
259 | (((scombo.shorts.flags & GTF_type_mask) != |
---|
260 | GTF_permit_access) || |
---|
261 | (scombo.shorts.domid != ld->domain_id)) ) |
---|
262 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
263 | "Bad flags (%x) or dom (%d). (expected dom %d)\n", |
---|
264 | scombo.shorts.flags, scombo.shorts.domid, |
---|
265 | ld->domain_id); |
---|
266 | |
---|
267 | new_scombo = scombo; |
---|
268 | new_scombo.shorts.flags |= GTF_reading; |
---|
269 | |
---|
270 | if ( !(op->flags & GNTMAP_readonly) ) |
---|
271 | { |
---|
272 | new_scombo.shorts.flags |= GTF_writing; |
---|
273 | if ( unlikely(scombo.shorts.flags & GTF_readonly) ) |
---|
274 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
275 | "Attempt to write-pin a r/o grant entry.\n"); |
---|
276 | } |
---|
277 | |
---|
278 | prev_scombo.word = cmpxchg((u32 *)&sha->flags, |
---|
279 | scombo.word, new_scombo.word); |
---|
280 | if ( likely(prev_scombo.word == scombo.word) ) |
---|
281 | break; |
---|
282 | |
---|
283 | if ( retries++ == 4 ) |
---|
284 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
285 | "Shared grant entry is unstable.\n"); |
---|
286 | |
---|
287 | scombo = prev_scombo; |
---|
288 | } |
---|
289 | |
---|
290 | if ( !act->pin ) |
---|
291 | { |
---|
292 | act->domid = scombo.shorts.domid; |
---|
293 | act->frame = gmfn_to_mfn(rd, sha->frame); |
---|
294 | } |
---|
295 | } |
---|
296 | |
---|
297 | if ( op->flags & GNTMAP_device_map ) |
---|
298 | act->pin += (op->flags & GNTMAP_readonly) ? |
---|
299 | GNTPIN_devr_inc : GNTPIN_devw_inc; |
---|
300 | if ( op->flags & GNTMAP_host_map ) |
---|
301 | act->pin += (op->flags & GNTMAP_readonly) ? |
---|
302 | GNTPIN_hstr_inc : GNTPIN_hstw_inc; |
---|
303 | |
---|
304 | frame = act->frame; |
---|
305 | |
---|
306 | spin_unlock(&rd->grant_table->lock); |
---|
307 | |
---|
308 | if ( unlikely(!mfn_valid(frame)) || |
---|
309 | unlikely(!((op->flags & GNTMAP_readonly) ? |
---|
310 | get_page(mfn_to_page(frame), rd) : |
---|
311 | get_page_and_type(mfn_to_page(frame), rd, |
---|
312 | PGT_writable_page))) ) |
---|
313 | { |
---|
314 | if ( !rd->is_dying ) |
---|
315 | gdprintk(XENLOG_WARNING, "Could not pin grant frame %lx\n", frame); |
---|
316 | rc = GNTST_general_error; |
---|
317 | goto undo_out; |
---|
318 | } |
---|
319 | |
---|
320 | if ( op->flags & GNTMAP_host_map ) |
---|
321 | { |
---|
322 | rc = create_grant_host_mapping(op->host_addr, frame, op->flags); |
---|
323 | if ( rc != GNTST_okay ) |
---|
324 | { |
---|
325 | if ( !(op->flags & GNTMAP_readonly) ) |
---|
326 | put_page_type(mfn_to_page(frame)); |
---|
327 | put_page(mfn_to_page(frame)); |
---|
328 | goto undo_out; |
---|
329 | } |
---|
330 | |
---|
331 | if ( op->flags & GNTMAP_device_map ) |
---|
332 | { |
---|
333 | (void)get_page(mfn_to_page(frame), rd); |
---|
334 | if ( !(op->flags & GNTMAP_readonly) ) |
---|
335 | get_page_type(mfn_to_page(frame), PGT_writable_page); |
---|
336 | } |
---|
337 | } |
---|
338 | |
---|
339 | TRACE_1D(TRC_MEM_PAGE_GRANT_MAP, op->dom); |
---|
340 | |
---|
341 | mt = &maptrack_entry(ld->grant_table, handle); |
---|
342 | mt->domid = op->dom; |
---|
343 | mt->ref = op->ref; |
---|
344 | mt->flags = op->flags; |
---|
345 | |
---|
346 | op->dev_bus_addr = (u64)frame << PAGE_SHIFT; |
---|
347 | op->handle = handle; |
---|
348 | op->status = GNTST_okay; |
---|
349 | |
---|
350 | rcu_unlock_domain(rd); |
---|
351 | return; |
---|
352 | |
---|
353 | undo_out: |
---|
354 | spin_lock(&rd->grant_table->lock); |
---|
355 | |
---|
356 | act = &active_entry(rd->grant_table, op->ref); |
---|
357 | sha = &shared_entry(rd->grant_table, op->ref); |
---|
358 | |
---|
359 | if ( op->flags & GNTMAP_device_map ) |
---|
360 | act->pin -= (op->flags & GNTMAP_readonly) ? |
---|
361 | GNTPIN_devr_inc : GNTPIN_devw_inc; |
---|
362 | if ( op->flags & GNTMAP_host_map ) |
---|
363 | act->pin -= (op->flags & GNTMAP_readonly) ? |
---|
364 | GNTPIN_hstr_inc : GNTPIN_hstw_inc; |
---|
365 | |
---|
366 | if ( !(op->flags & GNTMAP_readonly) && |
---|
367 | !(act->pin & (GNTPIN_hstw_mask|GNTPIN_devw_mask)) ) |
---|
368 | gnttab_clear_flag(_GTF_writing, &sha->flags); |
---|
369 | |
---|
370 | if ( !act->pin ) |
---|
371 | gnttab_clear_flag(_GTF_reading, &sha->flags); |
---|
372 | |
---|
373 | unlock_out: |
---|
374 | spin_unlock(&rd->grant_table->lock); |
---|
375 | op->status = rc; |
---|
376 | put_maptrack_handle(ld->grant_table, handle); |
---|
377 | rcu_unlock_domain(rd); |
---|
378 | } |
---|
379 | |
---|
380 | static long |
---|
381 | gnttab_map_grant_ref( |
---|
382 | XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) uop, unsigned int count) |
---|
383 | { |
---|
384 | int i; |
---|
385 | struct gnttab_map_grant_ref op; |
---|
386 | |
---|
387 | for ( i = 0; i < count; i++ ) |
---|
388 | { |
---|
389 | if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) ) |
---|
390 | return -EFAULT; |
---|
391 | __gnttab_map_grant_ref(&op); |
---|
392 | if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) ) |
---|
393 | return -EFAULT; |
---|
394 | } |
---|
395 | |
---|
396 | return 0; |
---|
397 | } |
---|
398 | |
---|
399 | static void |
---|
400 | __gnttab_unmap_grant_ref( |
---|
401 | struct gnttab_unmap_grant_ref *op) |
---|
402 | { |
---|
403 | domid_t dom; |
---|
404 | grant_ref_t ref; |
---|
405 | struct domain *ld, *rd; |
---|
406 | struct active_grant_entry *act; |
---|
407 | grant_entry_t *sha; |
---|
408 | struct grant_mapping *map; |
---|
409 | u16 flags; |
---|
410 | s16 rc = 0; |
---|
411 | unsigned long frame; |
---|
412 | |
---|
413 | ld = current->domain; |
---|
414 | |
---|
415 | frame = (unsigned long)(op->dev_bus_addr >> PAGE_SHIFT); |
---|
416 | |
---|
417 | if ( unlikely(op->handle >= ld->grant_table->maptrack_limit) ) |
---|
418 | { |
---|
419 | gdprintk(XENLOG_INFO, "Bad handle (%d).\n", op->handle); |
---|
420 | op->status = GNTST_bad_handle; |
---|
421 | return; |
---|
422 | } |
---|
423 | |
---|
424 | map = &maptrack_entry(ld->grant_table, op->handle); |
---|
425 | |
---|
426 | if ( unlikely(!map->flags) ) |
---|
427 | { |
---|
428 | gdprintk(XENLOG_INFO, "Zero flags for handle (%d).\n", op->handle); |
---|
429 | op->status = GNTST_bad_handle; |
---|
430 | return; |
---|
431 | } |
---|
432 | |
---|
433 | dom = map->domid; |
---|
434 | ref = map->ref; |
---|
435 | flags = map->flags; |
---|
436 | |
---|
437 | if ( unlikely((rd = rcu_lock_domain_by_id(dom)) == NULL) ) |
---|
438 | { |
---|
439 | /* This can happen when a grant is implicitly unmapped. */ |
---|
440 | gdprintk(XENLOG_INFO, "Could not find domain %d\n", dom); |
---|
441 | domain_crash(ld); /* naughty... */ |
---|
442 | return; |
---|
443 | } |
---|
444 | |
---|
445 | TRACE_1D(TRC_MEM_PAGE_GRANT_UNMAP, dom); |
---|
446 | |
---|
447 | spin_lock(&rd->grant_table->lock); |
---|
448 | |
---|
449 | act = &active_entry(rd->grant_table, ref); |
---|
450 | sha = &shared_entry(rd->grant_table, ref); |
---|
451 | |
---|
452 | if ( frame == 0 ) |
---|
453 | { |
---|
454 | frame = act->frame; |
---|
455 | } |
---|
456 | else |
---|
457 | { |
---|
458 | if ( unlikely(frame != act->frame) ) |
---|
459 | PIN_FAIL(unmap_out, GNTST_general_error, |
---|
460 | "Bad frame number doesn't match gntref.\n"); |
---|
461 | if ( flags & GNTMAP_device_map ) |
---|
462 | { |
---|
463 | ASSERT(act->pin & (GNTPIN_devw_mask | GNTPIN_devr_mask)); |
---|
464 | map->flags &= ~GNTMAP_device_map; |
---|
465 | if ( flags & GNTMAP_readonly ) |
---|
466 | { |
---|
467 | act->pin -= GNTPIN_devr_inc; |
---|
468 | put_page(mfn_to_page(frame)); |
---|
469 | } |
---|
470 | else |
---|
471 | { |
---|
472 | act->pin -= GNTPIN_devw_inc; |
---|
473 | put_page_and_type(mfn_to_page(frame)); |
---|
474 | } |
---|
475 | } |
---|
476 | } |
---|
477 | |
---|
478 | if ( (op->host_addr != 0) && (flags & GNTMAP_host_map) ) |
---|
479 | { |
---|
480 | if ( (rc = destroy_grant_host_mapping(op->host_addr, |
---|
481 | frame, flags)) < 0 ) |
---|
482 | goto unmap_out; |
---|
483 | |
---|
484 | ASSERT(act->pin & (GNTPIN_hstw_mask | GNTPIN_hstr_mask)); |
---|
485 | map->flags &= ~GNTMAP_host_map; |
---|
486 | if ( flags & GNTMAP_readonly ) |
---|
487 | { |
---|
488 | act->pin -= GNTPIN_hstr_inc; |
---|
489 | put_page(mfn_to_page(frame)); |
---|
490 | } |
---|
491 | else |
---|
492 | { |
---|
493 | act->pin -= GNTPIN_hstw_inc; |
---|
494 | put_page_and_type(mfn_to_page(frame)); |
---|
495 | } |
---|
496 | } |
---|
497 | |
---|
498 | if ( (map->flags & (GNTMAP_device_map|GNTMAP_host_map)) == 0 ) |
---|
499 | { |
---|
500 | map->flags = 0; |
---|
501 | put_maptrack_handle(ld->grant_table, op->handle); |
---|
502 | } |
---|
503 | |
---|
504 | /* If just unmapped a writable mapping, mark as dirtied */ |
---|
505 | if ( !(flags & GNTMAP_readonly) ) |
---|
506 | gnttab_mark_dirty(rd, frame); |
---|
507 | |
---|
508 | if ( ((act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0) && |
---|
509 | !(flags & GNTMAP_readonly) ) |
---|
510 | gnttab_clear_flag(_GTF_writing, &sha->flags); |
---|
511 | |
---|
512 | if ( act->pin == 0 ) |
---|
513 | gnttab_clear_flag(_GTF_reading, &sha->flags); |
---|
514 | |
---|
515 | unmap_out: |
---|
516 | op->status = rc; |
---|
517 | spin_unlock(&rd->grant_table->lock); |
---|
518 | rcu_unlock_domain(rd); |
---|
519 | } |
---|
520 | |
---|
521 | static long |
---|
522 | gnttab_unmap_grant_ref( |
---|
523 | XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) uop, unsigned int count) |
---|
524 | { |
---|
525 | int i; |
---|
526 | struct gnttab_unmap_grant_ref op; |
---|
527 | |
---|
528 | for ( i = 0; i < count; i++ ) |
---|
529 | { |
---|
530 | if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) ) |
---|
531 | goto fault; |
---|
532 | __gnttab_unmap_grant_ref(&op); |
---|
533 | if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) ) |
---|
534 | goto fault; |
---|
535 | } |
---|
536 | |
---|
537 | flush_tlb_mask(current->domain->domain_dirty_cpumask); |
---|
538 | return 0; |
---|
539 | |
---|
540 | fault: |
---|
541 | flush_tlb_mask(current->domain->domain_dirty_cpumask); |
---|
542 | return -EFAULT; |
---|
543 | } |
---|
544 | |
---|
545 | int |
---|
546 | gnttab_grow_table(struct domain *d, unsigned int req_nr_frames) |
---|
547 | { |
---|
548 | /* d's grant table lock must be held by the caller */ |
---|
549 | |
---|
550 | struct grant_table *gt = d->grant_table; |
---|
551 | unsigned int i; |
---|
552 | |
---|
553 | ASSERT(req_nr_frames <= max_nr_grant_frames); |
---|
554 | |
---|
555 | gdprintk(XENLOG_INFO, |
---|
556 | "Expanding dom (%d) grant table from (%d) to (%d) frames.\n", |
---|
557 | d->domain_id, nr_grant_frames(gt), req_nr_frames); |
---|
558 | |
---|
559 | /* Active */ |
---|
560 | for ( i = nr_active_grant_frames(gt); |
---|
561 | i < num_act_frames_from_sha_frames(req_nr_frames); i++ ) |
---|
562 | { |
---|
563 | if ( (gt->active[i] = alloc_xenheap_page()) == NULL ) |
---|
564 | goto active_alloc_failed; |
---|
565 | memset(gt->active[i], 0, PAGE_SIZE); |
---|
566 | } |
---|
567 | |
---|
568 | /* Shared */ |
---|
569 | for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ ) |
---|
570 | { |
---|
571 | if ( (gt->shared[i] = alloc_xenheap_page()) == NULL ) |
---|
572 | goto shared_alloc_failed; |
---|
573 | memset(gt->shared[i], 0, PAGE_SIZE); |
---|
574 | } |
---|
575 | |
---|
576 | /* Share the new shared frames with the recipient domain */ |
---|
577 | for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ ) |
---|
578 | gnttab_create_shared_page(d, gt, i); |
---|
579 | |
---|
580 | gt->nr_grant_frames = req_nr_frames; |
---|
581 | |
---|
582 | return 1; |
---|
583 | |
---|
584 | shared_alloc_failed: |
---|
585 | for ( i = nr_grant_frames(gt); i < req_nr_frames; i++ ) |
---|
586 | { |
---|
587 | free_xenheap_page(gt->shared[i]); |
---|
588 | gt->shared[i] = NULL; |
---|
589 | } |
---|
590 | active_alloc_failed: |
---|
591 | for ( i = nr_active_grant_frames(gt); |
---|
592 | i < num_act_frames_from_sha_frames(req_nr_frames); i++ ) |
---|
593 | { |
---|
594 | free_xenheap_page(gt->active[i]); |
---|
595 | gt->active[i] = NULL; |
---|
596 | } |
---|
597 | gdprintk(XENLOG_INFO, "Allocation failure when expanding grant table.\n"); |
---|
598 | return 0; |
---|
599 | } |
---|
600 | |
---|
601 | static long |
---|
602 | gnttab_setup_table( |
---|
603 | XEN_GUEST_HANDLE(gnttab_setup_table_t) uop, unsigned int count) |
---|
604 | { |
---|
605 | struct gnttab_setup_table op; |
---|
606 | struct domain *d; |
---|
607 | int i; |
---|
608 | unsigned long gmfn; |
---|
609 | domid_t dom; |
---|
610 | |
---|
611 | if ( count != 1 ) |
---|
612 | return -EINVAL; |
---|
613 | |
---|
614 | if ( unlikely(copy_from_guest(&op, uop, 1) != 0) ) |
---|
615 | { |
---|
616 | gdprintk(XENLOG_INFO, "Fault while reading gnttab_setup_table_t.\n"); |
---|
617 | return -EFAULT; |
---|
618 | } |
---|
619 | |
---|
620 | if ( unlikely(op.nr_frames > max_nr_grant_frames) ) |
---|
621 | { |
---|
622 | gdprintk(XENLOG_INFO, "Xen only supports up to %d grant-table frames" |
---|
623 | " per domain.\n", |
---|
624 | max_nr_grant_frames); |
---|
625 | op.status = GNTST_general_error; |
---|
626 | goto out; |
---|
627 | } |
---|
628 | |
---|
629 | dom = op.dom; |
---|
630 | if ( dom == DOMID_SELF ) |
---|
631 | { |
---|
632 | dom = current->domain->domain_id; |
---|
633 | } |
---|
634 | else if ( unlikely(!IS_PRIV(current->domain)) ) |
---|
635 | { |
---|
636 | op.status = GNTST_permission_denied; |
---|
637 | goto out; |
---|
638 | } |
---|
639 | |
---|
640 | if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) ) |
---|
641 | { |
---|
642 | gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom); |
---|
643 | op.status = GNTST_bad_domain; |
---|
644 | goto out; |
---|
645 | } |
---|
646 | |
---|
647 | spin_lock(&d->grant_table->lock); |
---|
648 | |
---|
649 | if ( (op.nr_frames > nr_grant_frames(d->grant_table)) && |
---|
650 | !gnttab_grow_table(d, op.nr_frames) ) |
---|
651 | { |
---|
652 | gdprintk(XENLOG_INFO, |
---|
653 | "Expand grant table to %d failed. Current: %d Max: %d.\n", |
---|
654 | op.nr_frames, |
---|
655 | nr_grant_frames(d->grant_table), |
---|
656 | max_nr_grant_frames); |
---|
657 | op.status = GNTST_general_error; |
---|
658 | goto setup_unlock_out; |
---|
659 | } |
---|
660 | |
---|
661 | op.status = GNTST_okay; |
---|
662 | for ( i = 0; i < op.nr_frames; i++ ) |
---|
663 | { |
---|
664 | gmfn = gnttab_shared_gmfn(d, d->grant_table, i); |
---|
665 | (void)copy_to_guest_offset(op.frame_list, i, &gmfn, 1); |
---|
666 | } |
---|
667 | |
---|
668 | setup_unlock_out: |
---|
669 | spin_unlock(&d->grant_table->lock); |
---|
670 | |
---|
671 | rcu_unlock_domain(d); |
---|
672 | |
---|
673 | out: |
---|
674 | if ( unlikely(copy_to_guest(uop, &op, 1)) ) |
---|
675 | return -EFAULT; |
---|
676 | |
---|
677 | return 0; |
---|
678 | } |
---|
679 | |
---|
680 | static long |
---|
681 | gnttab_query_size( |
---|
682 | XEN_GUEST_HANDLE(gnttab_query_size_t) uop, unsigned int count) |
---|
683 | { |
---|
684 | struct gnttab_query_size op; |
---|
685 | struct domain *d; |
---|
686 | domid_t dom; |
---|
687 | |
---|
688 | if ( count != 1 ) |
---|
689 | return -EINVAL; |
---|
690 | |
---|
691 | if ( unlikely(copy_from_guest(&op, uop, 1) != 0) ) |
---|
692 | { |
---|
693 | gdprintk(XENLOG_INFO, "Fault while reading gnttab_query_size_t.\n"); |
---|
694 | return -EFAULT; |
---|
695 | } |
---|
696 | |
---|
697 | dom = op.dom; |
---|
698 | if ( dom == DOMID_SELF ) |
---|
699 | { |
---|
700 | dom = current->domain->domain_id; |
---|
701 | } |
---|
702 | else if ( unlikely(!IS_PRIV(current->domain)) ) |
---|
703 | { |
---|
704 | op.status = GNTST_permission_denied; |
---|
705 | goto query_out; |
---|
706 | } |
---|
707 | |
---|
708 | if ( unlikely((d = rcu_lock_domain_by_id(dom)) == NULL) ) |
---|
709 | { |
---|
710 | gdprintk(XENLOG_INFO, "Bad domid %d.\n", dom); |
---|
711 | op.status = GNTST_bad_domain; |
---|
712 | goto query_out; |
---|
713 | } |
---|
714 | |
---|
715 | spin_lock(&d->grant_table->lock); |
---|
716 | |
---|
717 | op.nr_frames = nr_grant_frames(d->grant_table); |
---|
718 | op.max_nr_frames = max_nr_grant_frames; |
---|
719 | op.status = GNTST_okay; |
---|
720 | |
---|
721 | spin_unlock(&d->grant_table->lock); |
---|
722 | |
---|
723 | rcu_unlock_domain(d); |
---|
724 | |
---|
725 | query_out: |
---|
726 | if ( unlikely(copy_to_guest(uop, &op, 1)) ) |
---|
727 | return -EFAULT; |
---|
728 | |
---|
729 | return 0; |
---|
730 | } |
---|
731 | |
---|
732 | /* |
---|
733 | * Check that the given grant reference (rd,ref) allows 'ld' to transfer |
---|
734 | * ownership of a page frame. If so, lock down the grant entry. |
---|
735 | */ |
---|
736 | static int |
---|
737 | gnttab_prepare_for_transfer( |
---|
738 | struct domain *rd, struct domain *ld, grant_ref_t ref) |
---|
739 | { |
---|
740 | struct grant_table *rgt; |
---|
741 | struct grant_entry *sha; |
---|
742 | union grant_combo scombo, prev_scombo, new_scombo; |
---|
743 | int retries = 0; |
---|
744 | |
---|
745 | if ( unlikely((rgt = rd->grant_table) == NULL) ) |
---|
746 | { |
---|
747 | gdprintk(XENLOG_INFO, "Dom %d has no grant table.\n", rd->domain_id); |
---|
748 | return 0; |
---|
749 | } |
---|
750 | |
---|
751 | spin_lock(&rgt->lock); |
---|
752 | |
---|
753 | if ( unlikely(ref >= nr_grant_entries(rd->grant_table)) ) |
---|
754 | { |
---|
755 | gdprintk(XENLOG_INFO, |
---|
756 | "Bad grant reference (%d) for transfer to domain(%d).\n", |
---|
757 | ref, rd->domain_id); |
---|
758 | goto fail; |
---|
759 | } |
---|
760 | |
---|
761 | sha = &shared_entry(rgt, ref); |
---|
762 | |
---|
763 | scombo.word = *(u32 *)&sha->flags; |
---|
764 | |
---|
765 | for ( ; ; ) |
---|
766 | { |
---|
767 | if ( unlikely(scombo.shorts.flags != GTF_accept_transfer) || |
---|
768 | unlikely(scombo.shorts.domid != ld->domain_id) ) |
---|
769 | { |
---|
770 | gdprintk(XENLOG_INFO, "Bad flags (%x) or dom (%d). " |
---|
771 | "(NB. expected dom %d)\n", |
---|
772 | scombo.shorts.flags, scombo.shorts.domid, |
---|
773 | ld->domain_id); |
---|
774 | goto fail; |
---|
775 | } |
---|
776 | |
---|
777 | new_scombo = scombo; |
---|
778 | new_scombo.shorts.flags |= GTF_transfer_committed; |
---|
779 | |
---|
780 | prev_scombo.word = cmpxchg((u32 *)&sha->flags, |
---|
781 | scombo.word, new_scombo.word); |
---|
782 | if ( likely(prev_scombo.word == scombo.word) ) |
---|
783 | break; |
---|
784 | |
---|
785 | if ( retries++ == 4 ) |
---|
786 | { |
---|
787 | gdprintk(XENLOG_WARNING, "Shared grant entry is unstable.\n"); |
---|
788 | goto fail; |
---|
789 | } |
---|
790 | |
---|
791 | scombo = prev_scombo; |
---|
792 | } |
---|
793 | |
---|
794 | spin_unlock(&rgt->lock); |
---|
795 | return 1; |
---|
796 | |
---|
797 | fail: |
---|
798 | spin_unlock(&rgt->lock); |
---|
799 | return 0; |
---|
800 | } |
---|
801 | |
---|
802 | static long |
---|
803 | gnttab_transfer( |
---|
804 | XEN_GUEST_HANDLE(gnttab_transfer_t) uop, unsigned int count) |
---|
805 | { |
---|
806 | struct domain *d = current->domain; |
---|
807 | struct domain *e; |
---|
808 | struct page_info *page; |
---|
809 | int i; |
---|
810 | grant_entry_t *sha; |
---|
811 | struct gnttab_transfer gop; |
---|
812 | unsigned long mfn; |
---|
813 | |
---|
814 | for ( i = 0; i < count; i++ ) |
---|
815 | { |
---|
816 | /* Read from caller address space. */ |
---|
817 | if ( unlikely(__copy_from_guest_offset(&gop, uop, i, 1)) ) |
---|
818 | { |
---|
819 | gdprintk(XENLOG_INFO, "gnttab_transfer: error reading req %d/%d\n", |
---|
820 | i, count); |
---|
821 | return -EFAULT; |
---|
822 | } |
---|
823 | |
---|
824 | mfn = gmfn_to_mfn(d, gop.mfn); |
---|
825 | |
---|
826 | /* Check the passed page frame for basic validity. */ |
---|
827 | if ( unlikely(!mfn_valid(mfn)) ) |
---|
828 | { |
---|
829 | gdprintk(XENLOG_INFO, "gnttab_transfer: out-of-range %lx\n", |
---|
830 | (unsigned long)gop.mfn); |
---|
831 | gop.status = GNTST_bad_page; |
---|
832 | goto copyback; |
---|
833 | } |
---|
834 | |
---|
835 | page = mfn_to_page(mfn); |
---|
836 | if ( unlikely(IS_XEN_HEAP_FRAME(page)) ) |
---|
837 | { |
---|
838 | gdprintk(XENLOG_INFO, "gnttab_transfer: xen frame %lx\n", |
---|
839 | (unsigned long)gop.mfn); |
---|
840 | gop.status = GNTST_bad_page; |
---|
841 | goto copyback; |
---|
842 | } |
---|
843 | |
---|
844 | if ( steal_page(d, page, 0) < 0 ) |
---|
845 | { |
---|
846 | gop.status = GNTST_bad_page; |
---|
847 | goto copyback; |
---|
848 | } |
---|
849 | |
---|
850 | /* Find the target domain. */ |
---|
851 | if ( unlikely((e = rcu_lock_domain_by_id(gop.domid)) == NULL) ) |
---|
852 | { |
---|
853 | gdprintk(XENLOG_INFO, "gnttab_transfer: can't find domain %d\n", |
---|
854 | gop.domid); |
---|
855 | page->count_info &= ~(PGC_count_mask|PGC_allocated); |
---|
856 | free_domheap_page(page); |
---|
857 | gop.status = GNTST_bad_domain; |
---|
858 | goto copyback; |
---|
859 | } |
---|
860 | |
---|
861 | spin_lock(&e->page_alloc_lock); |
---|
862 | |
---|
863 | /* |
---|
864 | * Check that 'e' will accept the page and has reservation |
---|
865 | * headroom. Also, a domain mustn't have PGC_allocated |
---|
866 | * pages when it is dying. |
---|
867 | */ |
---|
868 | if ( unlikely(e->is_dying) || |
---|
869 | unlikely(e->tot_pages >= e->max_pages) || |
---|
870 | unlikely(!gnttab_prepare_for_transfer(e, d, gop.ref)) ) |
---|
871 | { |
---|
872 | if ( !e->is_dying ) |
---|
873 | gdprintk(XENLOG_INFO, "gnttab_transfer: " |
---|
874 | "Transferee has no reservation " |
---|
875 | "headroom (%d,%d) or provided a bad grant ref (%08x) " |
---|
876 | "or is dying (%d)\n", |
---|
877 | e->tot_pages, e->max_pages, gop.ref, e->is_dying); |
---|
878 | spin_unlock(&e->page_alloc_lock); |
---|
879 | rcu_unlock_domain(e); |
---|
880 | page->count_info &= ~(PGC_count_mask|PGC_allocated); |
---|
881 | free_domheap_page(page); |
---|
882 | gop.status = GNTST_general_error; |
---|
883 | goto copyback; |
---|
884 | } |
---|
885 | |
---|
886 | /* Okay, add the page to 'e'. */ |
---|
887 | if ( unlikely(e->tot_pages++ == 0) ) |
---|
888 | get_knownalive_domain(e); |
---|
889 | list_add_tail(&page->list, &e->page_list); |
---|
890 | page_set_owner(page, e); |
---|
891 | |
---|
892 | spin_unlock(&e->page_alloc_lock); |
---|
893 | |
---|
894 | TRACE_1D(TRC_MEM_PAGE_GRANT_TRANSFER, e->domain_id); |
---|
895 | |
---|
896 | /* Tell the guest about its new page frame. */ |
---|
897 | spin_lock(&e->grant_table->lock); |
---|
898 | |
---|
899 | sha = &shared_entry(e->grant_table, gop.ref); |
---|
900 | guest_physmap_add_page(e, sha->frame, mfn); |
---|
901 | sha->frame = mfn; |
---|
902 | wmb(); |
---|
903 | sha->flags |= GTF_transfer_completed; |
---|
904 | |
---|
905 | spin_unlock(&e->grant_table->lock); |
---|
906 | |
---|
907 | rcu_unlock_domain(e); |
---|
908 | |
---|
909 | gop.status = GNTST_okay; |
---|
910 | |
---|
911 | copyback: |
---|
912 | if ( unlikely(__copy_to_guest_offset(uop, i, &gop, 1)) ) |
---|
913 | { |
---|
914 | gdprintk(XENLOG_INFO, "gnttab_transfer: error writing resp " |
---|
915 | "%d/%d\n", i, count); |
---|
916 | return -EFAULT; |
---|
917 | } |
---|
918 | } |
---|
919 | |
---|
920 | return 0; |
---|
921 | } |
---|
922 | |
---|
923 | /* Undo __acquire_grant_for_copy. Again, this has no effect on page |
---|
924 | type and reference counts. */ |
---|
925 | static void |
---|
926 | __release_grant_for_copy( |
---|
927 | struct domain *rd, unsigned long gref, int readonly) |
---|
928 | { |
---|
929 | grant_entry_t *sha; |
---|
930 | struct active_grant_entry *act; |
---|
931 | unsigned long r_frame; |
---|
932 | |
---|
933 | spin_lock(&rd->grant_table->lock); |
---|
934 | |
---|
935 | act = &active_entry(rd->grant_table, gref); |
---|
936 | sha = &shared_entry(rd->grant_table, gref); |
---|
937 | r_frame = act->frame; |
---|
938 | |
---|
939 | if ( readonly ) |
---|
940 | { |
---|
941 | act->pin -= GNTPIN_hstr_inc; |
---|
942 | } |
---|
943 | else |
---|
944 | { |
---|
945 | gnttab_mark_dirty(rd, r_frame); |
---|
946 | |
---|
947 | act->pin -= GNTPIN_hstw_inc; |
---|
948 | if ( !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) ) |
---|
949 | gnttab_clear_flag(_GTF_writing, &sha->flags); |
---|
950 | } |
---|
951 | |
---|
952 | if ( !act->pin ) |
---|
953 | gnttab_clear_flag(_GTF_reading, &sha->flags); |
---|
954 | |
---|
955 | spin_unlock(&rd->grant_table->lock); |
---|
956 | } |
---|
957 | |
---|
958 | /* Grab a frame number from a grant entry and update the flags and pin |
---|
959 | count as appropriate. Note that this does *not* update the page |
---|
960 | type or reference counts, and does not check that the mfn is |
---|
961 | actually valid. */ |
---|
962 | static int |
---|
963 | __acquire_grant_for_copy( |
---|
964 | struct domain *rd, unsigned long gref, int readonly, |
---|
965 | unsigned long *frame) |
---|
966 | { |
---|
967 | grant_entry_t *sha; |
---|
968 | struct active_grant_entry *act; |
---|
969 | s16 rc = GNTST_okay; |
---|
970 | int retries = 0; |
---|
971 | union grant_combo scombo, prev_scombo, new_scombo; |
---|
972 | |
---|
973 | spin_lock(&rd->grant_table->lock); |
---|
974 | |
---|
975 | if ( unlikely(gref >= nr_grant_entries(rd->grant_table)) ) |
---|
976 | PIN_FAIL(unlock_out, GNTST_bad_gntref, |
---|
977 | "Bad grant reference %ld\n", gref); |
---|
978 | |
---|
979 | act = &active_entry(rd->grant_table, gref); |
---|
980 | sha = &shared_entry(rd->grant_table, gref); |
---|
981 | |
---|
982 | /* If already pinned, check the active domid and avoid refcnt overflow. */ |
---|
983 | if ( act->pin && |
---|
984 | ((act->domid != current->domain->domain_id) || |
---|
985 | (act->pin & 0x80808080U) != 0) ) |
---|
986 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
987 | "Bad domain (%d != %d), or risk of counter overflow %08x\n", |
---|
988 | act->domid, current->domain->domain_id, act->pin); |
---|
989 | |
---|
990 | if ( !act->pin || |
---|
991 | (!readonly && !(act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask))) ) |
---|
992 | { |
---|
993 | scombo.word = *(u32 *)&sha->flags; |
---|
994 | |
---|
995 | for ( ; ; ) |
---|
996 | { |
---|
997 | /* If not already pinned, check the grant domid and type. */ |
---|
998 | if ( !act->pin && |
---|
999 | (((scombo.shorts.flags & GTF_type_mask) != |
---|
1000 | GTF_permit_access) || |
---|
1001 | (scombo.shorts.domid != current->domain->domain_id)) ) |
---|
1002 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
1003 | "Bad flags (%x) or dom (%d). (expected dom %d)\n", |
---|
1004 | scombo.shorts.flags, scombo.shorts.domid, |
---|
1005 | current->domain->domain_id); |
---|
1006 | |
---|
1007 | new_scombo = scombo; |
---|
1008 | new_scombo.shorts.flags |= GTF_reading; |
---|
1009 | |
---|
1010 | if ( !readonly ) |
---|
1011 | { |
---|
1012 | new_scombo.shorts.flags |= GTF_writing; |
---|
1013 | if ( unlikely(scombo.shorts.flags & GTF_readonly) ) |
---|
1014 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
1015 | "Attempt to write-pin a r/o grant entry.\n"); |
---|
1016 | } |
---|
1017 | |
---|
1018 | prev_scombo.word = cmpxchg((u32 *)&sha->flags, |
---|
1019 | scombo.word, new_scombo.word); |
---|
1020 | if ( likely(prev_scombo.word == scombo.word) ) |
---|
1021 | break; |
---|
1022 | |
---|
1023 | if ( retries++ == 4 ) |
---|
1024 | PIN_FAIL(unlock_out, GNTST_general_error, |
---|
1025 | "Shared grant entry is unstable.\n"); |
---|
1026 | |
---|
1027 | scombo = prev_scombo; |
---|
1028 | } |
---|
1029 | |
---|
1030 | if ( !act->pin ) |
---|
1031 | { |
---|
1032 | act->domid = scombo.shorts.domid; |
---|
1033 | act->frame = gmfn_to_mfn(rd, sha->frame); |
---|
1034 | } |
---|
1035 | } |
---|
1036 | |
---|
1037 | act->pin += readonly ? GNTPIN_hstr_inc : GNTPIN_hstw_inc; |
---|
1038 | |
---|
1039 | *frame = act->frame; |
---|
1040 | |
---|
1041 | unlock_out: |
---|
1042 | spin_unlock(&rd->grant_table->lock); |
---|
1043 | return rc; |
---|
1044 | } |
---|
1045 | |
---|
1046 | static void |
---|
1047 | __gnttab_copy( |
---|
1048 | struct gnttab_copy *op) |
---|
1049 | { |
---|
1050 | struct domain *sd = NULL, *dd = NULL; |
---|
1051 | unsigned long s_frame, d_frame; |
---|
1052 | char *sp, *dp; |
---|
1053 | s16 rc = GNTST_okay; |
---|
1054 | int have_d_grant = 0, have_s_grant = 0, have_s_ref = 0; |
---|
1055 | int src_is_gref, dest_is_gref; |
---|
1056 | |
---|
1057 | if ( ((op->source.offset + op->len) > PAGE_SIZE) || |
---|
1058 | ((op->dest.offset + op->len) > PAGE_SIZE) ) |
---|
1059 | PIN_FAIL(error_out, GNTST_bad_copy_arg, "copy beyond page area.\n"); |
---|
1060 | |
---|
1061 | src_is_gref = op->flags & GNTCOPY_source_gref; |
---|
1062 | dest_is_gref = op->flags & GNTCOPY_dest_gref; |
---|
1063 | |
---|
1064 | if ( (op->source.domid != DOMID_SELF && !src_is_gref ) || |
---|
1065 | (op->dest.domid != DOMID_SELF && !dest_is_gref) ) |
---|
1066 | PIN_FAIL(error_out, GNTST_permission_denied, |
---|
1067 | "only allow copy-by-mfn for DOMID_SELF.\n"); |
---|
1068 | |
---|
1069 | if ( op->source.domid == DOMID_SELF ) |
---|
1070 | sd = rcu_lock_current_domain(); |
---|
1071 | else if ( (sd = rcu_lock_domain_by_id(op->source.domid)) == NULL ) |
---|
1072 | PIN_FAIL(error_out, GNTST_bad_domain, |
---|
1073 | "couldn't find %d\n", op->source.domid); |
---|
1074 | |
---|
1075 | if ( op->dest.domid == DOMID_SELF ) |
---|
1076 | dd = rcu_lock_current_domain(); |
---|
1077 | else if ( (dd = rcu_lock_domain_by_id(op->dest.domid)) == NULL ) |
---|
1078 | PIN_FAIL(error_out, GNTST_bad_domain, |
---|
1079 | "couldn't find %d\n", op->dest.domid); |
---|
1080 | |
---|
1081 | if ( src_is_gref ) |
---|
1082 | { |
---|
1083 | rc = __acquire_grant_for_copy(sd, op->source.u.ref, 1, &s_frame); |
---|
1084 | if ( rc != GNTST_okay ) |
---|
1085 | goto error_out; |
---|
1086 | have_s_grant = 1; |
---|
1087 | } |
---|
1088 | else |
---|
1089 | { |
---|
1090 | s_frame = gmfn_to_mfn(sd, op->source.u.gmfn); |
---|
1091 | } |
---|
1092 | if ( unlikely(!mfn_valid(s_frame)) ) |
---|
1093 | PIN_FAIL(error_out, GNTST_general_error, |
---|
1094 | "source frame %lx invalid.\n", s_frame); |
---|
1095 | if ( !get_page(mfn_to_page(s_frame), sd) ) |
---|
1096 | { |
---|
1097 | if ( !sd->is_dying ) |
---|
1098 | gdprintk(XENLOG_WARNING, "Could not get src frame %lx\n", s_frame); |
---|
1099 | rc = GNTST_general_error; |
---|
1100 | goto error_out; |
---|
1101 | } |
---|
1102 | have_s_ref = 1; |
---|
1103 | |
---|
1104 | if ( dest_is_gref ) |
---|
1105 | { |
---|
1106 | rc = __acquire_grant_for_copy(dd, op->dest.u.ref, 0, &d_frame); |
---|
1107 | if ( rc != GNTST_okay ) |
---|
1108 | goto error_out; |
---|
1109 | have_d_grant = 1; |
---|
1110 | } |
---|
1111 | else |
---|
1112 | { |
---|
1113 | d_frame = gmfn_to_mfn(dd, op->dest.u.gmfn); |
---|
1114 | } |
---|
1115 | if ( unlikely(!mfn_valid(d_frame)) ) |
---|
1116 | PIN_FAIL(error_out, GNTST_general_error, |
---|
1117 | "destination frame %lx invalid.\n", d_frame); |
---|
1118 | if ( !get_page_and_type(mfn_to_page(d_frame), dd, PGT_writable_page) ) |
---|
1119 | { |
---|
1120 | if ( !dd->is_dying ) |
---|
1121 | gdprintk(XENLOG_WARNING, "Could not get dst frame %lx\n", d_frame); |
---|
1122 | rc = GNTST_general_error; |
---|
1123 | goto error_out; |
---|
1124 | } |
---|
1125 | |
---|
1126 | sp = map_domain_page(s_frame); |
---|
1127 | dp = map_domain_page(d_frame); |
---|
1128 | |
---|
1129 | memcpy(dp + op->dest.offset, sp + op->source.offset, op->len); |
---|
1130 | |
---|
1131 | unmap_domain_page(dp); |
---|
1132 | unmap_domain_page(sp); |
---|
1133 | |
---|
1134 | gnttab_mark_dirty(dd, d_frame); |
---|
1135 | |
---|
1136 | put_page_and_type(mfn_to_page(d_frame)); |
---|
1137 | error_out: |
---|
1138 | if ( have_s_ref ) |
---|
1139 | put_page(mfn_to_page(s_frame)); |
---|
1140 | if ( have_s_grant ) |
---|
1141 | __release_grant_for_copy(sd, op->source.u.ref, 1); |
---|
1142 | if ( have_d_grant ) |
---|
1143 | __release_grant_for_copy(dd, op->dest.u.ref, 0); |
---|
1144 | if ( sd ) |
---|
1145 | rcu_unlock_domain(sd); |
---|
1146 | if ( dd ) |
---|
1147 | rcu_unlock_domain(dd); |
---|
1148 | op->status = rc; |
---|
1149 | } |
---|
1150 | |
---|
1151 | static long |
---|
1152 | gnttab_copy( |
---|
1153 | XEN_GUEST_HANDLE(gnttab_copy_t) uop, unsigned int count) |
---|
1154 | { |
---|
1155 | int i; |
---|
1156 | struct gnttab_copy op; |
---|
1157 | |
---|
1158 | for ( i = 0; i < count; i++ ) |
---|
1159 | { |
---|
1160 | if ( unlikely(__copy_from_guest_offset(&op, uop, i, 1)) ) |
---|
1161 | return -EFAULT; |
---|
1162 | __gnttab_copy(&op); |
---|
1163 | if ( unlikely(__copy_to_guest_offset(uop, i, &op, 1)) ) |
---|
1164 | return -EFAULT; |
---|
1165 | } |
---|
1166 | return 0; |
---|
1167 | } |
---|
1168 | |
---|
1169 | long |
---|
1170 | do_grant_table_op( |
---|
1171 | unsigned int cmd, XEN_GUEST_HANDLE(void) uop, unsigned int count) |
---|
1172 | { |
---|
1173 | long rc; |
---|
1174 | struct domain *d = current->domain; |
---|
1175 | |
---|
1176 | if ( count > 512 ) |
---|
1177 | return -EINVAL; |
---|
1178 | |
---|
1179 | LOCK_BIGLOCK(d); |
---|
1180 | |
---|
1181 | rc = -EFAULT; |
---|
1182 | switch ( cmd ) |
---|
1183 | { |
---|
1184 | case GNTTABOP_map_grant_ref: |
---|
1185 | { |
---|
1186 | XEN_GUEST_HANDLE(gnttab_map_grant_ref_t) map = |
---|
1187 | guest_handle_cast(uop, gnttab_map_grant_ref_t); |
---|
1188 | if ( unlikely(!guest_handle_okay(map, count)) ) |
---|
1189 | goto out; |
---|
1190 | rc = -EPERM; |
---|
1191 | if ( unlikely(!grant_operation_permitted(d)) ) |
---|
1192 | goto out; |
---|
1193 | rc = gnttab_map_grant_ref(map, count); |
---|
1194 | break; |
---|
1195 | } |
---|
1196 | case GNTTABOP_unmap_grant_ref: |
---|
1197 | { |
---|
1198 | XEN_GUEST_HANDLE(gnttab_unmap_grant_ref_t) unmap = |
---|
1199 | guest_handle_cast(uop, gnttab_unmap_grant_ref_t); |
---|
1200 | if ( unlikely(!guest_handle_okay(unmap, count)) ) |
---|
1201 | goto out; |
---|
1202 | rc = -EPERM; |
---|
1203 | if ( unlikely(!grant_operation_permitted(d)) ) |
---|
1204 | goto out; |
---|
1205 | rc = gnttab_unmap_grant_ref(unmap, count); |
---|
1206 | break; |
---|
1207 | } |
---|
1208 | case GNTTABOP_setup_table: |
---|
1209 | { |
---|
1210 | rc = gnttab_setup_table( |
---|
1211 | guest_handle_cast(uop, gnttab_setup_table_t), count); |
---|
1212 | break; |
---|
1213 | } |
---|
1214 | case GNTTABOP_transfer: |
---|
1215 | { |
---|
1216 | XEN_GUEST_HANDLE(gnttab_transfer_t) transfer = |
---|
1217 | guest_handle_cast(uop, gnttab_transfer_t); |
---|
1218 | if ( unlikely(!guest_handle_okay(transfer, count)) ) |
---|
1219 | goto out; |
---|
1220 | rc = -EPERM; |
---|
1221 | if ( unlikely(!grant_operation_permitted(d)) ) |
---|
1222 | goto out; |
---|
1223 | rc = gnttab_transfer(transfer, count); |
---|
1224 | break; |
---|
1225 | } |
---|
1226 | case GNTTABOP_copy: |
---|
1227 | { |
---|
1228 | XEN_GUEST_HANDLE(gnttab_copy_t) copy = |
---|
1229 | guest_handle_cast(uop, gnttab_copy_t); |
---|
1230 | if ( unlikely(!guest_handle_okay(copy, count)) ) |
---|
1231 | goto out; |
---|
1232 | rc = gnttab_copy(copy, count); |
---|
1233 | break; |
---|
1234 | } |
---|
1235 | case GNTTABOP_query_size: |
---|
1236 | { |
---|
1237 | rc = gnttab_query_size( |
---|
1238 | guest_handle_cast(uop, gnttab_query_size_t), count); |
---|
1239 | break; |
---|
1240 | } |
---|
1241 | default: |
---|
1242 | rc = -ENOSYS; |
---|
1243 | break; |
---|
1244 | } |
---|
1245 | |
---|
1246 | out: |
---|
1247 | UNLOCK_BIGLOCK(d); |
---|
1248 | |
---|
1249 | return rc; |
---|
1250 | } |
---|
1251 | |
---|
1252 | #ifdef CONFIG_COMPAT |
---|
1253 | #include "compat/grant_table.c" |
---|
1254 | #endif |
---|
1255 | |
---|
1256 | static unsigned int max_nr_active_grant_frames(void) |
---|
1257 | { |
---|
1258 | return (((max_nr_grant_frames * (PAGE_SIZE / sizeof(grant_entry_t))) + |
---|
1259 | ((PAGE_SIZE / sizeof(struct active_grant_entry))-1)) |
---|
1260 | / (PAGE_SIZE / sizeof(struct active_grant_entry))); |
---|
1261 | } |
---|
1262 | |
---|
1263 | int |
---|
1264 | grant_table_create( |
---|
1265 | struct domain *d) |
---|
1266 | { |
---|
1267 | struct grant_table *t; |
---|
1268 | int i; |
---|
1269 | |
---|
1270 | /* If this sizeof assertion fails, fix the function: shared_index */ |
---|
1271 | ASSERT(sizeof(grant_entry_t) == 8); |
---|
1272 | |
---|
1273 | if ( (t = xmalloc(struct grant_table)) == NULL ) |
---|
1274 | goto no_mem_0; |
---|
1275 | |
---|
1276 | /* Simple stuff. */ |
---|
1277 | memset(t, 0, sizeof(*t)); |
---|
1278 | spin_lock_init(&t->lock); |
---|
1279 | t->nr_grant_frames = INITIAL_NR_GRANT_FRAMES; |
---|
1280 | |
---|
1281 | /* Active grant table. */ |
---|
1282 | if ( (t->active = xmalloc_array(struct active_grant_entry *, |
---|
1283 | max_nr_active_grant_frames())) == NULL ) |
---|
1284 | goto no_mem_1; |
---|
1285 | memset(t->active, 0, max_nr_active_grant_frames() * sizeof(t->active[0])); |
---|
1286 | for ( i = 0; |
---|
1287 | i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ ) |
---|
1288 | { |
---|
1289 | if ( (t->active[i] = alloc_xenheap_page()) == NULL ) |
---|
1290 | goto no_mem_2; |
---|
1291 | memset(t->active[i], 0, PAGE_SIZE); |
---|
1292 | } |
---|
1293 | |
---|
1294 | /* Tracking of mapped foreign frames table */ |
---|
1295 | if ( (t->maptrack = xmalloc_array(struct grant_mapping *, |
---|
1296 | max_nr_maptrack_frames())) == NULL ) |
---|
1297 | goto no_mem_2; |
---|
1298 | memset(t->maptrack, 0, max_nr_maptrack_frames() * sizeof(t->maptrack[0])); |
---|
1299 | if ( (t->maptrack[0] = alloc_xenheap_page()) == NULL ) |
---|
1300 | goto no_mem_3; |
---|
1301 | memset(t->maptrack[0], 0, PAGE_SIZE); |
---|
1302 | t->maptrack_limit = PAGE_SIZE / sizeof(struct grant_mapping); |
---|
1303 | for ( i = 0; i < t->maptrack_limit; i++ ) |
---|
1304 | t->maptrack[0][i].ref = i+1; |
---|
1305 | |
---|
1306 | /* Shared grant table. */ |
---|
1307 | if ( (t->shared = xmalloc_array(struct grant_entry *, |
---|
1308 | max_nr_grant_frames)) == NULL ) |
---|
1309 | goto no_mem_3; |
---|
1310 | memset(t->shared, 0, max_nr_grant_frames * sizeof(t->shared[0])); |
---|
1311 | for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ ) |
---|
1312 | { |
---|
1313 | if ( (t->shared[i] = alloc_xenheap_page()) == NULL ) |
---|
1314 | goto no_mem_4; |
---|
1315 | memset(t->shared[i], 0, PAGE_SIZE); |
---|
1316 | } |
---|
1317 | |
---|
1318 | for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ ) |
---|
1319 | gnttab_create_shared_page(d, t, i); |
---|
1320 | |
---|
1321 | /* Okay, install the structure. */ |
---|
1322 | d->grant_table = t; |
---|
1323 | return 0; |
---|
1324 | |
---|
1325 | no_mem_4: |
---|
1326 | for ( i = 0; i < INITIAL_NR_GRANT_FRAMES; i++ ) |
---|
1327 | free_xenheap_page(t->shared[i]); |
---|
1328 | xfree(t->shared); |
---|
1329 | no_mem_3: |
---|
1330 | free_xenheap_page(t->maptrack[0]); |
---|
1331 | xfree(t->maptrack); |
---|
1332 | no_mem_2: |
---|
1333 | for ( i = 0; |
---|
1334 | i < num_act_frames_from_sha_frames(INITIAL_NR_GRANT_FRAMES); i++ ) |
---|
1335 | free_xenheap_page(t->active[i]); |
---|
1336 | xfree(t->active); |
---|
1337 | no_mem_1: |
---|
1338 | xfree(t); |
---|
1339 | no_mem_0: |
---|
1340 | return -ENOMEM; |
---|
1341 | } |
---|
1342 | |
---|
1343 | void |
---|
1344 | gnttab_release_mappings( |
---|
1345 | struct domain *d) |
---|
1346 | { |
---|
1347 | struct grant_table *gt = d->grant_table; |
---|
1348 | struct grant_mapping *map; |
---|
1349 | grant_ref_t ref; |
---|
1350 | grant_handle_t handle; |
---|
1351 | struct domain *rd; |
---|
1352 | struct active_grant_entry *act; |
---|
1353 | struct grant_entry *sha; |
---|
1354 | |
---|
1355 | BUG_ON(!d->is_dying); |
---|
1356 | |
---|
1357 | for ( handle = 0; handle < gt->maptrack_limit; handle++ ) |
---|
1358 | { |
---|
1359 | map = &maptrack_entry(gt, handle); |
---|
1360 | if ( !(map->flags & (GNTMAP_device_map|GNTMAP_host_map)) ) |
---|
1361 | continue; |
---|
1362 | |
---|
1363 | ref = map->ref; |
---|
1364 | |
---|
1365 | gdprintk(XENLOG_INFO, "Grant release (%hu) ref:(%hu) " |
---|
1366 | "flags:(%x) dom:(%hu)\n", |
---|
1367 | handle, ref, map->flags, map->domid); |
---|
1368 | |
---|
1369 | rd = rcu_lock_domain_by_id(map->domid); |
---|
1370 | if ( rd == NULL ) |
---|
1371 | { |
---|
1372 | /* Nothing to clear up... */ |
---|
1373 | map->flags = 0; |
---|
1374 | continue; |
---|
1375 | } |
---|
1376 | |
---|
1377 | spin_lock(&rd->grant_table->lock); |
---|
1378 | |
---|
1379 | act = &active_entry(rd->grant_table, ref); |
---|
1380 | sha = &shared_entry(rd->grant_table, ref); |
---|
1381 | |
---|
1382 | if ( map->flags & GNTMAP_readonly ) |
---|
1383 | { |
---|
1384 | if ( map->flags & GNTMAP_device_map ) |
---|
1385 | { |
---|
1386 | BUG_ON(!(act->pin & GNTPIN_devr_mask)); |
---|
1387 | act->pin -= GNTPIN_devr_inc; |
---|
1388 | put_page(mfn_to_page(act->frame)); |
---|
1389 | } |
---|
1390 | |
---|
1391 | if ( map->flags & GNTMAP_host_map ) |
---|
1392 | { |
---|
1393 | BUG_ON(!(act->pin & GNTPIN_hstr_mask)); |
---|
1394 | act->pin -= GNTPIN_hstr_inc; |
---|
1395 | gnttab_release_put_page(mfn_to_page(act->frame)); |
---|
1396 | } |
---|
1397 | } |
---|
1398 | else |
---|
1399 | { |
---|
1400 | if ( map->flags & GNTMAP_device_map ) |
---|
1401 | { |
---|
1402 | BUG_ON(!(act->pin & GNTPIN_devw_mask)); |
---|
1403 | act->pin -= GNTPIN_devw_inc; |
---|
1404 | put_page_and_type(mfn_to_page(act->frame)); |
---|
1405 | } |
---|
1406 | |
---|
1407 | if ( map->flags & GNTMAP_host_map ) |
---|
1408 | { |
---|
1409 | BUG_ON(!(act->pin & GNTPIN_hstw_mask)); |
---|
1410 | act->pin -= GNTPIN_hstw_inc; |
---|
1411 | gnttab_release_put_page_and_type(mfn_to_page(act->frame)); |
---|
1412 | } |
---|
1413 | |
---|
1414 | if ( (act->pin & (GNTPIN_devw_mask|GNTPIN_hstw_mask)) == 0 ) |
---|
1415 | gnttab_clear_flag(_GTF_writing, &sha->flags); |
---|
1416 | } |
---|
1417 | |
---|
1418 | if ( act->pin == 0 ) |
---|
1419 | gnttab_clear_flag(_GTF_reading, &sha->flags); |
---|
1420 | |
---|
1421 | spin_unlock(&rd->grant_table->lock); |
---|
1422 | |
---|
1423 | rcu_unlock_domain(rd); |
---|
1424 | |
---|
1425 | map->flags = 0; |
---|
1426 | } |
---|
1427 | } |
---|
1428 | |
---|
1429 | |
---|
1430 | void |
---|
1431 | grant_table_destroy( |
---|
1432 | struct domain *d) |
---|
1433 | { |
---|
1434 | struct grant_table *t = d->grant_table; |
---|
1435 | int i; |
---|
1436 | |
---|
1437 | if ( t == NULL ) |
---|
1438 | return; |
---|
1439 | |
---|
1440 | for ( i = 0; i < nr_grant_frames(t); i++ ) |
---|
1441 | free_xenheap_page(t->shared[i]); |
---|
1442 | xfree(t->shared); |
---|
1443 | |
---|
1444 | for ( i = 0; i < nr_maptrack_frames(t); i++ ) |
---|
1445 | free_xenheap_page(t->maptrack[i]); |
---|
1446 | xfree(t->maptrack); |
---|
1447 | |
---|
1448 | for ( i = 0; i < nr_active_grant_frames(t); i++ ) |
---|
1449 | free_xenheap_page(t->active[i]); |
---|
1450 | xfree(t->active); |
---|
1451 | |
---|
1452 | xfree(t); |
---|
1453 | d->grant_table = NULL; |
---|
1454 | } |
---|
1455 | |
---|
1456 | /* |
---|
1457 | * Local variables: |
---|
1458 | * mode: C |
---|
1459 | * c-set-style: "BSD" |
---|
1460 | * c-basic-offset: 4 |
---|
1461 | * tab-width: 4 |
---|
1462 | * indent-tabs-mode: nil |
---|
1463 | * End: |
---|
1464 | */ |
---|