1 | ******************************************************************************** |
---|
2 | A Rough Introduction to Using Grant Tables |
---|
3 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
4 | Christopher Clark, March, 2005. |
---|
5 | |
---|
6 | Grant tables are a mechanism for sharing and transferring frames between |
---|
7 | domains, without requiring the participating domains to be privileged. |
---|
8 | |
---|
9 | The first mode of use allows domA to grant domB access to a specific frame, |
---|
10 | whilst retaining ownership. The block front driver uses this to grant memory |
---|
11 | access to the block back driver, so that it may read or write as requested. |
---|
12 | |
---|
13 | 1. domA creates a grant access reference, and transmits the ref id to domB. |
---|
14 | 2. domB uses the reference to map the granted frame. |
---|
15 | 3. domB performs the memory access. |
---|
16 | 4. domB unmaps the granted frame. |
---|
17 | 5. domA removes its grant. |
---|
18 | |
---|
19 | |
---|
20 | The second mode allows domA to accept a transfer of ownership of a frame from |
---|
21 | domB. The net front and back driver will use this for packet tx/rx. This |
---|
22 | mechanism is still being implemented, though the xen<->guest interface design |
---|
23 | is complete. |
---|
24 | |
---|
25 | 1. domA creates an accept transfer grant reference, and transmits it to domB. |
---|
26 | 2. domB uses the ref to hand over a frame it owns. |
---|
27 | 3. domA accepts the transfer |
---|
28 | 4. domA clears the used reference. |
---|
29 | |
---|
30 | |
---|
31 | ******************************************************************************** |
---|
32 | Data structures |
---|
33 | ~~~~~~~~~~~~~~~ |
---|
34 | |
---|
35 | The following data structures are used by Xen and the guests to implement |
---|
36 | grant tables: |
---|
37 | |
---|
38 | 1. Shared grant entries |
---|
39 | 2. Active grant entries |
---|
40 | 3. Map tracking |
---|
41 | |
---|
42 | These are not the users primary interface to grant tables, but are discussed |
---|
43 | because an understanding of how they work may be useful. Each of these is a |
---|
44 | finite resource. |
---|
45 | |
---|
46 | Shared grant entries |
---|
47 | ~~~~~~~~~~~~~~~~~~~~ |
---|
48 | |
---|
49 | A set of pages are shared between Xen and a guest, holding the shared grant |
---|
50 | entries. The guest writes into these entries to create grant references. The |
---|
51 | index of the entry is transmitted to the remote domain: this is the |
---|
52 | reference used to activate an entry. Xen will write into a shared entry to |
---|
53 | indicate to a guest that its grant is in use. |
---|
54 | sha->domid : remote domain being granted rights |
---|
55 | sha->frame : machine frame being granted |
---|
56 | sha->flags : allow access, allow transfer, remote is reading/writing, etc. |
---|
57 | |
---|
58 | Active grant entries |
---|
59 | ~~~~~~~~~~~~~~~~~~~~ |
---|
60 | |
---|
61 | Xen maintains a set of private frames per domain, holding the active grant |
---|
62 | entries for safety, and to reference count mappings. |
---|
63 | act->domid : remote domain being granted rights |
---|
64 | act->frame : machine frame being granted |
---|
65 | act->pin : used to hold reference counts |
---|
66 | |
---|
67 | Map tracking |
---|
68 | ~~~~~~~~~~~~ |
---|
69 | |
---|
70 | Every time a frame is mapped, a map track entry is stored in the metadata of |
---|
71 | the mapping domain. The index of this entry is returned from the map call, |
---|
72 | and is used to unmap the frame. Map track entries are also searched whenever a |
---|
73 | page table entry containing a foreign frame number is overwritten: the first |
---|
74 | matching map track entry is then removed, as if unmap had been invoked. |
---|
75 | These are not used by the transfer mechanism. |
---|
76 | map->domid : owner of the mapped frame |
---|
77 | map->ref_and_flags : grant reference, ro/rw, mapped for host or device access |
---|
78 | |
---|
79 | ******************************************************************************** |
---|
80 | |
---|
81 | Granting a foreign domain access to frames |
---|
82 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
83 | |
---|
84 | domA [frame]--> domB |
---|
85 | |
---|
86 | |
---|
87 | domA: #include <asm-xen/gnttab.h> |
---|
88 | grant_ref_t gref[BATCH_SIZE]; |
---|
89 | |
---|
90 | for ( i = 0; i < BATCH_SIZE; i++ ) |
---|
91 | gref[i] = gnttab_grant_foreign_access( domBid, mfn, (readonly ? 1 : 0) ); |
---|
92 | |
---|
93 | |
---|
94 | .. gref is then somehow transmitted to domB for use. |
---|
95 | |
---|
96 | |
---|
97 | Mapping foreign frames |
---|
98 | ~~~~~~~~~~~~~~~~~~~~~~ |
---|
99 | |
---|
100 | domB: #include <asm-xen/hypervisor.h> |
---|
101 | unsigned long mmap_vstart; |
---|
102 | gnttab_op_t aop[BATCH_SIZE]; |
---|
103 | grant_ref_t mapped_handle[BATCH_SIZE]; |
---|
104 | |
---|
105 | if ( (mmap_vstart = allocate_empty_lowmem_region(BATCH_SIZE)) == 0 ) |
---|
106 | BUG(); |
---|
107 | |
---|
108 | for ( i = 0; i < BATCH_SIZE; i++ ) |
---|
109 | { |
---|
110 | aop[i].u.map_grant_ref.host_virt_addr = |
---|
111 | mmap_vstart + (i * PAGE_SIZE); |
---|
112 | aop[i].u.map_grant_ref.dom = domAid; |
---|
113 | aop[i].u.map_grant_ref.ref = gref[i]; |
---|
114 | aop[i].u.map_grant_ref.flags = ( GNTMAP_host_map | GNTMAP_readonly ); |
---|
115 | } |
---|
116 | |
---|
117 | if ( unlikely(HYPERVISOR_grant_table_op( |
---|
118 | GNTTABOP_map_grant_ref, aop, BATCH_SIZE))) |
---|
119 | BUG(); |
---|
120 | |
---|
121 | for ( i = 0; i < BATCH_SIZE; i++ ) |
---|
122 | { |
---|
123 | if ( unlikely(aop[i].u.map_grant_ref.handle < 0) ) |
---|
124 | { |
---|
125 | tidyup_all(aop, i); |
---|
126 | goto panic; |
---|
127 | } |
---|
128 | |
---|
129 | phys_to_machine_mapping[__pa(mmap_vstart + (i * PAGE_SIZE))>>PAGE_SHIFT] = |
---|
130 | FOREIGN_FRAME(aop[i].u.map_grant_ref.dev_bus_addr); |
---|
131 | |
---|
132 | mapped_handle[i] = aop[i].u.map_grant_ref.handle; |
---|
133 | } |
---|
134 | |
---|
135 | |
---|
136 | |
---|
137 | Unmapping foreign frames |
---|
138 | ~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
139 | |
---|
140 | domB: |
---|
141 | for ( i = 0; i < BATCH_SIZE; i++ ) |
---|
142 | { |
---|
143 | aop[i].u.unmap_grant_ref.host_virt_addr = mmap_vstart + (i * PAGE_SIZE); |
---|
144 | aop[i].u.unmap_grant_ref.dev_bus_addr = 0; |
---|
145 | aop[i].u.unmap_grant_ref.handle = mapped_handle[i]; |
---|
146 | } |
---|
147 | if ( unlikely(HYPERVISOR_grant_table_op( |
---|
148 | GNTTABOP_unmap_grant_ref, aop, BATCH_SIZE))) |
---|
149 | BUG(); |
---|
150 | |
---|
151 | |
---|
152 | Ending foreign access |
---|
153 | ~~~~~~~~~~~~~~~~~~~~~ |
---|
154 | |
---|
155 | Note that this only prevents further mappings; it does _not_ revoke access. |
---|
156 | Should _only_ be used when the remote domain has unmapped the frame. |
---|
157 | gnttab_query_foreign_access( gref ) will indicate the state of any mapping. |
---|
158 | |
---|
159 | domA: |
---|
160 | if ( gnttab_query_foreign_access( gref[i] ) == 0 ) |
---|
161 | gnttab_end_foreign_access( gref[i], readonly ); |
---|
162 | |
---|
163 | TODO: readonly yet to be implemented. |
---|
164 | |
---|
165 | |
---|
166 | ******************************************************************************** |
---|
167 | |
---|
168 | Transferring ownership of a frame to another domain |
---|
169 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
170 | |
---|
171 | [ XXX: Transfer mechanism is alpha-calibre code, untested, use at own risk XXX ] |
---|
172 | [ XXX: show use of batch operations below, rather than single frame XXX ] |
---|
173 | [ XXX: linux internal interface could/should be wrapped to be tidier XXX ] |
---|
174 | |
---|
175 | |
---|
176 | Prepare to accept a frame from a foreign domain |
---|
177 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
178 | |
---|
179 | domA: |
---|
180 | if ( (p = alloc_page(GFP_HIGHUSER)) == NULL ) |
---|
181 | { |
---|
182 | printk("Cannot alloc a frame to surrender\n"); |
---|
183 | break; |
---|
184 | } |
---|
185 | pfn = p - mem_map; |
---|
186 | mfn = phys_to_machine_mapping[pfn]; |
---|
187 | |
---|
188 | if ( !PageHighMem(p) ) |
---|
189 | { |
---|
190 | v = phys_to_virt(pfn << PAGE_SHIFT); |
---|
191 | scrub_pages(v, 1); |
---|
192 | queue_l1_entry_update(get_ptep((unsigned long)v), 0); |
---|
193 | } |
---|
194 | |
---|
195 | /* Ensure that ballooned highmem pages don't have cached mappings. */ |
---|
196 | kmap_flush_unused(); |
---|
197 | |
---|
198 | /* Flush updates through and flush the TLB. */ |
---|
199 | xen_tlb_flush(); |
---|
200 | |
---|
201 | phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY; |
---|
202 | |
---|
203 | if ( HYPERVISOR_dom_mem_op( |
---|
204 | MEMOP_decrease_reservation, &mfn, 1, 0) != 1 ) |
---|
205 | { |
---|
206 | printk("MEMOP_decrease_reservation failed\n"); |
---|
207 | /* er... ok. free the page then */ |
---|
208 | __free_page(p); |
---|
209 | break; |
---|
210 | } |
---|
211 | |
---|
212 | accepting_pfn = pfn; |
---|
213 | ref = gnttab_grant_foreign_transfer( (domid_t) args.arg[0], pfn ); |
---|
214 | printk("Accepting dom %lu frame at ref (%d)\n", args.arg[0], ref); |
---|
215 | |
---|
216 | |
---|
217 | Transfer a frame to a foreign domain |
---|
218 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
219 | |
---|
220 | domB: |
---|
221 | mmu_update_t update; |
---|
222 | domid_t domid; |
---|
223 | grant_ref_t gref; |
---|
224 | unsigned long pfn, mfn, *v; |
---|
225 | struct page *transfer_page = 0; |
---|
226 | |
---|
227 | /* alloc a page and grant access. |
---|
228 | * alloc page returns a page struct. */ |
---|
229 | if ( (transfer_page = alloc_page(GFP_HIGHUSER)) == NULL ) |
---|
230 | return -ENOMEM; |
---|
231 | |
---|
232 | pfn = transfer_page - mem_map; |
---|
233 | mfn = phys_to_machine_mapping[pfn]; |
---|
234 | |
---|
235 | /* need to remove all references to this page */ |
---|
236 | if ( !PageHighMem(transfer_page) ) |
---|
237 | { |
---|
238 | v = phys_to_virt(pfn << PAGE_SHIFT); |
---|
239 | scrub_pages(v, 1); |
---|
240 | sprintf((char *)v, "This page (%lx) was transferred.\n", mfn); |
---|
241 | queue_l1_entry_update(get_ptep((unsigned long)v), 0); |
---|
242 | } |
---|
243 | #ifdef CONFIG_XEN_SCRUB_PAGES |
---|
244 | else |
---|
245 | { |
---|
246 | v = kmap(transfer_page); |
---|
247 | scrub_pages(v, 1); |
---|
248 | sprintf((char *)v, "This page (%lx) was transferred.\n", mfn); |
---|
249 | kunmap(transfer_page); |
---|
250 | } |
---|
251 | #endif |
---|
252 | /* Delete any cached kmappings */ |
---|
253 | kmap_flush_unused(); |
---|
254 | |
---|
255 | /* Flush updates through and flush the TLB */ |
---|
256 | xen_tlb_flush(); |
---|
257 | |
---|
258 | /* invalidate in P2M */ |
---|
259 | phys_to_machine_mapping[pfn] = INVALID_P2M_ENTRY; |
---|
260 | |
---|
261 | domid = (domid_t)args.arg[0]; |
---|
262 | gref = (grant_ref_t)args.arg[1]; |
---|
263 | |
---|
264 | update.ptr = MMU_EXTENDED_COMMAND; |
---|
265 | update.ptr |= ((gref & 0x00FF) << 2); |
---|
266 | update.ptr |= mfn << PAGE_SHIFT; |
---|
267 | |
---|
268 | update.val = MMUEXT_TRANSFER_PAGE; |
---|
269 | update.val |= (domid << 16); |
---|
270 | update.val |= (gref & 0xFF00); |
---|
271 | |
---|
272 | ret = HYPERVISOR_mmu_update(&update, 1, NULL); |
---|
273 | |
---|
274 | |
---|
275 | Map a transferred frame |
---|
276 | ~~~~~~~~~~~~~~~~~~~~~~~ |
---|
277 | |
---|
278 | TODO: |
---|
279 | |
---|
280 | |
---|
281 | Clear the used transfer reference |
---|
282 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
283 | |
---|
284 | TODO: |
---|
285 | |
---|
286 | |
---|
287 | ******************************************************************************** |
---|
288 | |
---|
289 | Using a private reserve of grant references |
---|
290 | ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
---|
291 | |
---|
292 | Where it is known in advance how many grant references are required, and |
---|
293 | failure to allocate them on demand would cause difficulty, a batch can be |
---|
294 | allocated and held in a private reserve. |
---|
295 | |
---|
296 | To reserve a private batch: |
---|
297 | |
---|
298 | /* housekeeping data - treat as opaque: */ |
---|
299 | grant_ref_t gref_head, gref_terminal; |
---|
300 | |
---|
301 | if ( 0 > gnttab_alloc_grant_references( number_to_reserve, |
---|
302 | &gref_head, &gref_terminal )) |
---|
303 | return -ENOSPC; |
---|
304 | |
---|
305 | |
---|
306 | To release a batch back to the shared pool: |
---|
307 | |
---|
308 | gnttab_free_grant_references( number_reserved, gref_head ); |
---|
309 | |
---|
310 | |
---|
311 | To claim a reserved reference: |
---|
312 | |
---|
313 | ref = gnttab_claim_grant_reference( &gref_head, gref_terminal ); |
---|
314 | |
---|
315 | |
---|
316 | To release a claimed reference back to the reserve pool: |
---|
317 | |
---|
318 | gnttab_release_grant_reference( &gref_head, gref ); |
---|
319 | |
---|
320 | |
---|
321 | To use a claimed reference to grant access, use these alternative functions |
---|
322 | that take an additional parameter of the grant reference to use: |
---|
323 | |
---|
324 | gnttab_grant_foreign_access_ref |
---|
325 | gnttab_grant_foreign_transfer_ref |
---|