1 | /****************************************************************************** |
---|
2 | * drivers/xen/netback/netback.c |
---|
3 | * |
---|
4 | * Back-end of the driver for virtual network devices. This portion of the |
---|
5 | * driver exports a 'unified' network-device interface that can be accessed |
---|
6 | * by any operating system that implements a compatible front end. A |
---|
7 | * reference front-end implementation can be found in: |
---|
8 | * drivers/xen/netfront/netfront.c |
---|
9 | * |
---|
10 | * Copyright (c) 2002-2005, K A Fraser |
---|
11 | * |
---|
12 | * This program is free software; you can redistribute it and/or |
---|
13 | * modify it under the terms of the GNU General Public License version 2 |
---|
14 | * as published by the Free Software Foundation; or, when distributed |
---|
15 | * separately from the Linux kernel or incorporated into other |
---|
16 | * software packages, subject to the following license: |
---|
17 | * |
---|
18 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
---|
19 | * of this source file (the "Software"), to deal in the Software without |
---|
20 | * restriction, including without limitation the rights to use, copy, modify, |
---|
21 | * merge, publish, distribute, sublicense, and/or sell copies of the Software, |
---|
22 | * and to permit persons to whom the Software is furnished to do so, subject to |
---|
23 | * the following conditions: |
---|
24 | * |
---|
25 | * The above copyright notice and this permission notice shall be included in |
---|
26 | * all copies or substantial portions of the Software. |
---|
27 | * |
---|
28 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
---|
29 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
---|
30 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
---|
31 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
---|
32 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING |
---|
33 | * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS |
---|
34 | * IN THE SOFTWARE. |
---|
35 | */ |
---|
36 | |
---|
37 | #include "common.h" |
---|
38 | #include <xen/balloon.h> |
---|
39 | #include <xen/interface/memory.h> |
---|
40 | |
---|
41 | /*define NETBE_DEBUG_INTERRUPT*/ |
---|
42 | |
---|
43 | /* extra field used in struct page */ |
---|
44 | #define netif_page_index(pg) (*(long *)&(pg)->mapping) |
---|
45 | |
---|
46 | struct netbk_rx_meta { |
---|
47 | skb_frag_t frag; |
---|
48 | int id; |
---|
49 | int copy:1; |
---|
50 | }; |
---|
51 | |
---|
52 | static void netif_idx_release(u16 pending_idx); |
---|
53 | static void netif_page_release(struct page *page); |
---|
54 | static void make_tx_response(netif_t *netif, |
---|
55 | netif_tx_request_t *txp, |
---|
56 | s8 st); |
---|
57 | static netif_rx_response_t *make_rx_response(netif_t *netif, |
---|
58 | u16 id, |
---|
59 | s8 st, |
---|
60 | u16 offset, |
---|
61 | u16 size, |
---|
62 | u16 flags); |
---|
63 | |
---|
64 | static void net_tx_action(unsigned long unused); |
---|
65 | static DECLARE_TASKLET(net_tx_tasklet, net_tx_action, 0); |
---|
66 | |
---|
67 | static void net_rx_action(unsigned long unused); |
---|
68 | static DECLARE_TASKLET(net_rx_tasklet, net_rx_action, 0); |
---|
69 | |
---|
70 | static struct timer_list net_timer; |
---|
71 | |
---|
72 | #define MAX_PENDING_REQS 256 |
---|
73 | |
---|
74 | static struct sk_buff_head rx_queue; |
---|
75 | |
---|
76 | static struct page **mmap_pages; |
---|
77 | static inline unsigned long idx_to_kaddr(unsigned int idx) |
---|
78 | { |
---|
79 | return (unsigned long)pfn_to_kaddr(page_to_pfn(mmap_pages[idx])); |
---|
80 | } |
---|
81 | |
---|
82 | #define PKT_PROT_LEN 64 |
---|
83 | |
---|
84 | static struct pending_tx_info { |
---|
85 | netif_tx_request_t req; |
---|
86 | netif_t *netif; |
---|
87 | } pending_tx_info[MAX_PENDING_REQS]; |
---|
88 | static u16 pending_ring[MAX_PENDING_REQS]; |
---|
89 | typedef unsigned int PEND_RING_IDX; |
---|
90 | #define MASK_PEND_IDX(_i) ((_i)&(MAX_PENDING_REQS-1)) |
---|
91 | static PEND_RING_IDX pending_prod, pending_cons; |
---|
92 | #define NR_PENDING_REQS (MAX_PENDING_REQS - pending_prod + pending_cons) |
---|
93 | |
---|
94 | /* Freed TX SKBs get batched on this ring before return to pending_ring. */ |
---|
95 | static u16 dealloc_ring[MAX_PENDING_REQS]; |
---|
96 | static PEND_RING_IDX dealloc_prod, dealloc_cons; |
---|
97 | |
---|
98 | static struct sk_buff_head tx_queue; |
---|
99 | |
---|
100 | static grant_handle_t grant_tx_handle[MAX_PENDING_REQS]; |
---|
101 | static gnttab_unmap_grant_ref_t tx_unmap_ops[MAX_PENDING_REQS]; |
---|
102 | static gnttab_map_grant_ref_t tx_map_ops[MAX_PENDING_REQS]; |
---|
103 | |
---|
104 | static struct list_head net_schedule_list; |
---|
105 | static spinlock_t net_schedule_list_lock; |
---|
106 | |
---|
107 | #define MAX_MFN_ALLOC 64 |
---|
108 | static unsigned long mfn_list[MAX_MFN_ALLOC]; |
---|
109 | static unsigned int alloc_index = 0; |
---|
110 | |
---|
111 | static inline unsigned long alloc_mfn(void) |
---|
112 | { |
---|
113 | BUG_ON(alloc_index == 0); |
---|
114 | return mfn_list[--alloc_index]; |
---|
115 | } |
---|
116 | |
---|
117 | static int check_mfn(int nr) |
---|
118 | { |
---|
119 | struct xen_memory_reservation reservation = { |
---|
120 | .extent_order = 0, |
---|
121 | .domid = DOMID_SELF |
---|
122 | }; |
---|
123 | |
---|
124 | if (likely(alloc_index >= nr)) |
---|
125 | return 0; |
---|
126 | |
---|
127 | set_xen_guest_handle(reservation.extent_start, mfn_list + alloc_index); |
---|
128 | reservation.nr_extents = MAX_MFN_ALLOC - alloc_index; |
---|
129 | alloc_index += HYPERVISOR_memory_op(XENMEM_increase_reservation, |
---|
130 | &reservation); |
---|
131 | |
---|
132 | return alloc_index >= nr ? 0 : -ENOMEM; |
---|
133 | } |
---|
134 | |
---|
135 | static inline void maybe_schedule_tx_action(void) |
---|
136 | { |
---|
137 | smp_mb(); |
---|
138 | if ((NR_PENDING_REQS < (MAX_PENDING_REQS/2)) && |
---|
139 | !list_empty(&net_schedule_list)) |
---|
140 | tasklet_schedule(&net_tx_tasklet); |
---|
141 | } |
---|
142 | |
---|
143 | static struct sk_buff *netbk_copy_skb(struct sk_buff *skb) |
---|
144 | { |
---|
145 | struct skb_shared_info *ninfo; |
---|
146 | struct sk_buff *nskb; |
---|
147 | unsigned long offset; |
---|
148 | int ret; |
---|
149 | int len; |
---|
150 | int headlen; |
---|
151 | |
---|
152 | BUG_ON(skb_shinfo(skb)->frag_list != NULL); |
---|
153 | |
---|
154 | nskb = alloc_skb(SKB_MAX_HEAD(0), GFP_ATOMIC | __GFP_NOWARN); |
---|
155 | if (unlikely(!nskb)) |
---|
156 | goto err; |
---|
157 | |
---|
158 | skb_reserve(nskb, 16 + NET_IP_ALIGN); |
---|
159 | headlen = nskb->end - nskb->data; |
---|
160 | if (headlen > skb_headlen(skb)) |
---|
161 | headlen = skb_headlen(skb); |
---|
162 | ret = skb_copy_bits(skb, 0, __skb_put(nskb, headlen), headlen); |
---|
163 | BUG_ON(ret); |
---|
164 | |
---|
165 | ninfo = skb_shinfo(nskb); |
---|
166 | ninfo->gso_size = skb_shinfo(skb)->gso_size; |
---|
167 | ninfo->gso_type = skb_shinfo(skb)->gso_type; |
---|
168 | |
---|
169 | offset = headlen; |
---|
170 | len = skb->len - headlen; |
---|
171 | |
---|
172 | nskb->len = skb->len; |
---|
173 | nskb->data_len = len; |
---|
174 | nskb->truesize += len; |
---|
175 | |
---|
176 | while (len) { |
---|
177 | struct page *page; |
---|
178 | int copy; |
---|
179 | int zero; |
---|
180 | |
---|
181 | if (unlikely(ninfo->nr_frags >= MAX_SKB_FRAGS)) { |
---|
182 | dump_stack(); |
---|
183 | goto err_free; |
---|
184 | } |
---|
185 | |
---|
186 | copy = len >= PAGE_SIZE ? PAGE_SIZE : len; |
---|
187 | zero = len >= PAGE_SIZE ? 0 : __GFP_ZERO; |
---|
188 | |
---|
189 | page = alloc_page(GFP_ATOMIC | __GFP_NOWARN | zero); |
---|
190 | if (unlikely(!page)) |
---|
191 | goto err_free; |
---|
192 | |
---|
193 | ret = skb_copy_bits(skb, offset, page_address(page), copy); |
---|
194 | BUG_ON(ret); |
---|
195 | |
---|
196 | ninfo->frags[ninfo->nr_frags].page = page; |
---|
197 | ninfo->frags[ninfo->nr_frags].page_offset = 0; |
---|
198 | ninfo->frags[ninfo->nr_frags].size = copy; |
---|
199 | ninfo->nr_frags++; |
---|
200 | |
---|
201 | offset += copy; |
---|
202 | len -= copy; |
---|
203 | } |
---|
204 | |
---|
205 | offset = nskb->data - skb->data; |
---|
206 | |
---|
207 | nskb->h.raw = skb->h.raw + offset; |
---|
208 | nskb->nh.raw = skb->nh.raw + offset; |
---|
209 | nskb->mac.raw = skb->mac.raw + offset; |
---|
210 | |
---|
211 | return nskb; |
---|
212 | |
---|
213 | err_free: |
---|
214 | kfree_skb(nskb); |
---|
215 | err: |
---|
216 | return NULL; |
---|
217 | } |
---|
218 | |
---|
219 | static inline int netbk_max_required_rx_slots(netif_t *netif) |
---|
220 | { |
---|
221 | if (netif->features & (NETIF_F_SG|NETIF_F_TSO)) |
---|
222 | return MAX_SKB_FRAGS + 2; /* header + extra_info + frags */ |
---|
223 | return 1; /* all in one */ |
---|
224 | } |
---|
225 | |
---|
226 | static inline int netbk_queue_full(netif_t *netif) |
---|
227 | { |
---|
228 | RING_IDX peek = netif->rx_req_cons_peek; |
---|
229 | RING_IDX needed = netbk_max_required_rx_slots(netif); |
---|
230 | |
---|
231 | return ((netif->rx.sring->req_prod - peek) < needed) || |
---|
232 | ((netif->rx.rsp_prod_pvt + NET_RX_RING_SIZE - peek) < needed); |
---|
233 | } |
---|
234 | |
---|
235 | static void tx_queue_callback(unsigned long data) |
---|
236 | { |
---|
237 | netif_t *netif = (netif_t *)data; |
---|
238 | if (netif_schedulable(netif)) |
---|
239 | netif_wake_queue(netif->dev); |
---|
240 | } |
---|
241 | |
---|
242 | int netif_be_start_xmit(struct sk_buff *skb, struct net_device *dev) |
---|
243 | { |
---|
244 | netif_t *netif = netdev_priv(dev); |
---|
245 | |
---|
246 | BUG_ON(skb->dev != dev); |
---|
247 | |
---|
248 | /* Drop the packet if the target domain has no receive buffers. */ |
---|
249 | if (unlikely(!netif_schedulable(netif) || netbk_queue_full(netif))) |
---|
250 | goto drop; |
---|
251 | |
---|
252 | /* |
---|
253 | * Copy the packet here if it's destined for a flipping interface |
---|
254 | * but isn't flippable (e.g. extra references to data). |
---|
255 | * XXX For now we also copy skbuffs whose head crosses a page |
---|
256 | * boundary, because netbk_gop_skb can't handle them. |
---|
257 | */ |
---|
258 | if (!netif->copying_receiver || |
---|
259 | ((skb_headlen(skb) + offset_in_page(skb->data)) >= PAGE_SIZE)) { |
---|
260 | struct sk_buff *nskb = netbk_copy_skb(skb); |
---|
261 | if ( unlikely(nskb == NULL) ) |
---|
262 | goto drop; |
---|
263 | /* Copy only the header fields we use in this driver. */ |
---|
264 | nskb->dev = skb->dev; |
---|
265 | nskb->ip_summed = skb->ip_summed; |
---|
266 | nskb->proto_data_valid = skb->proto_data_valid; |
---|
267 | dev_kfree_skb(skb); |
---|
268 | skb = nskb; |
---|
269 | } |
---|
270 | |
---|
271 | netif->rx_req_cons_peek += skb_shinfo(skb)->nr_frags + 1 + |
---|
272 | !!skb_shinfo(skb)->gso_size; |
---|
273 | netif_get(netif); |
---|
274 | |
---|
275 | if (netbk_can_queue(dev) && netbk_queue_full(netif)) { |
---|
276 | netif->rx.sring->req_event = netif->rx_req_cons_peek + |
---|
277 | netbk_max_required_rx_slots(netif); |
---|
278 | mb(); /* request notification /then/ check & stop the queue */ |
---|
279 | if (netbk_queue_full(netif)) { |
---|
280 | netif_stop_queue(dev); |
---|
281 | /* |
---|
282 | * Schedule 500ms timeout to restart the queue, thus |
---|
283 | * ensuring that an inactive queue will be drained. |
---|
284 | * Packets will be immediately be dropped until more |
---|
285 | * receive buffers become available (see |
---|
286 | * netbk_queue_full() check above). |
---|
287 | */ |
---|
288 | netif->tx_queue_timeout.data = (unsigned long)netif; |
---|
289 | netif->tx_queue_timeout.function = tx_queue_callback; |
---|
290 | __mod_timer(&netif->tx_queue_timeout, jiffies + HZ/2); |
---|
291 | } |
---|
292 | } |
---|
293 | |
---|
294 | skb_queue_tail(&rx_queue, skb); |
---|
295 | tasklet_schedule(&net_rx_tasklet); |
---|
296 | |
---|
297 | return 0; |
---|
298 | |
---|
299 | drop: |
---|
300 | netif->stats.tx_dropped++; |
---|
301 | dev_kfree_skb(skb); |
---|
302 | return 0; |
---|
303 | } |
---|
304 | |
---|
305 | #if 0 |
---|
306 | static void xen_network_done_notify(void) |
---|
307 | { |
---|
308 | static struct net_device *eth0_dev = NULL; |
---|
309 | if (unlikely(eth0_dev == NULL)) |
---|
310 | eth0_dev = __dev_get_by_name("eth0"); |
---|
311 | netif_rx_schedule(eth0_dev); |
---|
312 | } |
---|
313 | /* |
---|
314 | * Add following to poll() function in NAPI driver (Tigon3 is example): |
---|
315 | * if ( xen_network_done() ) |
---|
316 | * tg3_enable_ints(tp); |
---|
317 | */ |
---|
318 | int xen_network_done(void) |
---|
319 | { |
---|
320 | return skb_queue_empty(&rx_queue); |
---|
321 | } |
---|
322 | #endif |
---|
323 | |
---|
324 | struct netrx_pending_operations { |
---|
325 | unsigned trans_prod, trans_cons; |
---|
326 | unsigned mmu_prod, mmu_cons; |
---|
327 | unsigned mcl_prod, mcl_cons; |
---|
328 | unsigned copy_prod, copy_cons; |
---|
329 | unsigned meta_prod, meta_cons; |
---|
330 | mmu_update_t *mmu; |
---|
331 | gnttab_transfer_t *trans; |
---|
332 | gnttab_copy_t *copy; |
---|
333 | multicall_entry_t *mcl; |
---|
334 | struct netbk_rx_meta *meta; |
---|
335 | }; |
---|
336 | |
---|
337 | /* Set up the grant operations for this fragment. If it's a flipping |
---|
338 | interface, we also set up the unmap request from here. */ |
---|
339 | static u16 netbk_gop_frag(netif_t *netif, struct netbk_rx_meta *meta, |
---|
340 | int i, struct netrx_pending_operations *npo, |
---|
341 | struct page *page, unsigned long size, |
---|
342 | unsigned long offset) |
---|
343 | { |
---|
344 | mmu_update_t *mmu; |
---|
345 | gnttab_transfer_t *gop; |
---|
346 | gnttab_copy_t *copy_gop; |
---|
347 | multicall_entry_t *mcl; |
---|
348 | netif_rx_request_t *req; |
---|
349 | unsigned long old_mfn, new_mfn; |
---|
350 | |
---|
351 | old_mfn = virt_to_mfn(page_address(page)); |
---|
352 | |
---|
353 | req = RING_GET_REQUEST(&netif->rx, netif->rx.req_cons + i); |
---|
354 | if (netif->copying_receiver) { |
---|
355 | /* The fragment needs to be copied rather than |
---|
356 | flipped. */ |
---|
357 | meta->copy = 1; |
---|
358 | copy_gop = npo->copy + npo->copy_prod++; |
---|
359 | copy_gop->flags = GNTCOPY_dest_gref; |
---|
360 | if (PageForeign(page)) { |
---|
361 | struct pending_tx_info *src_pend = |
---|
362 | &pending_tx_info[netif_page_index(page)]; |
---|
363 | copy_gop->source.domid = src_pend->netif->domid; |
---|
364 | copy_gop->source.u.ref = src_pend->req.gref; |
---|
365 | copy_gop->flags |= GNTCOPY_source_gref; |
---|
366 | } else { |
---|
367 | copy_gop->source.domid = DOMID_SELF; |
---|
368 | copy_gop->source.u.gmfn = old_mfn; |
---|
369 | } |
---|
370 | copy_gop->source.offset = offset; |
---|
371 | copy_gop->dest.domid = netif->domid; |
---|
372 | copy_gop->dest.offset = 0; |
---|
373 | copy_gop->dest.u.ref = req->gref; |
---|
374 | copy_gop->len = size; |
---|
375 | } else { |
---|
376 | meta->copy = 0; |
---|
377 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
378 | new_mfn = alloc_mfn(); |
---|
379 | |
---|
380 | /* |
---|
381 | * Set the new P2M table entry before |
---|
382 | * reassigning the old data page. Heed the |
---|
383 | * comment in pgtable-2level.h:pte_page(). :-) |
---|
384 | */ |
---|
385 | set_phys_to_machine(page_to_pfn(page), new_mfn); |
---|
386 | |
---|
387 | mcl = npo->mcl + npo->mcl_prod++; |
---|
388 | MULTI_update_va_mapping(mcl, |
---|
389 | (unsigned long)page_address(page), |
---|
390 | pfn_pte_ma(new_mfn, PAGE_KERNEL), |
---|
391 | 0); |
---|
392 | |
---|
393 | mmu = npo->mmu + npo->mmu_prod++; |
---|
394 | mmu->ptr = ((maddr_t)new_mfn << PAGE_SHIFT) | |
---|
395 | MMU_MACHPHYS_UPDATE; |
---|
396 | mmu->val = page_to_pfn(page); |
---|
397 | } |
---|
398 | |
---|
399 | gop = npo->trans + npo->trans_prod++; |
---|
400 | gop->mfn = old_mfn; |
---|
401 | gop->domid = netif->domid; |
---|
402 | gop->ref = req->gref; |
---|
403 | } |
---|
404 | return req->id; |
---|
405 | } |
---|
406 | |
---|
407 | static void netbk_gop_skb(struct sk_buff *skb, |
---|
408 | struct netrx_pending_operations *npo) |
---|
409 | { |
---|
410 | netif_t *netif = netdev_priv(skb->dev); |
---|
411 | int nr_frags = skb_shinfo(skb)->nr_frags; |
---|
412 | int i; |
---|
413 | int extra; |
---|
414 | struct netbk_rx_meta *head_meta, *meta; |
---|
415 | |
---|
416 | head_meta = npo->meta + npo->meta_prod++; |
---|
417 | head_meta->frag.page_offset = skb_shinfo(skb)->gso_type; |
---|
418 | head_meta->frag.size = skb_shinfo(skb)->gso_size; |
---|
419 | extra = !!head_meta->frag.size + 1; |
---|
420 | |
---|
421 | for (i = 0; i < nr_frags; i++) { |
---|
422 | meta = npo->meta + npo->meta_prod++; |
---|
423 | meta->frag = skb_shinfo(skb)->frags[i]; |
---|
424 | meta->id = netbk_gop_frag(netif, meta, i + extra, npo, |
---|
425 | meta->frag.page, |
---|
426 | meta->frag.size, |
---|
427 | meta->frag.page_offset); |
---|
428 | } |
---|
429 | |
---|
430 | /* |
---|
431 | * This must occur at the end to ensure that we don't trash skb_shinfo |
---|
432 | * until we're done. We know that the head doesn't cross a page |
---|
433 | * boundary because such packets get copied in netif_be_start_xmit. |
---|
434 | */ |
---|
435 | head_meta->id = netbk_gop_frag(netif, head_meta, 0, npo, |
---|
436 | virt_to_page(skb->data), |
---|
437 | skb_headlen(skb), |
---|
438 | offset_in_page(skb->data)); |
---|
439 | |
---|
440 | netif->rx.req_cons += nr_frags + extra; |
---|
441 | } |
---|
442 | |
---|
443 | static inline void netbk_free_pages(int nr_frags, struct netbk_rx_meta *meta) |
---|
444 | { |
---|
445 | int i; |
---|
446 | |
---|
447 | for (i = 0; i < nr_frags; i++) |
---|
448 | put_page(meta[i].frag.page); |
---|
449 | } |
---|
450 | |
---|
451 | /* This is a twin to netbk_gop_skb. Assume that netbk_gop_skb was |
---|
452 | used to set up the operations on the top of |
---|
453 | netrx_pending_operations, which have since been done. Check that |
---|
454 | they didn't give any errors and advance over them. */ |
---|
455 | static int netbk_check_gop(int nr_frags, domid_t domid, |
---|
456 | struct netrx_pending_operations *npo) |
---|
457 | { |
---|
458 | multicall_entry_t *mcl; |
---|
459 | gnttab_transfer_t *gop; |
---|
460 | gnttab_copy_t *copy_op; |
---|
461 | int status = NETIF_RSP_OKAY; |
---|
462 | int i; |
---|
463 | |
---|
464 | for (i = 0; i <= nr_frags; i++) { |
---|
465 | if (npo->meta[npo->meta_cons + i].copy) { |
---|
466 | copy_op = npo->copy + npo->copy_cons++; |
---|
467 | if (copy_op->status != GNTST_okay) { |
---|
468 | DPRINTK("Bad status %d from copy to DOM%d.\n", |
---|
469 | copy_op->status, domid); |
---|
470 | status = NETIF_RSP_ERROR; |
---|
471 | } |
---|
472 | } else { |
---|
473 | if (!xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
474 | mcl = npo->mcl + npo->mcl_cons++; |
---|
475 | /* The update_va_mapping() must not fail. */ |
---|
476 | BUG_ON(mcl->result != 0); |
---|
477 | } |
---|
478 | |
---|
479 | gop = npo->trans + npo->trans_cons++; |
---|
480 | /* Check the reassignment error code. */ |
---|
481 | if (gop->status != 0) { |
---|
482 | DPRINTK("Bad status %d from grant transfer to DOM%u\n", |
---|
483 | gop->status, domid); |
---|
484 | /* |
---|
485 | * Page no longer belongs to us unless |
---|
486 | * GNTST_bad_page, but that should be |
---|
487 | * a fatal error anyway. |
---|
488 | */ |
---|
489 | BUG_ON(gop->status == GNTST_bad_page); |
---|
490 | status = NETIF_RSP_ERROR; |
---|
491 | } |
---|
492 | } |
---|
493 | } |
---|
494 | |
---|
495 | return status; |
---|
496 | } |
---|
497 | |
---|
498 | static void netbk_add_frag_responses(netif_t *netif, int status, |
---|
499 | struct netbk_rx_meta *meta, int nr_frags) |
---|
500 | { |
---|
501 | int i; |
---|
502 | unsigned long offset; |
---|
503 | |
---|
504 | for (i = 0; i < nr_frags; i++) { |
---|
505 | int id = meta[i].id; |
---|
506 | int flags = (i == nr_frags - 1) ? 0 : NETRXF_more_data; |
---|
507 | |
---|
508 | if (meta[i].copy) |
---|
509 | offset = 0; |
---|
510 | else |
---|
511 | offset = meta[i].frag.page_offset; |
---|
512 | make_rx_response(netif, id, status, offset, |
---|
513 | meta[i].frag.size, flags); |
---|
514 | } |
---|
515 | } |
---|
516 | |
---|
517 | static void net_rx_action(unsigned long unused) |
---|
518 | { |
---|
519 | netif_t *netif = NULL; |
---|
520 | s8 status; |
---|
521 | u16 id, irq, flags; |
---|
522 | netif_rx_response_t *resp; |
---|
523 | multicall_entry_t *mcl; |
---|
524 | struct sk_buff_head rxq; |
---|
525 | struct sk_buff *skb; |
---|
526 | int notify_nr = 0; |
---|
527 | int ret; |
---|
528 | int nr_frags; |
---|
529 | int count; |
---|
530 | unsigned long offset; |
---|
531 | |
---|
532 | /* |
---|
533 | * Putting hundreds of bytes on the stack is considered rude. |
---|
534 | * Static works because a tasklet can only be on one CPU at any time. |
---|
535 | */ |
---|
536 | static multicall_entry_t rx_mcl[NET_RX_RING_SIZE+3]; |
---|
537 | static mmu_update_t rx_mmu[NET_RX_RING_SIZE]; |
---|
538 | static gnttab_transfer_t grant_trans_op[NET_RX_RING_SIZE]; |
---|
539 | static gnttab_copy_t grant_copy_op[NET_RX_RING_SIZE]; |
---|
540 | static unsigned char rx_notify[NR_IRQS]; |
---|
541 | static u16 notify_list[NET_RX_RING_SIZE]; |
---|
542 | static struct netbk_rx_meta meta[NET_RX_RING_SIZE]; |
---|
543 | |
---|
544 | struct netrx_pending_operations npo = { |
---|
545 | mmu: rx_mmu, |
---|
546 | trans: grant_trans_op, |
---|
547 | copy: grant_copy_op, |
---|
548 | mcl: rx_mcl, |
---|
549 | meta: meta}; |
---|
550 | |
---|
551 | skb_queue_head_init(&rxq); |
---|
552 | |
---|
553 | count = 0; |
---|
554 | |
---|
555 | while ((skb = skb_dequeue(&rx_queue)) != NULL) { |
---|
556 | nr_frags = skb_shinfo(skb)->nr_frags; |
---|
557 | *(int *)skb->cb = nr_frags; |
---|
558 | |
---|
559 | if (!xen_feature(XENFEAT_auto_translated_physmap) && |
---|
560 | !((netif_t *)netdev_priv(skb->dev))->copying_receiver && |
---|
561 | check_mfn(nr_frags + 1)) { |
---|
562 | /* Memory squeeze? Back off for an arbitrary while. */ |
---|
563 | if ( net_ratelimit() ) |
---|
564 | WPRINTK("Memory squeeze in netback " |
---|
565 | "driver.\n"); |
---|
566 | mod_timer(&net_timer, jiffies + HZ); |
---|
567 | skb_queue_head(&rx_queue, skb); |
---|
568 | break; |
---|
569 | } |
---|
570 | |
---|
571 | netbk_gop_skb(skb, &npo); |
---|
572 | |
---|
573 | count += nr_frags + 1; |
---|
574 | |
---|
575 | __skb_queue_tail(&rxq, skb); |
---|
576 | |
---|
577 | /* Filled the batch queue? */ |
---|
578 | if (count + MAX_SKB_FRAGS >= NET_RX_RING_SIZE) |
---|
579 | break; |
---|
580 | } |
---|
581 | |
---|
582 | if (npo.mcl_prod && |
---|
583 | !xen_feature(XENFEAT_auto_translated_physmap)) { |
---|
584 | mcl = npo.mcl + npo.mcl_prod++; |
---|
585 | |
---|
586 | BUG_ON(mcl[-1].op != __HYPERVISOR_update_va_mapping); |
---|
587 | mcl[-1].args[MULTI_UVMFLAGS_INDEX] = UVMF_TLB_FLUSH|UVMF_ALL; |
---|
588 | |
---|
589 | mcl->op = __HYPERVISOR_mmu_update; |
---|
590 | mcl->args[0] = (unsigned long)rx_mmu; |
---|
591 | mcl->args[1] = npo.mmu_prod; |
---|
592 | mcl->args[2] = 0; |
---|
593 | mcl->args[3] = DOMID_SELF; |
---|
594 | } |
---|
595 | |
---|
596 | if (npo.trans_prod) { |
---|
597 | mcl = npo.mcl + npo.mcl_prod++; |
---|
598 | mcl->op = __HYPERVISOR_grant_table_op; |
---|
599 | mcl->args[0] = GNTTABOP_transfer; |
---|
600 | mcl->args[1] = (unsigned long)grant_trans_op; |
---|
601 | mcl->args[2] = npo.trans_prod; |
---|
602 | } |
---|
603 | |
---|
604 | if (npo.copy_prod) { |
---|
605 | mcl = npo.mcl + npo.mcl_prod++; |
---|
606 | mcl->op = __HYPERVISOR_grant_table_op; |
---|
607 | mcl->args[0] = GNTTABOP_copy; |
---|
608 | mcl->args[1] = (unsigned long)grant_copy_op; |
---|
609 | mcl->args[2] = npo.copy_prod; |
---|
610 | } |
---|
611 | |
---|
612 | /* Nothing to do? */ |
---|
613 | if (!npo.mcl_prod) |
---|
614 | return; |
---|
615 | |
---|
616 | BUG_ON(npo.copy_prod > NET_RX_RING_SIZE); |
---|
617 | BUG_ON(npo.mmu_prod > NET_RX_RING_SIZE); |
---|
618 | BUG_ON(npo.trans_prod > NET_RX_RING_SIZE); |
---|
619 | BUG_ON(npo.mcl_prod > NET_RX_RING_SIZE+3); |
---|
620 | BUG_ON(npo.meta_prod > NET_RX_RING_SIZE); |
---|
621 | |
---|
622 | ret = HYPERVISOR_multicall(npo.mcl, npo.mcl_prod); |
---|
623 | BUG_ON(ret != 0); |
---|
624 | |
---|
625 | while ((skb = __skb_dequeue(&rxq)) != NULL) { |
---|
626 | nr_frags = *(int *)skb->cb; |
---|
627 | |
---|
628 | netif = netdev_priv(skb->dev); |
---|
629 | /* We can't rely on skb_release_data to release the |
---|
630 | pages used by fragments for us, since it tries to |
---|
631 | touch the pages in the fraglist. If we're in |
---|
632 | flipping mode, that doesn't work. In copying mode, |
---|
633 | we still have access to all of the pages, and so |
---|
634 | it's safe to let release_data deal with it. */ |
---|
635 | /* (Freeing the fragments is safe since we copy |
---|
636 | non-linear skbs destined for flipping interfaces) */ |
---|
637 | if (!netif->copying_receiver) { |
---|
638 | atomic_set(&(skb_shinfo(skb)->dataref), 1); |
---|
639 | skb_shinfo(skb)->frag_list = NULL; |
---|
640 | skb_shinfo(skb)->nr_frags = 0; |
---|
641 | netbk_free_pages(nr_frags, meta + npo.meta_cons + 1); |
---|
642 | } |
---|
643 | |
---|
644 | netif->stats.tx_bytes += skb->len; |
---|
645 | netif->stats.tx_packets++; |
---|
646 | |
---|
647 | status = netbk_check_gop(nr_frags, netif->domid, &npo); |
---|
648 | |
---|
649 | id = meta[npo.meta_cons].id; |
---|
650 | flags = nr_frags ? NETRXF_more_data : 0; |
---|
651 | |
---|
652 | if (skb->ip_summed == CHECKSUM_HW) /* local packet? */ |
---|
653 | flags |= NETRXF_csum_blank | NETRXF_data_validated; |
---|
654 | else if (skb->proto_data_valid) /* remote but checksummed? */ |
---|
655 | flags |= NETRXF_data_validated; |
---|
656 | |
---|
657 | if (meta[npo.meta_cons].copy) |
---|
658 | offset = 0; |
---|
659 | else |
---|
660 | offset = offset_in_page(skb->data); |
---|
661 | resp = make_rx_response(netif, id, status, offset, |
---|
662 | skb_headlen(skb), flags); |
---|
663 | |
---|
664 | if (meta[npo.meta_cons].frag.size) { |
---|
665 | struct netif_extra_info *gso = |
---|
666 | (struct netif_extra_info *) |
---|
667 | RING_GET_RESPONSE(&netif->rx, |
---|
668 | netif->rx.rsp_prod_pvt++); |
---|
669 | |
---|
670 | resp->flags |= NETRXF_extra_info; |
---|
671 | |
---|
672 | gso->u.gso.size = meta[npo.meta_cons].frag.size; |
---|
673 | gso->u.gso.type = XEN_NETIF_GSO_TYPE_TCPV4; |
---|
674 | gso->u.gso.pad = 0; |
---|
675 | gso->u.gso.features = 0; |
---|
676 | |
---|
677 | gso->type = XEN_NETIF_EXTRA_TYPE_GSO; |
---|
678 | gso->flags = 0; |
---|
679 | } |
---|
680 | |
---|
681 | netbk_add_frag_responses(netif, status, |
---|
682 | meta + npo.meta_cons + 1, |
---|
683 | nr_frags); |
---|
684 | |
---|
685 | RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->rx, ret); |
---|
686 | irq = netif->irq; |
---|
687 | if (ret && !rx_notify[irq]) { |
---|
688 | rx_notify[irq] = 1; |
---|
689 | notify_list[notify_nr++] = irq; |
---|
690 | } |
---|
691 | |
---|
692 | if (netif_queue_stopped(netif->dev) && |
---|
693 | netif_schedulable(netif) && |
---|
694 | !netbk_queue_full(netif)) |
---|
695 | netif_wake_queue(netif->dev); |
---|
696 | |
---|
697 | netif_put(netif); |
---|
698 | dev_kfree_skb(skb); |
---|
699 | npo.meta_cons += nr_frags + 1; |
---|
700 | } |
---|
701 | |
---|
702 | while (notify_nr != 0) { |
---|
703 | irq = notify_list[--notify_nr]; |
---|
704 | rx_notify[irq] = 0; |
---|
705 | notify_remote_via_irq(irq); |
---|
706 | } |
---|
707 | |
---|
708 | /* More work to do? */ |
---|
709 | if (!skb_queue_empty(&rx_queue) && !timer_pending(&net_timer)) |
---|
710 | tasklet_schedule(&net_rx_tasklet); |
---|
711 | #if 0 |
---|
712 | else |
---|
713 | xen_network_done_notify(); |
---|
714 | #endif |
---|
715 | } |
---|
716 | |
---|
717 | static void net_alarm(unsigned long unused) |
---|
718 | { |
---|
719 | tasklet_schedule(&net_rx_tasklet); |
---|
720 | } |
---|
721 | |
---|
722 | struct net_device_stats *netif_be_get_stats(struct net_device *dev) |
---|
723 | { |
---|
724 | netif_t *netif = netdev_priv(dev); |
---|
725 | return &netif->stats; |
---|
726 | } |
---|
727 | |
---|
728 | static int __on_net_schedule_list(netif_t *netif) |
---|
729 | { |
---|
730 | return netif->list.next != NULL; |
---|
731 | } |
---|
732 | |
---|
733 | static void remove_from_net_schedule_list(netif_t *netif) |
---|
734 | { |
---|
735 | spin_lock_irq(&net_schedule_list_lock); |
---|
736 | if (likely(__on_net_schedule_list(netif))) { |
---|
737 | list_del(&netif->list); |
---|
738 | netif->list.next = NULL; |
---|
739 | netif_put(netif); |
---|
740 | } |
---|
741 | spin_unlock_irq(&net_schedule_list_lock); |
---|
742 | } |
---|
743 | |
---|
744 | static void add_to_net_schedule_list_tail(netif_t *netif) |
---|
745 | { |
---|
746 | if (__on_net_schedule_list(netif)) |
---|
747 | return; |
---|
748 | |
---|
749 | spin_lock_irq(&net_schedule_list_lock); |
---|
750 | if (!__on_net_schedule_list(netif) && |
---|
751 | likely(netif_schedulable(netif))) { |
---|
752 | list_add_tail(&netif->list, &net_schedule_list); |
---|
753 | netif_get(netif); |
---|
754 | } |
---|
755 | spin_unlock_irq(&net_schedule_list_lock); |
---|
756 | } |
---|
757 | |
---|
758 | /* |
---|
759 | * Note on CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER: |
---|
760 | * If this driver is pipelining transmit requests then we can be very |
---|
761 | * aggressive in avoiding new-packet notifications -- frontend only needs to |
---|
762 | * send a notification if there are no outstanding unreceived responses. |
---|
763 | * If we may be buffer transmit buffers for any reason then we must be rather |
---|
764 | * more conservative and treat this as the final check for pending work. |
---|
765 | */ |
---|
766 | void netif_schedule_work(netif_t *netif) |
---|
767 | { |
---|
768 | int more_to_do; |
---|
769 | |
---|
770 | #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER |
---|
771 | more_to_do = RING_HAS_UNCONSUMED_REQUESTS(&netif->tx); |
---|
772 | #else |
---|
773 | RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do); |
---|
774 | #endif |
---|
775 | |
---|
776 | if (more_to_do) { |
---|
777 | add_to_net_schedule_list_tail(netif); |
---|
778 | maybe_schedule_tx_action(); |
---|
779 | } |
---|
780 | } |
---|
781 | |
---|
782 | void netif_deschedule_work(netif_t *netif) |
---|
783 | { |
---|
784 | remove_from_net_schedule_list(netif); |
---|
785 | } |
---|
786 | |
---|
787 | |
---|
788 | static void tx_add_credit(netif_t *netif) |
---|
789 | { |
---|
790 | unsigned long max_burst, max_credit; |
---|
791 | |
---|
792 | /* |
---|
793 | * Allow a burst big enough to transmit a jumbo packet of up to 128kB. |
---|
794 | * Otherwise the interface can seize up due to insufficient credit. |
---|
795 | */ |
---|
796 | max_burst = RING_GET_REQUEST(&netif->tx, netif->tx.req_cons)->size; |
---|
797 | max_burst = min(max_burst, 131072UL); |
---|
798 | max_burst = max(max_burst, netif->credit_bytes); |
---|
799 | |
---|
800 | /* Take care that adding a new chunk of credit doesn't wrap to zero. */ |
---|
801 | max_credit = netif->remaining_credit + netif->credit_bytes; |
---|
802 | if (max_credit < netif->remaining_credit) |
---|
803 | max_credit = ULONG_MAX; /* wrapped: clamp to ULONG_MAX */ |
---|
804 | |
---|
805 | netif->remaining_credit = min(max_credit, max_burst); |
---|
806 | } |
---|
807 | |
---|
808 | static void tx_credit_callback(unsigned long data) |
---|
809 | { |
---|
810 | netif_t *netif = (netif_t *)data; |
---|
811 | tx_add_credit(netif); |
---|
812 | netif_schedule_work(netif); |
---|
813 | } |
---|
814 | |
---|
815 | inline static void net_tx_action_dealloc(void) |
---|
816 | { |
---|
817 | gnttab_unmap_grant_ref_t *gop; |
---|
818 | u16 pending_idx; |
---|
819 | PEND_RING_IDX dc, dp; |
---|
820 | netif_t *netif; |
---|
821 | int ret; |
---|
822 | |
---|
823 | dc = dealloc_cons; |
---|
824 | dp = dealloc_prod; |
---|
825 | |
---|
826 | /* Ensure we see all indexes enqueued by netif_idx_release(). */ |
---|
827 | smp_rmb(); |
---|
828 | |
---|
829 | /* |
---|
830 | * Free up any grants we have finished using |
---|
831 | */ |
---|
832 | gop = tx_unmap_ops; |
---|
833 | while (dc != dp) { |
---|
834 | pending_idx = dealloc_ring[MASK_PEND_IDX(dc++)]; |
---|
835 | gnttab_set_unmap_op(gop, idx_to_kaddr(pending_idx), |
---|
836 | GNTMAP_host_map, |
---|
837 | grant_tx_handle[pending_idx]); |
---|
838 | gop++; |
---|
839 | } |
---|
840 | ret = HYPERVISOR_grant_table_op( |
---|
841 | GNTTABOP_unmap_grant_ref, tx_unmap_ops, gop - tx_unmap_ops); |
---|
842 | BUG_ON(ret); |
---|
843 | |
---|
844 | while (dealloc_cons != dp) { |
---|
845 | pending_idx = dealloc_ring[MASK_PEND_IDX(dealloc_cons++)]; |
---|
846 | |
---|
847 | netif = pending_tx_info[pending_idx].netif; |
---|
848 | |
---|
849 | make_tx_response(netif, &pending_tx_info[pending_idx].req, |
---|
850 | NETIF_RSP_OKAY); |
---|
851 | |
---|
852 | pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx; |
---|
853 | |
---|
854 | netif_put(netif); |
---|
855 | } |
---|
856 | } |
---|
857 | |
---|
858 | static void netbk_tx_err(netif_t *netif, netif_tx_request_t *txp, RING_IDX end) |
---|
859 | { |
---|
860 | RING_IDX cons = netif->tx.req_cons; |
---|
861 | |
---|
862 | do { |
---|
863 | make_tx_response(netif, txp, NETIF_RSP_ERROR); |
---|
864 | if (cons >= end) |
---|
865 | break; |
---|
866 | txp = RING_GET_REQUEST(&netif->tx, cons++); |
---|
867 | } while (1); |
---|
868 | netif->tx.req_cons = cons; |
---|
869 | netif_schedule_work(netif); |
---|
870 | netif_put(netif); |
---|
871 | } |
---|
872 | |
---|
873 | static int netbk_count_requests(netif_t *netif, netif_tx_request_t *first, |
---|
874 | netif_tx_request_t *txp, int work_to_do) |
---|
875 | { |
---|
876 | RING_IDX cons = netif->tx.req_cons; |
---|
877 | int frags = 0; |
---|
878 | |
---|
879 | if (!(first->flags & NETTXF_more_data)) |
---|
880 | return 0; |
---|
881 | |
---|
882 | do { |
---|
883 | if (frags >= work_to_do) { |
---|
884 | DPRINTK("Need more frags\n"); |
---|
885 | return -frags; |
---|
886 | } |
---|
887 | |
---|
888 | if (unlikely(frags >= MAX_SKB_FRAGS)) { |
---|
889 | DPRINTK("Too many frags\n"); |
---|
890 | return -frags; |
---|
891 | } |
---|
892 | |
---|
893 | memcpy(txp, RING_GET_REQUEST(&netif->tx, cons + frags), |
---|
894 | sizeof(*txp)); |
---|
895 | if (txp->size > first->size) { |
---|
896 | DPRINTK("Frags galore\n"); |
---|
897 | return -frags; |
---|
898 | } |
---|
899 | |
---|
900 | first->size -= txp->size; |
---|
901 | frags++; |
---|
902 | |
---|
903 | if (unlikely((txp->offset + txp->size) > PAGE_SIZE)) { |
---|
904 | DPRINTK("txp->offset: %x, size: %u\n", |
---|
905 | txp->offset, txp->size); |
---|
906 | return -frags; |
---|
907 | } |
---|
908 | } while ((txp++)->flags & NETTXF_more_data); |
---|
909 | |
---|
910 | return frags; |
---|
911 | } |
---|
912 | |
---|
913 | static gnttab_map_grant_ref_t *netbk_get_requests(netif_t *netif, |
---|
914 | struct sk_buff *skb, |
---|
915 | netif_tx_request_t *txp, |
---|
916 | gnttab_map_grant_ref_t *mop) |
---|
917 | { |
---|
918 | struct skb_shared_info *shinfo = skb_shinfo(skb); |
---|
919 | skb_frag_t *frags = shinfo->frags; |
---|
920 | unsigned long pending_idx = *((u16 *)skb->data); |
---|
921 | int i, start; |
---|
922 | |
---|
923 | /* Skip first skb fragment if it is on same page as header fragment. */ |
---|
924 | start = ((unsigned long)shinfo->frags[0].page == pending_idx); |
---|
925 | |
---|
926 | for (i = start; i < shinfo->nr_frags; i++, txp++) { |
---|
927 | pending_idx = pending_ring[MASK_PEND_IDX(pending_cons++)]; |
---|
928 | |
---|
929 | gnttab_set_map_op(mop++, idx_to_kaddr(pending_idx), |
---|
930 | GNTMAP_host_map | GNTMAP_readonly, |
---|
931 | txp->gref, netif->domid); |
---|
932 | |
---|
933 | memcpy(&pending_tx_info[pending_idx].req, txp, sizeof(*txp)); |
---|
934 | netif_get(netif); |
---|
935 | pending_tx_info[pending_idx].netif = netif; |
---|
936 | frags[i].page = (void *)pending_idx; |
---|
937 | } |
---|
938 | |
---|
939 | return mop; |
---|
940 | } |
---|
941 | |
---|
942 | static int netbk_tx_check_mop(struct sk_buff *skb, |
---|
943 | gnttab_map_grant_ref_t **mopp) |
---|
944 | { |
---|
945 | gnttab_map_grant_ref_t *mop = *mopp; |
---|
946 | int pending_idx = *((u16 *)skb->data); |
---|
947 | netif_t *netif = pending_tx_info[pending_idx].netif; |
---|
948 | netif_tx_request_t *txp; |
---|
949 | struct skb_shared_info *shinfo = skb_shinfo(skb); |
---|
950 | int nr_frags = shinfo->nr_frags; |
---|
951 | int i, err, start; |
---|
952 | |
---|
953 | /* Check status of header. */ |
---|
954 | err = mop->status; |
---|
955 | if (unlikely(err)) { |
---|
956 | txp = &pending_tx_info[pending_idx].req; |
---|
957 | make_tx_response(netif, txp, NETIF_RSP_ERROR); |
---|
958 | pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx; |
---|
959 | netif_put(netif); |
---|
960 | } else { |
---|
961 | set_phys_to_machine( |
---|
962 | __pa(idx_to_kaddr(pending_idx)) >> PAGE_SHIFT, |
---|
963 | FOREIGN_FRAME(mop->dev_bus_addr >> PAGE_SHIFT)); |
---|
964 | grant_tx_handle[pending_idx] = mop->handle; |
---|
965 | } |
---|
966 | |
---|
967 | /* Skip first skb fragment if it is on same page as header fragment. */ |
---|
968 | start = ((unsigned long)shinfo->frags[0].page == pending_idx); |
---|
969 | |
---|
970 | for (i = start; i < nr_frags; i++) { |
---|
971 | int j, newerr; |
---|
972 | |
---|
973 | pending_idx = (unsigned long)shinfo->frags[i].page; |
---|
974 | |
---|
975 | /* Check error status: if okay then remember grant handle. */ |
---|
976 | newerr = (++mop)->status; |
---|
977 | if (likely(!newerr)) { |
---|
978 | set_phys_to_machine( |
---|
979 | __pa(idx_to_kaddr(pending_idx))>>PAGE_SHIFT, |
---|
980 | FOREIGN_FRAME(mop->dev_bus_addr>>PAGE_SHIFT)); |
---|
981 | grant_tx_handle[pending_idx] = mop->handle; |
---|
982 | /* Had a previous error? Invalidate this fragment. */ |
---|
983 | if (unlikely(err)) |
---|
984 | netif_idx_release(pending_idx); |
---|
985 | continue; |
---|
986 | } |
---|
987 | |
---|
988 | /* Error on this fragment: respond to client with an error. */ |
---|
989 | txp = &pending_tx_info[pending_idx].req; |
---|
990 | make_tx_response(netif, txp, NETIF_RSP_ERROR); |
---|
991 | pending_ring[MASK_PEND_IDX(pending_prod++)] = pending_idx; |
---|
992 | netif_put(netif); |
---|
993 | |
---|
994 | /* Not the first error? Preceding frags already invalidated. */ |
---|
995 | if (err) |
---|
996 | continue; |
---|
997 | |
---|
998 | /* First error: invalidate header and preceding fragments. */ |
---|
999 | pending_idx = *((u16 *)skb->data); |
---|
1000 | netif_idx_release(pending_idx); |
---|
1001 | for (j = start; j < i; j++) { |
---|
1002 | pending_idx = (unsigned long)shinfo->frags[i].page; |
---|
1003 | netif_idx_release(pending_idx); |
---|
1004 | } |
---|
1005 | |
---|
1006 | /* Remember the error: invalidate all subsequent fragments. */ |
---|
1007 | err = newerr; |
---|
1008 | } |
---|
1009 | |
---|
1010 | *mopp = mop + 1; |
---|
1011 | return err; |
---|
1012 | } |
---|
1013 | |
---|
1014 | static void netbk_fill_frags(struct sk_buff *skb) |
---|
1015 | { |
---|
1016 | struct skb_shared_info *shinfo = skb_shinfo(skb); |
---|
1017 | int nr_frags = shinfo->nr_frags; |
---|
1018 | int i; |
---|
1019 | |
---|
1020 | for (i = 0; i < nr_frags; i++) { |
---|
1021 | skb_frag_t *frag = shinfo->frags + i; |
---|
1022 | netif_tx_request_t *txp; |
---|
1023 | unsigned long pending_idx; |
---|
1024 | |
---|
1025 | pending_idx = (unsigned long)frag->page; |
---|
1026 | txp = &pending_tx_info[pending_idx].req; |
---|
1027 | frag->page = virt_to_page(idx_to_kaddr(pending_idx)); |
---|
1028 | frag->size = txp->size; |
---|
1029 | frag->page_offset = txp->offset; |
---|
1030 | |
---|
1031 | skb->len += txp->size; |
---|
1032 | skb->data_len += txp->size; |
---|
1033 | skb->truesize += txp->size; |
---|
1034 | } |
---|
1035 | } |
---|
1036 | |
---|
1037 | int netbk_get_extras(netif_t *netif, struct netif_extra_info *extras, |
---|
1038 | int work_to_do) |
---|
1039 | { |
---|
1040 | struct netif_extra_info extra; |
---|
1041 | RING_IDX cons = netif->tx.req_cons; |
---|
1042 | |
---|
1043 | do { |
---|
1044 | if (unlikely(work_to_do-- <= 0)) { |
---|
1045 | DPRINTK("Missing extra info\n"); |
---|
1046 | return -EBADR; |
---|
1047 | } |
---|
1048 | |
---|
1049 | memcpy(&extra, RING_GET_REQUEST(&netif->tx, cons), |
---|
1050 | sizeof(extra)); |
---|
1051 | if (unlikely(!extra.type || |
---|
1052 | extra.type >= XEN_NETIF_EXTRA_TYPE_MAX)) { |
---|
1053 | netif->tx.req_cons = ++cons; |
---|
1054 | DPRINTK("Invalid extra type: %d\n", extra.type); |
---|
1055 | return -EINVAL; |
---|
1056 | } |
---|
1057 | |
---|
1058 | memcpy(&extras[extra.type - 1], &extra, sizeof(extra)); |
---|
1059 | netif->tx.req_cons = ++cons; |
---|
1060 | } while (extra.flags & XEN_NETIF_EXTRA_FLAG_MORE); |
---|
1061 | |
---|
1062 | return work_to_do; |
---|
1063 | } |
---|
1064 | |
---|
1065 | static int netbk_set_skb_gso(struct sk_buff *skb, struct netif_extra_info *gso) |
---|
1066 | { |
---|
1067 | if (!gso->u.gso.size) { |
---|
1068 | DPRINTK("GSO size must not be zero.\n"); |
---|
1069 | return -EINVAL; |
---|
1070 | } |
---|
1071 | |
---|
1072 | /* Currently only TCPv4 S.O. is supported. */ |
---|
1073 | if (gso->u.gso.type != XEN_NETIF_GSO_TYPE_TCPV4) { |
---|
1074 | DPRINTK("Bad GSO type %d.\n", gso->u.gso.type); |
---|
1075 | return -EINVAL; |
---|
1076 | } |
---|
1077 | |
---|
1078 | skb_shinfo(skb)->gso_size = gso->u.gso.size; |
---|
1079 | skb_shinfo(skb)->gso_type = SKB_GSO_TCPV4; |
---|
1080 | |
---|
1081 | /* Header must be checked, and gso_segs computed. */ |
---|
1082 | skb_shinfo(skb)->gso_type |= SKB_GSO_DODGY; |
---|
1083 | skb_shinfo(skb)->gso_segs = 0; |
---|
1084 | |
---|
1085 | return 0; |
---|
1086 | } |
---|
1087 | |
---|
1088 | /* Called after netfront has transmitted */ |
---|
1089 | static void net_tx_action(unsigned long unused) |
---|
1090 | { |
---|
1091 | struct list_head *ent; |
---|
1092 | struct sk_buff *skb; |
---|
1093 | netif_t *netif; |
---|
1094 | netif_tx_request_t txreq; |
---|
1095 | netif_tx_request_t txfrags[MAX_SKB_FRAGS]; |
---|
1096 | struct netif_extra_info extras[XEN_NETIF_EXTRA_TYPE_MAX - 1]; |
---|
1097 | u16 pending_idx; |
---|
1098 | RING_IDX i; |
---|
1099 | gnttab_map_grant_ref_t *mop; |
---|
1100 | unsigned int data_len; |
---|
1101 | int ret, work_to_do; |
---|
1102 | |
---|
1103 | if (dealloc_cons != dealloc_prod) |
---|
1104 | net_tx_action_dealloc(); |
---|
1105 | |
---|
1106 | mop = tx_map_ops; |
---|
1107 | while (((NR_PENDING_REQS + MAX_SKB_FRAGS) < MAX_PENDING_REQS) && |
---|
1108 | !list_empty(&net_schedule_list)) { |
---|
1109 | /* Get a netif from the list with work to do. */ |
---|
1110 | ent = net_schedule_list.next; |
---|
1111 | netif = list_entry(ent, netif_t, list); |
---|
1112 | netif_get(netif); |
---|
1113 | remove_from_net_schedule_list(netif); |
---|
1114 | |
---|
1115 | RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, work_to_do); |
---|
1116 | if (!work_to_do) { |
---|
1117 | netif_put(netif); |
---|
1118 | continue; |
---|
1119 | } |
---|
1120 | |
---|
1121 | i = netif->tx.req_cons; |
---|
1122 | rmb(); /* Ensure that we see the request before we copy it. */ |
---|
1123 | memcpy(&txreq, RING_GET_REQUEST(&netif->tx, i), sizeof(txreq)); |
---|
1124 | |
---|
1125 | /* Credit-based scheduling. */ |
---|
1126 | if (txreq.size > netif->remaining_credit) { |
---|
1127 | unsigned long now = jiffies; |
---|
1128 | unsigned long next_credit = |
---|
1129 | netif->credit_timeout.expires + |
---|
1130 | msecs_to_jiffies(netif->credit_usec / 1000); |
---|
1131 | |
---|
1132 | /* Timer could already be pending in rare cases. */ |
---|
1133 | if (timer_pending(&netif->credit_timeout)) { |
---|
1134 | netif_put(netif); |
---|
1135 | continue; |
---|
1136 | } |
---|
1137 | |
---|
1138 | /* Passed the point where we can replenish credit? */ |
---|
1139 | if (time_after_eq(now, next_credit)) { |
---|
1140 | netif->credit_timeout.expires = now; |
---|
1141 | tx_add_credit(netif); |
---|
1142 | } |
---|
1143 | |
---|
1144 | /* Still too big to send right now? Set a callback. */ |
---|
1145 | if (txreq.size > netif->remaining_credit) { |
---|
1146 | netif->credit_timeout.data = |
---|
1147 | (unsigned long)netif; |
---|
1148 | netif->credit_timeout.function = |
---|
1149 | tx_credit_callback; |
---|
1150 | __mod_timer(&netif->credit_timeout, |
---|
1151 | next_credit); |
---|
1152 | netif_put(netif); |
---|
1153 | continue; |
---|
1154 | } |
---|
1155 | } |
---|
1156 | netif->remaining_credit -= txreq.size; |
---|
1157 | |
---|
1158 | work_to_do--; |
---|
1159 | netif->tx.req_cons = ++i; |
---|
1160 | |
---|
1161 | memset(extras, 0, sizeof(extras)); |
---|
1162 | if (txreq.flags & NETTXF_extra_info) { |
---|
1163 | work_to_do = netbk_get_extras(netif, extras, |
---|
1164 | work_to_do); |
---|
1165 | i = netif->tx.req_cons; |
---|
1166 | if (unlikely(work_to_do < 0)) { |
---|
1167 | netbk_tx_err(netif, &txreq, i); |
---|
1168 | continue; |
---|
1169 | } |
---|
1170 | } |
---|
1171 | |
---|
1172 | ret = netbk_count_requests(netif, &txreq, txfrags, work_to_do); |
---|
1173 | if (unlikely(ret < 0)) { |
---|
1174 | netbk_tx_err(netif, &txreq, i - ret); |
---|
1175 | continue; |
---|
1176 | } |
---|
1177 | i += ret; |
---|
1178 | |
---|
1179 | if (unlikely(txreq.size < ETH_HLEN)) { |
---|
1180 | DPRINTK("Bad packet size: %d\n", txreq.size); |
---|
1181 | netbk_tx_err(netif, &txreq, i); |
---|
1182 | continue; |
---|
1183 | } |
---|
1184 | |
---|
1185 | /* No crossing a page as the payload mustn't fragment. */ |
---|
1186 | if (unlikely((txreq.offset + txreq.size) > PAGE_SIZE)) { |
---|
1187 | DPRINTK("txreq.offset: %x, size: %u, end: %lu\n", |
---|
1188 | txreq.offset, txreq.size, |
---|
1189 | (txreq.offset &~PAGE_MASK) + txreq.size); |
---|
1190 | netbk_tx_err(netif, &txreq, i); |
---|
1191 | continue; |
---|
1192 | } |
---|
1193 | |
---|
1194 | pending_idx = pending_ring[MASK_PEND_IDX(pending_cons)]; |
---|
1195 | |
---|
1196 | data_len = (txreq.size > PKT_PROT_LEN && |
---|
1197 | ret < MAX_SKB_FRAGS) ? |
---|
1198 | PKT_PROT_LEN : txreq.size; |
---|
1199 | |
---|
1200 | skb = alloc_skb(data_len + 16 + NET_IP_ALIGN, |
---|
1201 | GFP_ATOMIC | __GFP_NOWARN); |
---|
1202 | if (unlikely(skb == NULL)) { |
---|
1203 | DPRINTK("Can't allocate a skb in start_xmit.\n"); |
---|
1204 | netbk_tx_err(netif, &txreq, i); |
---|
1205 | break; |
---|
1206 | } |
---|
1207 | |
---|
1208 | /* Packets passed to netif_rx() must have some headroom. */ |
---|
1209 | skb_reserve(skb, 16 + NET_IP_ALIGN); |
---|
1210 | |
---|
1211 | if (extras[XEN_NETIF_EXTRA_TYPE_GSO - 1].type) { |
---|
1212 | struct netif_extra_info *gso; |
---|
1213 | gso = &extras[XEN_NETIF_EXTRA_TYPE_GSO - 1]; |
---|
1214 | |
---|
1215 | if (netbk_set_skb_gso(skb, gso)) { |
---|
1216 | kfree_skb(skb); |
---|
1217 | netbk_tx_err(netif, &txreq, i); |
---|
1218 | continue; |
---|
1219 | } |
---|
1220 | } |
---|
1221 | |
---|
1222 | gnttab_set_map_op(mop, idx_to_kaddr(pending_idx), |
---|
1223 | GNTMAP_host_map | GNTMAP_readonly, |
---|
1224 | txreq.gref, netif->domid); |
---|
1225 | mop++; |
---|
1226 | |
---|
1227 | memcpy(&pending_tx_info[pending_idx].req, |
---|
1228 | &txreq, sizeof(txreq)); |
---|
1229 | pending_tx_info[pending_idx].netif = netif; |
---|
1230 | *((u16 *)skb->data) = pending_idx; |
---|
1231 | |
---|
1232 | __skb_put(skb, data_len); |
---|
1233 | |
---|
1234 | skb_shinfo(skb)->nr_frags = ret; |
---|
1235 | if (data_len < txreq.size) { |
---|
1236 | skb_shinfo(skb)->nr_frags++; |
---|
1237 | skb_shinfo(skb)->frags[0].page = |
---|
1238 | (void *)(unsigned long)pending_idx; |
---|
1239 | } else { |
---|
1240 | /* Discriminate from any valid pending_idx value. */ |
---|
1241 | skb_shinfo(skb)->frags[0].page = (void *)~0UL; |
---|
1242 | } |
---|
1243 | |
---|
1244 | __skb_queue_tail(&tx_queue, skb); |
---|
1245 | |
---|
1246 | pending_cons++; |
---|
1247 | |
---|
1248 | mop = netbk_get_requests(netif, skb, txfrags, mop); |
---|
1249 | |
---|
1250 | netif->tx.req_cons = i; |
---|
1251 | netif_schedule_work(netif); |
---|
1252 | |
---|
1253 | if ((mop - tx_map_ops) >= ARRAY_SIZE(tx_map_ops)) |
---|
1254 | break; |
---|
1255 | } |
---|
1256 | |
---|
1257 | if (mop == tx_map_ops) |
---|
1258 | return; |
---|
1259 | |
---|
1260 | ret = HYPERVISOR_grant_table_op( |
---|
1261 | GNTTABOP_map_grant_ref, tx_map_ops, mop - tx_map_ops); |
---|
1262 | BUG_ON(ret); |
---|
1263 | |
---|
1264 | mop = tx_map_ops; |
---|
1265 | while ((skb = __skb_dequeue(&tx_queue)) != NULL) { |
---|
1266 | netif_tx_request_t *txp; |
---|
1267 | |
---|
1268 | pending_idx = *((u16 *)skb->data); |
---|
1269 | netif = pending_tx_info[pending_idx].netif; |
---|
1270 | txp = &pending_tx_info[pending_idx].req; |
---|
1271 | |
---|
1272 | /* Check the remap error code. */ |
---|
1273 | if (unlikely(netbk_tx_check_mop(skb, &mop))) { |
---|
1274 | DPRINTK("netback grant failed.\n"); |
---|
1275 | skb_shinfo(skb)->nr_frags = 0; |
---|
1276 | kfree_skb(skb); |
---|
1277 | continue; |
---|
1278 | } |
---|
1279 | |
---|
1280 | data_len = skb->len; |
---|
1281 | memcpy(skb->data, |
---|
1282 | (void *)(idx_to_kaddr(pending_idx)|txp->offset), |
---|
1283 | data_len); |
---|
1284 | if (data_len < txp->size) { |
---|
1285 | /* Append the packet payload as a fragment. */ |
---|
1286 | txp->offset += data_len; |
---|
1287 | txp->size -= data_len; |
---|
1288 | } else { |
---|
1289 | /* Schedule a response immediately. */ |
---|
1290 | netif_idx_release(pending_idx); |
---|
1291 | } |
---|
1292 | |
---|
1293 | /* |
---|
1294 | * Old frontends do not assert data_validated but we |
---|
1295 | * can infer it from csum_blank so test both flags. |
---|
1296 | */ |
---|
1297 | if (txp->flags & (NETTXF_data_validated|NETTXF_csum_blank)) { |
---|
1298 | skb->ip_summed = CHECKSUM_UNNECESSARY; |
---|
1299 | skb->proto_data_valid = 1; |
---|
1300 | } else { |
---|
1301 | skb->ip_summed = CHECKSUM_NONE; |
---|
1302 | skb->proto_data_valid = 0; |
---|
1303 | } |
---|
1304 | skb->proto_csum_blank = !!(txp->flags & NETTXF_csum_blank); |
---|
1305 | |
---|
1306 | netbk_fill_frags(skb); |
---|
1307 | |
---|
1308 | skb->dev = netif->dev; |
---|
1309 | skb->protocol = eth_type_trans(skb, skb->dev); |
---|
1310 | |
---|
1311 | netif->stats.rx_bytes += skb->len; |
---|
1312 | netif->stats.rx_packets++; |
---|
1313 | |
---|
1314 | netif_rx(skb); |
---|
1315 | netif->dev->last_rx = jiffies; |
---|
1316 | } |
---|
1317 | } |
---|
1318 | |
---|
1319 | static void netif_idx_release(u16 pending_idx) |
---|
1320 | { |
---|
1321 | static DEFINE_SPINLOCK(_lock); |
---|
1322 | unsigned long flags; |
---|
1323 | |
---|
1324 | spin_lock_irqsave(&_lock, flags); |
---|
1325 | dealloc_ring[MASK_PEND_IDX(dealloc_prod)] = pending_idx; |
---|
1326 | /* Sync with net_tx_action_dealloc: insert idx /then/ incr producer. */ |
---|
1327 | smp_wmb(); |
---|
1328 | dealloc_prod++; |
---|
1329 | spin_unlock_irqrestore(&_lock, flags); |
---|
1330 | |
---|
1331 | tasklet_schedule(&net_tx_tasklet); |
---|
1332 | } |
---|
1333 | |
---|
1334 | static void netif_page_release(struct page *page) |
---|
1335 | { |
---|
1336 | /* Ready for next use. */ |
---|
1337 | init_page_count(page); |
---|
1338 | |
---|
1339 | netif_idx_release(netif_page_index(page)); |
---|
1340 | } |
---|
1341 | |
---|
1342 | irqreturn_t netif_be_int(int irq, void *dev_id, struct pt_regs *regs) |
---|
1343 | { |
---|
1344 | netif_t *netif = dev_id; |
---|
1345 | |
---|
1346 | add_to_net_schedule_list_tail(netif); |
---|
1347 | maybe_schedule_tx_action(); |
---|
1348 | |
---|
1349 | if (netif_schedulable(netif) && !netbk_queue_full(netif)) |
---|
1350 | netif_wake_queue(netif->dev); |
---|
1351 | |
---|
1352 | return IRQ_HANDLED; |
---|
1353 | } |
---|
1354 | |
---|
1355 | static void make_tx_response(netif_t *netif, |
---|
1356 | netif_tx_request_t *txp, |
---|
1357 | s8 st) |
---|
1358 | { |
---|
1359 | RING_IDX i = netif->tx.rsp_prod_pvt; |
---|
1360 | netif_tx_response_t *resp; |
---|
1361 | int notify; |
---|
1362 | |
---|
1363 | resp = RING_GET_RESPONSE(&netif->tx, i); |
---|
1364 | resp->id = txp->id; |
---|
1365 | resp->status = st; |
---|
1366 | |
---|
1367 | if (txp->flags & NETTXF_extra_info) |
---|
1368 | RING_GET_RESPONSE(&netif->tx, ++i)->status = NETIF_RSP_NULL; |
---|
1369 | |
---|
1370 | netif->tx.rsp_prod_pvt = ++i; |
---|
1371 | RING_PUSH_RESPONSES_AND_CHECK_NOTIFY(&netif->tx, notify); |
---|
1372 | if (notify) |
---|
1373 | notify_remote_via_irq(netif->irq); |
---|
1374 | |
---|
1375 | #ifdef CONFIG_XEN_NETDEV_PIPELINED_TRANSMITTER |
---|
1376 | if (i == netif->tx.req_cons) { |
---|
1377 | int more_to_do; |
---|
1378 | RING_FINAL_CHECK_FOR_REQUESTS(&netif->tx, more_to_do); |
---|
1379 | if (more_to_do) |
---|
1380 | add_to_net_schedule_list_tail(netif); |
---|
1381 | } |
---|
1382 | #endif |
---|
1383 | } |
---|
1384 | |
---|
1385 | static netif_rx_response_t *make_rx_response(netif_t *netif, |
---|
1386 | u16 id, |
---|
1387 | s8 st, |
---|
1388 | u16 offset, |
---|
1389 | u16 size, |
---|
1390 | u16 flags) |
---|
1391 | { |
---|
1392 | RING_IDX i = netif->rx.rsp_prod_pvt; |
---|
1393 | netif_rx_response_t *resp; |
---|
1394 | |
---|
1395 | resp = RING_GET_RESPONSE(&netif->rx, i); |
---|
1396 | resp->offset = offset; |
---|
1397 | resp->flags = flags; |
---|
1398 | resp->id = id; |
---|
1399 | resp->status = (s16)size; |
---|
1400 | if (st < 0) |
---|
1401 | resp->status = (s16)st; |
---|
1402 | |
---|
1403 | netif->rx.rsp_prod_pvt = ++i; |
---|
1404 | |
---|
1405 | return resp; |
---|
1406 | } |
---|
1407 | |
---|
1408 | #ifdef NETBE_DEBUG_INTERRUPT |
---|
1409 | static irqreturn_t netif_be_dbg(int irq, void *dev_id, struct pt_regs *regs) |
---|
1410 | { |
---|
1411 | struct list_head *ent; |
---|
1412 | netif_t *netif; |
---|
1413 | int i = 0; |
---|
1414 | |
---|
1415 | printk(KERN_ALERT "netif_schedule_list:\n"); |
---|
1416 | spin_lock_irq(&net_schedule_list_lock); |
---|
1417 | |
---|
1418 | list_for_each (ent, &net_schedule_list) { |
---|
1419 | netif = list_entry(ent, netif_t, list); |
---|
1420 | printk(KERN_ALERT " %d: private(rx_req_cons=%08x " |
---|
1421 | "rx_resp_prod=%08x\n", |
---|
1422 | i, netif->rx.req_cons, netif->rx.rsp_prod_pvt); |
---|
1423 | printk(KERN_ALERT " tx_req_cons=%08x tx_resp_prod=%08x)\n", |
---|
1424 | netif->tx.req_cons, netif->tx.rsp_prod_pvt); |
---|
1425 | printk(KERN_ALERT " shared(rx_req_prod=%08x " |
---|
1426 | "rx_resp_prod=%08x\n", |
---|
1427 | netif->rx.sring->req_prod, netif->rx.sring->rsp_prod); |
---|
1428 | printk(KERN_ALERT " rx_event=%08x tx_req_prod=%08x\n", |
---|
1429 | netif->rx.sring->rsp_event, netif->tx.sring->req_prod); |
---|
1430 | printk(KERN_ALERT " tx_resp_prod=%08x, tx_event=%08x)\n", |
---|
1431 | netif->tx.sring->rsp_prod, netif->tx.sring->rsp_event); |
---|
1432 | i++; |
---|
1433 | } |
---|
1434 | |
---|
1435 | spin_unlock_irq(&net_schedule_list_lock); |
---|
1436 | printk(KERN_ALERT " ** End of netif_schedule_list **\n"); |
---|
1437 | |
---|
1438 | return IRQ_HANDLED; |
---|
1439 | } |
---|
1440 | #endif |
---|
1441 | |
---|
1442 | static int __init netback_init(void) |
---|
1443 | { |
---|
1444 | int i; |
---|
1445 | struct page *page; |
---|
1446 | |
---|
1447 | if (!is_running_on_xen()) |
---|
1448 | return -ENODEV; |
---|
1449 | |
---|
1450 | /* We can increase reservation by this much in net_rx_action(). */ |
---|
1451 | balloon_update_driver_allowance(NET_RX_RING_SIZE); |
---|
1452 | |
---|
1453 | skb_queue_head_init(&rx_queue); |
---|
1454 | skb_queue_head_init(&tx_queue); |
---|
1455 | |
---|
1456 | init_timer(&net_timer); |
---|
1457 | net_timer.data = 0; |
---|
1458 | net_timer.function = net_alarm; |
---|
1459 | |
---|
1460 | mmap_pages = alloc_empty_pages_and_pagevec(MAX_PENDING_REQS); |
---|
1461 | if (mmap_pages == NULL) { |
---|
1462 | printk("%s: out of memory\n", __FUNCTION__); |
---|
1463 | return -ENOMEM; |
---|
1464 | } |
---|
1465 | |
---|
1466 | for (i = 0; i < MAX_PENDING_REQS; i++) { |
---|
1467 | page = mmap_pages[i]; |
---|
1468 | SetPageForeign(page, netif_page_release); |
---|
1469 | netif_page_index(page) = i; |
---|
1470 | } |
---|
1471 | |
---|
1472 | pending_cons = 0; |
---|
1473 | pending_prod = MAX_PENDING_REQS; |
---|
1474 | for (i = 0; i < MAX_PENDING_REQS; i++) |
---|
1475 | pending_ring[i] = i; |
---|
1476 | |
---|
1477 | spin_lock_init(&net_schedule_list_lock); |
---|
1478 | INIT_LIST_HEAD(&net_schedule_list); |
---|
1479 | |
---|
1480 | netif_xenbus_init(); |
---|
1481 | |
---|
1482 | #ifdef NETBE_DEBUG_INTERRUPT |
---|
1483 | (void)bind_virq_to_irqhandler(VIRQ_DEBUG, |
---|
1484 | 0, |
---|
1485 | netif_be_dbg, |
---|
1486 | SA_SHIRQ, |
---|
1487 | "net-be-dbg", |
---|
1488 | &netif_be_dbg); |
---|
1489 | #endif |
---|
1490 | |
---|
1491 | return 0; |
---|
1492 | } |
---|
1493 | |
---|
1494 | module_init(netback_init); |
---|
1495 | |
---|
1496 | MODULE_LICENSE("Dual BSD/GPL"); |
---|