source: trunk/packages/xen-3.1/xen-3.1/linux-2.6-xen-sparse/drivers/xen/gntdev/gntdev.c @ 34

/******************************************************************************
 * gntdev.c
 * 
 * Device for accessing (in user-space) pages that have been granted by other
 * domains.
 *
 * Copyright (c) 2006-2007, D G Murray.
 * 
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 * 
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA  02111-1307  USA
 */

#include <asm/atomic.h>
#include <linux/module.h>
#include <linux/kernel.h>
#include <linux/init.h>
#include <linux/fs.h>
#include <linux/device.h>
#include <linux/mm.h>
#include <linux/mman.h>
#include <asm/uaccess.h>
#include <asm/io.h>
#include <xen/gnttab.h>
#include <asm/hypervisor.h>
#include <xen/balloon.h>
#include <xen/evtchn.h>
#include <xen/driver_util.h>

#include <linux/types.h>
#include <xen/public/gntdev.h>


#define DRIVER_AUTHOR "Derek G. Murray <Derek.Murray@cl.cam.ac.uk>"
#define DRIVER_DESC   "User-space granted page access driver"

MODULE_LICENSE("GPL");
MODULE_AUTHOR(DRIVER_AUTHOR);
MODULE_DESCRIPTION(DRIVER_DESC);

#define MAX_GRANTS 128

/* A slot can be in one of three states:
 *
 * 0. GNTDEV_SLOT_INVALID:
 *    This slot is not associated with a grant reference, and is therefore free
 *    to be overwritten by a new grant reference.
 *
 * 1. GNTDEV_SLOT_NOT_YET_MAPPED:
 *    This slot is associated with a grant reference (via the 
 *    IOCTL_GNTDEV_MAP_GRANT_REF ioctl), but it has not yet been mmap()-ed.
 *
 * 2. GNTDEV_SLOT_MAPPED:
 *    This slot is associated with a grant reference, and has been mmap()-ed.
 */
typedef enum gntdev_slot_state {
        GNTDEV_SLOT_INVALID = 0,
        GNTDEV_SLOT_NOT_YET_MAPPED,
        GNTDEV_SLOT_MAPPED
} gntdev_slot_state_t;

#define GNTDEV_INVALID_HANDLE    -1
#define GNTDEV_FREE_LIST_INVALID -1
/* Each opened instance of gntdev is associated with a list of grants,
 * represented by an array of elements of the following type,
 * gntdev_grant_info_t.
 */
typedef struct gntdev_grant_info {
        gntdev_slot_state_t state;
        union {
                uint32_t free_list_index;
                struct {
                        domid_t domid;
                        grant_ref_t ref;
                        grant_handle_t kernel_handle;
                        grant_handle_t user_handle;
                        uint64_t dev_bus_addr;
                } valid;
        } u;
} gntdev_grant_info_t;

/* Private data structure, which is stored in the file pointer for files
 * associated with this device.
 */
typedef struct gntdev_file_private_data {
  
        /* Array of grant information. */
        gntdev_grant_info_t grants[MAX_GRANTS];

        /* Read/write semaphore used to protect the grants array. */
        struct rw_semaphore grants_sem;

        /* An array of indices of free slots in the grants array.
         * N.B. An entry in this list may temporarily have the value
         * GNTDEV_FREE_LIST_INVALID if the corresponding slot has been removed
         * from the list by the contiguous allocator, but the list has not yet
         * been compressed. However, this is not visible across invocations of
         * the device.
         */
        int32_t free_list[MAX_GRANTS];
        
        /* The number of free slots in the grants array. */
        uint32_t free_list_size;

        /* Read/write semaphore used to protect the free list. */
        struct rw_semaphore free_list_sem;
        
        /* Index of the next slot after the most recent contiguous allocation, 
         * for use in a next-fit allocator.
         */
        uint32_t next_fit_index;

        /* Used to map grants into the kernel, before mapping them into user
         * space.
         */
        struct page **foreign_pages;

} gntdev_file_private_data_t;

/* Module lifecycle operations. */
static int __init gntdev_init(void);
static void __exit gntdev_exit(void);

module_init(gntdev_init);
module_exit(gntdev_exit);

/* File operations. */
static int gntdev_open(struct inode *inode, struct file *flip);
static int gntdev_release(struct inode *inode, struct file *flip);
static int gntdev_mmap(struct file *flip, struct vm_area_struct *vma);
static int gntdev_ioctl (struct inode *inode, struct file *flip,
                         unsigned int cmd, unsigned long arg);

static struct file_operations gntdev_fops = {
        .owner = THIS_MODULE,
        .open = gntdev_open,
        .release = gntdev_release,
        .mmap = gntdev_mmap,
        .ioctl = gntdev_ioctl
};

/* VM operations. */
static void gntdev_vma_close(struct vm_area_struct *vma);
static pte_t gntdev_clear_pte(struct vm_area_struct *vma, unsigned long addr,
                              pte_t *ptep, int is_fullmm);

static struct vm_operations_struct gntdev_vmops = {
        .close = gntdev_vma_close,
        .zap_pte = gntdev_clear_pte
};

/* Global variables. */

/* The driver major number, for use when unregistering the driver. */
static int gntdev_major;

#define GNTDEV_NAME "gntdev"

/* Memory mapping functions
 * ------------------------
 *
 * Every granted page is mapped into both kernel and user space, and the two
 * following functions return the respective virtual addresses of these pages.
 *
 * When shadow paging is disabled, the granted page is mapped directly into
 * user space; when it is enabled, it is mapped into the kernel and remapped
 * into user space using vm_insert_page() (see gntdev_mmap(), below).
 */

/* Returns the virtual address (in user space) of the @page_index'th page
 * in the given VM area.
 */
static inline unsigned long get_user_vaddr (struct vm_area_struct *vma,
                                            int page_index)
{
        return (unsigned long) vma->vm_start + (page_index << PAGE_SHIFT);
}

/* Returns the virtual address (in kernel space) of the @slot_index'th page
 * mapped by the gntdev instance that owns the given private data struct.
 */
static inline unsigned long get_kernel_vaddr (gntdev_file_private_data_t *priv,
                                              int slot_index)
{
        unsigned long pfn;
        void *kaddr;
        pfn = page_to_pfn(priv->foreign_pages[slot_index]);
        kaddr = pfn_to_kaddr(pfn);
        return (unsigned long) kaddr;
}

/* Helper functions. */

/* Adds information about a grant reference to the list of grants in the file's
 * private data structure. Returns non-zero on failure. On success, sets the
 * value of *offset to the offset that should be mmap()-ed in order to map the
 * grant reference.
 */
static int add_grant_reference(struct file *flip,
                               struct ioctl_gntdev_grant_ref *op,
                               uint64_t *offset)
{
        gntdev_file_private_data_t *private_data 
                = (gntdev_file_private_data_t *) flip->private_data;

        uint32_t slot_index;

        if (unlikely(private_data->free_list_size == 0)) {
                return -ENOMEM;
        }

        slot_index = private_data->free_list[--private_data->free_list_size];

        /* Copy the grant information into file's private data. */
        private_data->grants[slot_index].state = GNTDEV_SLOT_NOT_YET_MAPPED;
        private_data->grants[slot_index].u.valid.domid = op->domid;
        private_data->grants[slot_index].u.valid.ref = op->ref;

        /* The offset is calculated as the index of the chosen entry in the
         * file's private data's array of grant information. This is then
         * shifted to give an offset into the virtual "file address space".
         */
        *offset = slot_index << PAGE_SHIFT;

        return 0;
}

/* Adds the @count grant references to the contiguous range in the slot array
 * beginning at @first_slot. It is assumed that @first_slot was returned by a
 * previous invocation of find_contiguous_free_range(), during the same
 * invocation of the driver.
 */
static int add_grant_references(struct file *flip,
                                int count,
                                struct ioctl_gntdev_grant_ref *ops,
                                uint32_t first_slot)
{
        gntdev_file_private_data_t *private_data 
                = (gntdev_file_private_data_t *) flip->private_data;
        int i;
        
        for (i = 0; i < count; ++i) {

                /* First, mark the slot's entry in the free list as invalid. */
                int free_list_index = 
                        private_data->grants[first_slot+i].u.free_list_index;
                private_data->free_list[free_list_index] = 
                        GNTDEV_FREE_LIST_INVALID;

                /* Now, update the slot. */
                private_data->grants[first_slot+i].state = 
                        GNTDEV_SLOT_NOT_YET_MAPPED;
                private_data->grants[first_slot+i].u.valid.domid =
                        ops[i].domid;
                private_data->grants[first_slot+i].u.valid.ref = ops[i].ref;
        }

        return 0;       
}

/* Scans through the free list for @flip, removing entries that are marked as
 * GNTDEV_SLOT_INVALID. This will reduce the recorded size of the free list to
 * the number of valid entries.
 */
static void compress_free_list(struct file *flip) 
{
        gntdev_file_private_data_t *private_data 
                = (gntdev_file_private_data_t *) flip->private_data;
        int i, j = 0, old_size;
        
        old_size = private_data->free_list_size;
        for (i = 0; i < old_size; ++i) {
                if (private_data->free_list[i] != GNTDEV_FREE_LIST_INVALID) {
                        private_data->free_list[j] = 
                                private_data->free_list[i];
                        ++j;
                } else {
                        --private_data->free_list_size;
                }
        }
}

/* Searches the grant array in the private data of @flip for a range of
 * @num_slots contiguous slots in the GNTDEV_SLOT_INVALID state.
 *
 * Returns the index of the first slot if a range is found, otherwise -ENOMEM.
 */
static int find_contiguous_free_range(struct file *flip,
                                      uint32_t num_slots) 
{
        gntdev_file_private_data_t *private_data 
                = (gntdev_file_private_data_t *) flip->private_data;
        
        int i;
        int start_index = private_data->next_fit_index;
        int range_start = 0, range_length;

        if (private_data->free_list_size < num_slots) {
                return -ENOMEM;
        }

        /* First search from the start_index to the end of the array. */
        range_length = 0;
        for (i = start_index; i < MAX_GRANTS; ++i) {
                if (private_data->grants[i].state == GNTDEV_SLOT_INVALID) {
                        if (range_length == 0) {
                                range_start = i;
                        }
                        ++range_length;
                        if (range_length == num_slots) {
                                return range_start;
                        }
                }
        }
        
        /* Now search from the start of the array to the start_index. */
        range_length = 0;
        for (i = 0; i < start_index; ++i) {
                if (private_data->grants[i].state == GNTDEV_SLOT_INVALID) {
                        if (range_length == 0) {
                                range_start = i;
                        }
                        ++range_length;
                        if (range_length == num_slots) {
                                return range_start;
                        }
                }
        }
        
        return -ENOMEM;
}

/* Interface functions. */

/* Initialises the driver. Called when the module is loaded. */
static int __init gntdev_init(void)
{
        struct class *class;
        struct class_device *device;

        if (!is_running_on_xen()) {
                printk(KERN_ERR "You must be running Xen to use gntdev\n");
                return -ENODEV;
        }

        gntdev_major = register_chrdev(0, GNTDEV_NAME, &gntdev_fops);
        if (gntdev_major < 0)
        {
                printk(KERN_ERR "Could not register gntdev device\n");
                return -ENOMEM;
        }

        /* Note that if the sysfs code fails, we will still initialise the
         * device, and output the major number so that the device can be
         * created manually using mknod.
         */
        if ((class = get_xen_class()) == NULL) {
                printk(KERN_ERR "Error setting up xen_class\n");
                printk(KERN_ERR "gntdev created with major number = %d\n", 
                       gntdev_major);
                return 0;
        }

        device = class_device_create(class, NULL, MKDEV(gntdev_major, 0),
                                     NULL, GNTDEV_NAME);
        if (IS_ERR(device)) {
                printk(KERN_ERR "Error creating gntdev device in xen_class\n");
                printk(KERN_ERR "gntdev created with major number = %d\n",
                       gntdev_major);
                return 0;
        }

        return 0;
}

/* Cleans up and unregisters the driver. Called when the driver is unloaded.
 */
static void __exit gntdev_exit(void)
{
        struct class *class;
        if ((class = get_xen_class()) != NULL)
                class_device_destroy(class, MKDEV(gntdev_major, 0));
        unregister_chrdev(gntdev_major, GNTDEV_NAME);
}

/* Called when the device is opened. */
static int gntdev_open(struct inode *inode, struct file *flip)
{
        gntdev_file_private_data_t *private_data;
        int i;

        try_module_get(THIS_MODULE);

        /* Allocate space for the per-instance private data. */
        private_data = kmalloc(sizeof(*private_data), GFP_KERNEL);
        if (!private_data)
                goto nomem_out;

        /* Allocate space for the kernel-mapping of granted pages. */
        private_data->foreign_pages = 
                alloc_empty_pages_and_pagevec(MAX_GRANTS);
        if (!private_data->foreign_pages)
                goto nomem_out2;

        /* Initialise the free-list, which contains all slots at first.
         */
        for (i = 0; i < MAX_GRANTS; ++i) {
                private_data->free_list[MAX_GRANTS - i - 1] = i;
                private_data->grants[i].state = GNTDEV_SLOT_INVALID;
                private_data->grants[i].u.free_list_index = MAX_GRANTS - i - 1;
        }
        private_data->free_list_size = MAX_GRANTS;
        private_data->next_fit_index = 0;

        init_rwsem(&private_data->grants_sem);
        init_rwsem(&private_data->free_list_sem);

        flip->private_data = private_data;

        return 0;

nomem_out2:
        kfree(private_data);
nomem_out:
        return -ENOMEM;
}

/* Called when the device is closed.
 */
static int gntdev_release(struct inode *inode, struct file *flip)
{
        if (flip->private_data) {
                gntdev_file_private_data_t *private_data = 
                        (gntdev_file_private_data_t *) flip->private_data;
                if (private_data->foreign_pages) {
                        free_empty_pages_and_pagevec
                                (private_data->foreign_pages, MAX_GRANTS);
                }
                kfree(private_data);
        }
        module_put(THIS_MODULE);
        return 0;
}

/* Called when an attempt is made to mmap() the device. The private data from
 * @flip contains the list of grant references that can be mapped. The vm_pgoff
 * field of @vma contains the index into that list that refers to the grant
 * reference that will be mapped. Only mappings that are a multiple of
 * PAGE_SIZE are handled.
 */
static int gntdev_mmap (struct file *flip, struct vm_area_struct *vma) 
{
        struct gnttab_map_grant_ref op;
        unsigned long slot_index = vma->vm_pgoff;
        unsigned long kernel_vaddr, user_vaddr;
        uint32_t size = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
        uint64_t ptep;
        int ret;
        int flags;
        int i;
        struct page *page;
        gntdev_file_private_data_t *private_data = flip->private_data;

        if (unlikely(!private_data)) {
                printk(KERN_ERR "File's private data is NULL.\n");
                return -EINVAL;
        }

        if (unlikely((size <= 0) || (size + slot_index) > MAX_GRANTS)) {
                printk(KERN_ERR "Invalid number of pages or offset"
                       "(num_pages = %d, first_slot = %ld).\n",
                       size, slot_index);
                return -ENXIO;
        }

        if ((vma->vm_flags & VM_WRITE) && !(vma->vm_flags & VM_SHARED)) {
                printk(KERN_ERR "Writable mappings must be shared.\n");
                return -EINVAL;
        }

        /* Slots must be in the NOT_YET_MAPPED state. */
        down_write(&private_data->grants_sem);
        for (i = 0; i < size; ++i) {
                if (private_data->grants[slot_index + i].state != 
                    GNTDEV_SLOT_NOT_YET_MAPPED) {
                        printk(KERN_ERR "Slot (index = %ld) is in the wrong "
                               "state (%d).\n", slot_index + i, 
                               private_data->grants[slot_index + i].state);
                        up_write(&private_data->grants_sem);
                        return -EINVAL;
                }
        }

        /* Install the hook for unmapping. */
        vma->vm_ops = &gntdev_vmops;
    
        /* The VM area contains pages from another VM. */
        vma->vm_flags |= VM_FOREIGN;
        vma->vm_private_data = kzalloc(size * sizeof(struct page_struct *), 
                                       GFP_KERNEL);
        if (vma->vm_private_data == NULL) {
                printk(KERN_ERR "Couldn't allocate mapping structure for VM "
                       "area.\n");
                return -ENOMEM;
        }

        /* This flag prevents Bad PTE errors when the memory is unmapped. */
        vma->vm_flags |= VM_RESERVED;

        /* This flag prevents this VM area being copied on a fork(). A better
         * behaviour might be to explicitly carry out the appropriate mappings
         * on fork(), but I don't know if there's a hook for this.
         */
        vma->vm_flags |= VM_DONTCOPY;

#ifdef CONFIG_X86
        /* This flag ensures that the page tables are not unpinned before the
         * VM area is unmapped. Therefore Xen still recognises the PTE as
         * belonging to an L1 pagetable, and the grant unmap operation will
         * succeed, even if the process does not exit cleanly.
         */
        vma->vm_mm->context.has_foreign_mappings = 1;
#endif

        for (i = 0; i < size; ++i) {

                flags = GNTMAP_host_map;
                if (!(vma->vm_flags & VM_WRITE))
                        flags |= GNTMAP_readonly;

                kernel_vaddr = get_kernel_vaddr(private_data, slot_index + i);
                user_vaddr = get_user_vaddr(vma, i);
                page = pfn_to_page(__pa(kernel_vaddr) >> PAGE_SHIFT);

                gnttab_set_map_op(&op, kernel_vaddr, flags,   
                                  private_data->grants[slot_index+i]
                                  .u.valid.ref, 
                                  private_data->grants[slot_index+i]
                                  .u.valid.domid);

                /* Carry out the mapping of the grant reference. */
                ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref, 
                                                &op, 1);
                BUG_ON(ret);
                if (op.status) {
                        printk(KERN_ERR "Error mapping the grant reference "
                               "into the kernel (%d). domid = %d; ref = %d\n",
                               op.status,
                               private_data->grants[slot_index+i]
                               .u.valid.domid,
                               private_data->grants[slot_index+i]
                               .u.valid.ref);
                        goto undo_map_out;
                }

                /* Store a reference to the page that will be mapped into user
                 * space.
                 */
                ((struct page **) vma->vm_private_data)[i] = page;

                /* Mark mapped page as reserved. */
                SetPageReserved(page);

                /* Record the grant handle, for use in the unmap operation. */
                private_data->grants[slot_index+i].u.valid.kernel_handle = 
                        op.handle;
                private_data->grants[slot_index+i].u.valid.dev_bus_addr = 
                        op.dev_bus_addr;
                
                private_data->grants[slot_index+i].state = GNTDEV_SLOT_MAPPED;
                private_data->grants[slot_index+i].u.valid.user_handle =
                        GNTDEV_INVALID_HANDLE;

                /* Now perform the mapping to user space. */
                if (!xen_feature(XENFEAT_auto_translated_physmap)) {

                        /* NOT USING SHADOW PAGE TABLES. */
                        /* In this case, we map the grant(s) straight into user
                         * space.
                         */

                        /* Get the machine address of the PTE for the user 
                         *  page.
                         */
                        if ((ret = create_lookup_pte_addr(vma->vm_mm, 
                                                          vma->vm_start 
                                                          + (i << PAGE_SHIFT), 
                                                          &ptep)))
                        {
                                printk(KERN_ERR "Error obtaining PTE pointer "
                                       "(%d).\n", ret);
                                goto undo_map_out;
                        }
                        
                        /* Configure the map operation. */
                
                        /* The reference is to be used by host CPUs. */
                        flags = GNTMAP_host_map;
                        
                        /* Specifies a user space mapping. */
                        flags |= GNTMAP_application_map;
                        
                        /* The map request contains the machine address of the
                         * PTE to update.
                         */
                        flags |= GNTMAP_contains_pte;
                        
                        if (!(vma->vm_flags & VM_WRITE))
                                flags |= GNTMAP_readonly;

                        gnttab_set_map_op(&op, ptep, flags, 
                                          private_data->grants[slot_index+i]
                                          .u.valid.ref, 
                                          private_data->grants[slot_index+i]
                                          .u.valid.domid);

                        /* Carry out the mapping of the grant reference. */
                        ret = HYPERVISOR_grant_table_op(GNTTABOP_map_grant_ref,
                                                        &op, 1);
                        BUG_ON(ret);
                        if (op.status) {
                                printk(KERN_ERR "Error mapping the grant "
                                       "reference into user space (%d). domid "
                                       "= %d; ref = %d\n", op.status,
                                       private_data->grants[slot_index+i].u
                                       .valid.domid,
                                       private_data->grants[slot_index+i].u
                                       .valid.ref);
                                goto undo_map_out;
                        }
                        
                        /* Record the grant handle, for use in the unmap 
                         * operation. 
                         */
                        private_data->grants[slot_index+i].u.
                                valid.user_handle = op.handle;

                        /* Update p2m structure with the new mapping. */
                        set_phys_to_machine(__pa(kernel_vaddr) >> PAGE_SHIFT,
                                            FOREIGN_FRAME(private_data->
                                                          grants[slot_index+i]
                                                          .u.valid.dev_bus_addr
                                                          >> PAGE_SHIFT));
                } else {
                        /* USING SHADOW PAGE TABLES. */
                        /* In this case, we simply insert the page into the VM
                         * area. */
                        ret = vm_insert_page(vma, user_vaddr, page);
                }

        }

        up_write(&private_data->grants_sem);
        return 0;

undo_map_out:
        /* If we have a mapping failure, the unmapping will be taken care of
         * by do_mmap_pgoff(), which will eventually call gntdev_clear_pte().
         * All we need to do here is free the vma_private_data.
         */
        kfree(vma->vm_private_data);

        /* THIS IS VERY UNPLEASANT: do_mmap_pgoff() will set the vma->vm_file
         * to NULL on failure. However, we need this in gntdev_clear_pte() to
         * unmap the grants. Therefore, we smuggle a reference to the file's
         * private data in the VM area's private data pointer.
         */
        vma->vm_private_data = private_data;
        
        up_write(&private_data->grants_sem);

        return -ENOMEM;
}

static pte_t gntdev_clear_pte(struct vm_area_struct *vma, unsigned long addr,
                              pte_t *ptep, int is_fullmm)
{
        int slot_index, ret;
        pte_t copy;
        struct gnttab_unmap_grant_ref op;
        gntdev_file_private_data_t *private_data;

        /* THIS IS VERY UNPLEASANT: do_mmap_pgoff() will set the vma->vm_file
         * to NULL on failure. However, we need this in gntdev_clear_pte() to
         * unmap the grants. Therefore, we smuggle a reference to the file's
         * private data in the VM area's private data pointer.
         */
        if (vma->vm_file) {
                private_data = (gntdev_file_private_data_t *)
                        vma->vm_file->private_data;
        } else if (vma->vm_private_data) {
                private_data = (gntdev_file_private_data_t *)
                        vma->vm_private_data;
        } else {
                private_data = NULL; /* gcc warning */
                BUG();
        }

        /* Copy the existing value of the PTE for returning. */
        copy = *ptep;

        /* Calculate the grant relating to this PTE. */
        slot_index = vma->vm_pgoff + ((addr - vma->vm_start) >> PAGE_SHIFT);

        /* Only unmap grants if the slot has been mapped. This could be being
         * called from a failing mmap().
         */
        if (private_data->grants[slot_index].state == GNTDEV_SLOT_MAPPED) {

                /* First, we clear the user space mapping, if it has been made.
                 */
                if (private_data->grants[slot_index].u.valid.user_handle !=
                    GNTDEV_INVALID_HANDLE && 
                    !xen_feature(XENFEAT_auto_translated_physmap)) {
                        /* NOT USING SHADOW PAGE TABLES. */
                        gnttab_set_unmap_op(&op, virt_to_machine(ptep), 
                                            GNTMAP_contains_pte,
                                            private_data->grants[slot_index]
                                            .u.valid.user_handle);
                        ret = HYPERVISOR_grant_table_op(
                                GNTTABOP_unmap_grant_ref, &op, 1);
                        BUG_ON(ret);
                        if (op.status)
                                printk("User unmap grant status = %d\n", 
                                       op.status);
                } else {
                        /* USING SHADOW PAGE TABLES. */
                        pte_clear_full(vma->vm_mm, addr, ptep, is_fullmm);
                }

                /* Finally, we unmap the grant from kernel space. */
                gnttab_set_unmap_op(&op, 
                                    get_kernel_vaddr(private_data, slot_index),
                                    GNTMAP_host_map, 
                                    private_data->grants[slot_index].u.valid
                                    .kernel_handle);
                ret = HYPERVISOR_grant_table_op(GNTTABOP_unmap_grant_ref, 
                                                &op, 1);
                BUG_ON(ret);
                if (op.status)
                        printk("Kernel unmap grant status = %d\n", op.status);


                /* Return slot to the not-yet-mapped state, so that it may be
                 * mapped again, or removed by a subsequent ioctl.
                 */
                private_data->grants[slot_index].state = 
                        GNTDEV_SLOT_NOT_YET_MAPPED;

                /* Invalidate the physical to machine mapping for this page. */
                set_phys_to_machine(__pa(get_kernel_vaddr(private_data, 
                                                          slot_index)) 
                                    >> PAGE_SHIFT, INVALID_P2M_ENTRY);

        } else {
                pte_clear_full(vma->vm_mm, addr, ptep, is_fullmm);
        }

        return copy;
}

/* "Destructor" for a VM area.
 */
static void gntdev_vma_close(struct vm_area_struct *vma) {
        if (vma->vm_private_data) {
                kfree(vma->vm_private_data);
        }
}

/* Called when an ioctl is made on the device.
 */
static int gntdev_ioctl(struct inode *inode, struct file *flip,
                        unsigned int cmd, unsigned long arg)
{
        int rc = 0;
        gntdev_file_private_data_t *private_data = 
                (gntdev_file_private_data_t *) flip->private_data;

        switch (cmd) {
        case IOCTL_GNTDEV_MAP_GRANT_REF:
        {
                struct ioctl_gntdev_map_grant_ref op;
                down_write(&private_data->grants_sem);
                down_write(&private_data->free_list_sem);

                if ((rc = copy_from_user(&op, (void __user *) arg, 
                                         sizeof(op)))) {
                        rc = -EFAULT;
                        goto map_out;
                }
                if (unlikely(op.count <= 0)) {
                        rc = -EINVAL;
                        goto map_out;
                }

                if (op.count == 1) {
                        if ((rc = add_grant_reference(flip, &op.refs[0],
                                                      &op.index)) < 0) {
                                printk(KERN_ERR "Adding grant reference "
                                       "failed (%d).\n", rc);
                                goto map_out;
                        }
                } else {
                        struct ioctl_gntdev_grant_ref *refs, *u;
                        refs = kmalloc(op.count * sizeof(*refs), GFP_KERNEL);
                        if (!refs) {
                                rc = -ENOMEM;
                                goto map_out;
                        }
                        u = ((struct ioctl_gntdev_map_grant_ref *)arg)->refs;
                        if ((rc = copy_from_user(refs,
                                                 (void __user *)u,
                                                 sizeof(*refs) * op.count))) {
                                printk(KERN_ERR "Copying refs from user failed"
                                       " (%d).\n", rc);
                                rc = -EINVAL;
                                goto map_out;
                        }
                        if ((rc = find_contiguous_free_range(flip, op.count))
                            < 0) {
                                printk(KERN_ERR "Finding contiguous range "
                                       "failed (%d).\n", rc);
                                kfree(refs);
                                goto map_out;
                        }
                        op.index = rc << PAGE_SHIFT;
                        if ((rc = add_grant_references(flip, op.count,
                                                       refs, rc))) {
                                printk(KERN_ERR "Adding grant references "
                                       "failed (%d).\n", rc);
                                kfree(refs);
                                goto map_out;
                        }
                        compress_free_list(flip);
                        kfree(refs);
                }
                if ((rc = copy_to_user((void __user *) arg, 
                                       &op, 
                                       sizeof(op)))) {
                        printk(KERN_ERR "Copying result back to user failed "
                               "(%d)\n", rc);
                        rc = -EFAULT;
                        goto map_out;
                }
        map_out:
                up_write(&private_data->grants_sem);
                up_write(&private_data->free_list_sem);
                return rc;
        }
        case IOCTL_GNTDEV_UNMAP_GRANT_REF:
        {
                struct ioctl_gntdev_unmap_grant_ref op;
                int i, start_index;

                down_write(&private_data->grants_sem);
                down_write(&private_data->free_list_sem);

                if ((rc = copy_from_user(&op, 
                                         (void __user *) arg, 
                                         sizeof(op)))) {
                        rc = -EFAULT;
                        goto unmap_out;
                }

                start_index = op.index >> PAGE_SHIFT;

                /* First, check that all pages are in the NOT_YET_MAPPED
                 * state.
                 */
                for (i = 0; i < op.count; ++i) {
                        if (unlikely
                            (private_data->grants[start_index + i].state
                             != GNTDEV_SLOT_NOT_YET_MAPPED)) {
                                if (private_data->grants[start_index + i].state
                                    == GNTDEV_SLOT_INVALID) {
                                        printk(KERN_ERR
                                               "Tried to remove an invalid "
                                               "grant at offset 0x%x.",
                                               (start_index + i) 
                                               << PAGE_SHIFT);
                                        rc = -EINVAL;
                                } else {
                                        printk(KERN_ERR
                                               "Tried to remove a grant which "
                                               "is currently mmap()-ed at "
                                               "offset 0x%x.",
                                               (start_index + i) 
                                               << PAGE_SHIFT);
                                        rc = -EBUSY;
                                }
                                goto unmap_out;
                        }
                }

                /* Unmap pages and add them to the free list.
                 */
                for (i = 0; i < op.count; ++i) {
                        private_data->grants[start_index+i].state = 
                                GNTDEV_SLOT_INVALID;
                        private_data->grants[start_index+i].u.free_list_index =
                                private_data->free_list_size;
                        private_data->free_list[private_data->free_list_size] =
                                start_index + i;
                        ++private_data->free_list_size;
                }
                compress_free_list(flip);

        unmap_out:
                up_write(&private_data->grants_sem);
                up_write(&private_data->free_list_sem);
                return rc;
        }
        case IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR:
        {
                struct ioctl_gntdev_get_offset_for_vaddr op;
                struct vm_area_struct *vma;
                unsigned long vaddr;

                if ((rc = copy_from_user(&op, 
                                         (void __user *) arg, 
                                         sizeof(op)))) {
                        rc = -EFAULT;
                        goto get_offset_out;
                }
                vaddr = (unsigned long)op.vaddr;

                down_read(&current->mm->mmap_sem);              
                vma = find_vma(current->mm, vaddr);
                if (vma == NULL) {
                        rc = -EFAULT;
                        goto get_offset_unlock_out;
                }
                if ((!vma->vm_ops) || (vma->vm_ops != &gntdev_vmops)) {
                        printk(KERN_ERR "The vaddr specified does not belong "
                               "to a gntdev instance: %#lx\n", vaddr);
                        rc = -EFAULT;
                        goto get_offset_unlock_out;
                }
                if (vma->vm_start != vaddr) {
                        printk(KERN_ERR "The vaddr specified in an "
                               "IOCTL_GNTDEV_GET_OFFSET_FOR_VADDR must be at "
                               "the start of the VM area. vma->vm_start = "
                               "%#lx; vaddr = %#lx\n",
                               vma->vm_start, vaddr);
                        rc = -EFAULT;
                        goto get_offset_unlock_out;
                }
                op.offset = vma->vm_pgoff << PAGE_SHIFT;
                op.count = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT;
                up_read(&current->mm->mmap_sem);
                if ((rc = copy_to_user((void __user *) arg, 
                                       &op, 
                                       sizeof(op)))) {
                        rc = -EFAULT;
                        goto get_offset_out;
                }
                goto get_offset_out;
        get_offset_unlock_out:
                up_read(&current->mm->mmap_sem);
        get_offset_out:
                return rc;
        }
        default:
                return -ENOIOCTLCMD;
        }

        return 0;
}