From: kettenis Date: Sun, 4 Jun 2017 14:02:24 +0000 (+0000) Subject: Switch the TTM code over to the generic DRM VMA manager. X-Git-Url: http://artulab.com/gitweb/?a=commitdiff_plain;h=beea9bb1be8262c40e0433fbe4451d2581e1bab3;p=openbsd Switch the TTM code over to the generic DRM VMA manager. ok jsg@ --- diff --git a/sys/dev/pci/drm/ttm/ttm_bo.c b/sys/dev/pci/drm/ttm/ttm_bo.c index 523c25a606c..0f34925f6dd 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo.c +++ b/sys/dev/pci/drm/ttm/ttm_bo.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo.c,v 1.19 2015/10/23 08:21:58 kettenis Exp $ */ +/* $OpenBSD: ttm_bo.c,v 1.20 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -728,14 +728,7 @@ static void ttm_bo_release(struct kref *kref) struct ttm_bo_device *bdev = bo->bdev; struct ttm_mem_type_manager *man = &bdev->man[bo->mem.mem_type]; - write_lock(&bdev->vm_lock); - if (likely(bo->vm_node != NULL)) { - RB_REMOVE(ttm_bo_device_buffer_objects, - &bdev->addr_space_rb, bo); - drm_mm_put_block(bo->vm_node); - bo->vm_node = NULL; - } - write_unlock(&bdev->vm_lock); + drm_vma_offset_remove(&bdev->vma_manager, &bo->vma_node); ttm_mem_io_lock(man, false); ttm_mem_io_free_vm(bo); ttm_mem_io_unlock(man); @@ -1252,6 +1245,7 @@ int ttm_bo_init(struct ttm_bo_device *bdev, bo->acc_size = acc_size; bo->sg = sg; atomic_inc(&bo->glob->bo_count); + drm_vma_node_reset(&bo->vma_node); ret = ttm_bo_check_placement(bo, placement); if (unlikely(ret != 0)) @@ -1552,10 +1546,7 @@ int ttm_bo_device_release(struct ttm_bo_device *bdev) TTM_DEBUG("Swap list was clean\n"); spin_unlock(&glob->lru_lock); - BUG_ON(!drm_mm_clean(&bdev->addr_space_mm)); - write_lock(&bdev->vm_lock); - drm_mm_takedown(&bdev->addr_space_mm); - write_unlock(&bdev->vm_lock); + drm_vma_offset_manager_destroy(&bdev->vma_manager); return ret; } @@ -1569,7 +1560,6 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, { int ret = -EINVAL; - rw_init(&bdev->vm_lock, "ttmvm"); bdev->driver = driver; memset(bdev->man, 0, sizeof(bdev->man)); @@ -1582,9 +1572,8 @@ int ttm_bo_device_init(struct ttm_bo_device *bdev, if (unlikely(ret != 0)) goto out_no_sys; - RB_INIT(&bdev->addr_space_rb); - drm_mm_init(&bdev->addr_space_mm, file_page_offset, 0x10000000); - + drm_vma_offset_manager_init(&bdev->vma_manager, file_page_offset, + 0x10000000); INIT_DELAYED_WORK(&bdev->wq, ttm_bo_delayed_workqueue); INIT_LIST_HEAD(&bdev->ddestroy); bdev->dev_mapping = NULL; @@ -1665,14 +1654,6 @@ void ttm_bo_unmap_virtual(struct ttm_buffer_object *bo) EXPORT_SYMBOL(ttm_bo_unmap_virtual); -static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) -{ - struct ttm_bo_device *bdev = bo->bdev; - - /* The caller acquired bdev->vm_lock. */ - RB_INSERT(ttm_bo_device_buffer_objects, &bdev->addr_space_rb, bo); -} - /** * ttm_bo_setup_vm: * @@ -1687,38 +1668,9 @@ static void ttm_bo_vm_insert_rb(struct ttm_buffer_object *bo) static int ttm_bo_setup_vm(struct ttm_buffer_object *bo) { struct ttm_bo_device *bdev = bo->bdev; - int ret; - -retry_pre_get: - ret = drm_mm_pre_get(&bdev->addr_space_mm); - if (unlikely(ret != 0)) - return ret; - - write_lock(&bdev->vm_lock); - bo->vm_node = drm_mm_search_free(&bdev->addr_space_mm, - bo->mem.num_pages, 0, 0); - if (unlikely(bo->vm_node == NULL)) { - ret = -ENOMEM; - goto out_unlock; - } - - bo->vm_node = drm_mm_get_block_atomic(bo->vm_node, - bo->mem.num_pages, 0); - - if (unlikely(bo->vm_node == NULL)) { - write_unlock(&bdev->vm_lock); - goto retry_pre_get; - } - - ttm_bo_vm_insert_rb(bo); - write_unlock(&bdev->vm_lock); - bo->addr_space_offset = ((uint64_t) bo->vm_node->start) << PAGE_SHIFT; - - return 0; -out_unlock: - write_unlock(&bdev->vm_lock); - return ret; + return drm_vma_offset_add(&bdev->vma_manager, &bo->vma_node, + bo->mem.num_pages); } int ttm_bo_wait(struct ttm_buffer_object *bo, diff --git a/sys/dev/pci/drm/ttm/ttm_bo_api.h b/sys/dev/pci/drm/ttm/ttm_bo_api.h index 7fe2e5a15ce..feaa8ed7162 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_api.h +++ b/sys/dev/pci/drm/ttm/ttm_bo_api.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo_api.h,v 1.5 2015/09/27 11:09:26 jsg Exp $ */ +/* $OpenBSD: ttm_bo_api.h,v 1.6 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -33,6 +33,7 @@ #define _TTM_BO_API_H_ #include +#include struct ttm_bo_device; @@ -139,7 +140,6 @@ struct ttm_tt; * @type: The bo type. * @destroy: Destruction function. If NULL, kfree is used. * @num_pages: Actual number of pages. - * @addr_space_offset: Address space offset. * @acc_size: Accounted size for this object. * @kref: Reference count of this buffer object. When this refcount reaches * zero, the object is put on the delayed delete list. @@ -167,8 +167,7 @@ struct ttm_tt; * @reserved: Deadlock-free lock used for synchronization state transitions. * @sync_obj: Pointer to a synchronization object. * @priv_flags: Flags describing buffer object internal state. - * @vm_rb: Rb node for the vm rb tree. - * @vm_node: Address space manager node. + * @vma_node: Address space manager node. * @offset: The current GPU offset, which can have different meanings * depending on the memory type. For SYSTEM type memory, it should be 0. * @cur_placement: Hint of current placement. @@ -197,7 +196,6 @@ struct ttm_buffer_object { enum ttm_bo_type type; void (*destroy) (struct ttm_buffer_object *); unsigned long num_pages; - uint64_t addr_space_offset; size_t acc_size; /** @@ -251,13 +249,7 @@ struct ttm_buffer_object { void *sync_obj; unsigned long priv_flags; - /** - * Members protected by the bdev::vm_lock - */ - - RB_ENTRY(ttm_buffer_object) vm_rb; - struct drm_mm_node *vm_node; - + struct drm_vma_offset_node vma_node; /** * Special members that are protected by the reserve lock diff --git a/sys/dev/pci/drm/ttm/ttm_bo_driver.h b/sys/dev/pci/drm/ttm/ttm_bo_driver.h index 29af2244422..3032f944162 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_driver.h +++ b/sys/dev/pci/drm/ttm/ttm_bo_driver.h @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo_driver.h,v 1.7 2015/09/27 11:09:26 jsg Exp $ */ +/* $OpenBSD: ttm_bo_driver.h,v 1.8 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 Vmware, Inc., Palo Alto, CA., USA @@ -516,7 +516,7 @@ struct ttm_bo_global { * @man: An array of mem_type_managers. * @fence_lock: Protects the synchronizing members on *all* bos belonging * to this device. - * @addr_space_mm: Range manager for the device address space. + * @vma_manager: Address space manager * lru_lock: Spinlock that protects the buffer+device lru lists and * ddestroy lists. * @val_seq: Current validation sequence. @@ -534,7 +534,6 @@ struct ttm_bo_device { struct list_head device_list; struct ttm_bo_global *glob; struct ttm_bo_driver *driver; - rwlock_t vm_lock; struct ttm_mem_type_manager man[TTM_NUM_MEM_TYPES]; spinlock_t fence_lock; @@ -543,10 +542,9 @@ struct ttm_bo_device { bus_dma_tag_t dmat; /* - * Protected by the vm lock. + * Protected by internal locks. */ - RB_HEAD(ttm_bo_device_buffer_objects, ttm_buffer_object) addr_space_rb; - struct drm_mm addr_space_mm; + struct drm_vma_offset_manager vma_manager; /* * Protected by the global:lru lock. @@ -1011,10 +1009,4 @@ int ttm_agp_tt_populate(struct ttm_tt *ttm); void ttm_agp_tt_unpopulate(struct ttm_tt *ttm); #endif -int ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, - struct ttm_buffer_object *b); - -RB_PROTOTYPE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, - ttm_bo_cmp_rb_tree_items); - #endif diff --git a/sys/dev/pci/drm/ttm/ttm_bo_manager.c b/sys/dev/pci/drm/ttm/ttm_bo_manager.c index 92c38ad3c5b..ad948b5117a 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_manager.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_manager.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo_manager.c,v 1.5 2015/09/23 23:12:12 kettenis Exp $ */ +/* $OpenBSD: ttm_bo_manager.c,v 1.6 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2007-2010 VMware, Inc., Palo Alto, CA., USA @@ -53,34 +53,32 @@ static int ttm_bo_man_get_node(struct ttm_mem_type_manager *man, struct ttm_range_manager *rman = (struct ttm_range_manager *) man->priv; struct drm_mm *mm = &rman->mm; struct drm_mm_node *node = NULL; + enum drm_mm_search_flags sflags = DRM_MM_SEARCH_BEST; unsigned long lpfn; int ret; lpfn = placement->lpfn; if (!lpfn) lpfn = man->size; - do { - ret = drm_mm_pre_get(mm); - if (unlikely(ret)) - return ret; - spin_lock(&rman->lock); - node = drm_mm_search_free_in_range(mm, - mem->num_pages, mem->page_alignment, - placement->fpfn, lpfn, 1); - if (unlikely(node == NULL)) { - spin_unlock(&rman->lock); - return 0; - } - node = drm_mm_get_block_atomic_range(node, mem->num_pages, - mem->page_alignment, - placement->fpfn, - lpfn); - spin_unlock(&rman->lock); - } while (node == NULL); + node = kzalloc(sizeof(*node), GFP_KERNEL); + if (!node) + return -ENOMEM; + + spin_lock(&rman->lock); + ret = drm_mm_insert_node_in_range_generic(mm, node, mem->num_pages, + mem->page_alignment, 0, + placement->fpfn, lpfn, + sflags); + spin_unlock(&rman->lock); + + if (unlikely(ret)) { + kfree(node); + } else { + mem->mm_node = node; + mem->start = node->start; + } - mem->mm_node = node; - mem->start = node->start; return 0; } @@ -91,8 +89,10 @@ static void ttm_bo_man_put_node(struct ttm_mem_type_manager *man, if (mem->mm_node) { spin_lock(&rman->lock); - drm_mm_put_block(mem->mm_node); + drm_mm_remove_node(mem->mm_node); spin_unlock(&rman->lock); + + kfree(mem->mm_node); mem->mm_node = NULL; } } diff --git a/sys/dev/pci/drm/ttm/ttm_bo_util.c b/sys/dev/pci/drm/ttm/ttm_bo_util.c index e7734e5a799..136814e90a6 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_util.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_util.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo_util.c,v 1.15 2015/09/27 11:09:26 jsg Exp $ */ +/* $OpenBSD: ttm_bo_util.c,v 1.16 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2007-2009 VMware, Inc., Palo Alto, CA., USA @@ -31,6 +31,7 @@ #include #include +#include int ttm_mem_reg_ioremap(struct ttm_bo_device *, struct ttm_mem_reg *, void **); @@ -447,7 +448,7 @@ static int ttm_buffer_object_transfer(struct ttm_buffer_object *bo, INIT_LIST_HEAD(&fbo->lru); INIT_LIST_HEAD(&fbo->swap); INIT_LIST_HEAD(&fbo->io_reserve_lru); - fbo->vm_node = NULL; + drm_vma_node_reset(&fbo->vma_node); atomic_set(&fbo->cpu_writers, 0); spin_lock(&bdev->fence_lock); diff --git a/sys/dev/pci/drm/ttm/ttm_bo_vm.c b/sys/dev/pci/drm/ttm/ttm_bo_vm.c index 905ca27feae..c7fd2c13731 100644 --- a/sys/dev/pci/drm/ttm/ttm_bo_vm.c +++ b/sys/dev/pci/drm/ttm/ttm_bo_vm.c @@ -1,4 +1,4 @@ -/* $OpenBSD: ttm_bo_vm.c,v 1.10 2016/04/05 08:22:50 kettenis Exp $ */ +/* $OpenBSD: ttm_bo_vm.c,v 1.11 2017/06/04 14:02:24 kettenis Exp $ */ /************************************************************************** * * Copyright (c) 2006-2009 VMware, Inc., Palo Alto, CA., USA @@ -34,65 +34,12 @@ #include #include #include +#include #define TTM_BO_VM_NUM_PREFAULT 16 ssize_t ttm_bo_fbdev_io(struct ttm_buffer_object *, const char __user *, char __user *, size_t, off_t *, bool); -struct ttm_buffer_object * - ttm_bo_vm_lookup_rb(struct ttm_bo_device *, unsigned long, - unsigned long); - -#undef RB_ROOT -#define RB_ROOT(head) (head)->rbh_root - -RB_GENERATE(ttm_bo_device_buffer_objects, ttm_buffer_object, vm_rb, - ttm_bo_cmp_rb_tree_items); - -int -ttm_bo_cmp_rb_tree_items(struct ttm_buffer_object *a, - struct ttm_buffer_object *b) -{ - - if (a->vm_node->start < b->vm_node->start) { - return (-1); - } else if (a->vm_node->start > b->vm_node->start) { - return (1); - } else { - return (0); - } -} - -struct ttm_buffer_object * -ttm_bo_vm_lookup_rb(struct ttm_bo_device *bdev, - unsigned long page_start, - unsigned long num_pages) -{ - unsigned long cur_offset; - struct ttm_buffer_object *bo; - struct ttm_buffer_object *best_bo = NULL; - - bo = RB_ROOT(&bdev->addr_space_rb); - while (bo != NULL) { - cur_offset = bo->vm_node->start; - if (page_start >= cur_offset) { - best_bo = bo; - if (page_start == cur_offset) - break; - bo = RB_RIGHT(bo, vm_rb); - } else - bo = RB_LEFT(bo, vm_rb); - } - - if (unlikely(best_bo == NULL)) - return NULL; - - if (unlikely((best_bo->vm_node->start + best_bo->num_pages) < - (page_start + num_pages))) - return NULL; - - return best_bo; -} int ttm_bo_vm_fault(struct uvm_faultinfo *, vaddr_t, vm_page_t *, int, int, vm_fault_t, vm_prot_t, int); @@ -187,9 +134,9 @@ ttm_bo_vm_fault(struct uvm_faultinfo *ufi, vaddr_t vaddr, vm_page_t *pps, } page_offset = ((address - ufi->entry->start) >> PAGE_SHIFT) + - bo->vm_node->start - (ufi->entry->offset >> PAGE_SHIFT); + drm_vma_node_start(&bo->vma_node) - (ufi->entry->offset >> PAGE_SHIFT); page_last = ((ufi->entry->end - ufi->entry->start) >> PAGE_SHIFT) + - bo->vm_node->start - (ufi->entry->offset >> PAGE_SHIFT); + drm_vma_node_start(&bo->vma_node) - (ufi->entry->offset >> PAGE_SHIFT); if (unlikely(page_offset >= bo->num_pages)) { retval = VM_PAGER_ERROR; @@ -301,6 +248,30 @@ struct uvm_pagerops ttm_bo_vm_ops = { .pgo_detach = ttm_bo_vm_detach }; +static struct ttm_buffer_object *ttm_bo_vm_lookup(struct ttm_bo_device *bdev, + unsigned long offset, + unsigned long pages) +{ + struct drm_vma_offset_node *node; + struct ttm_buffer_object *bo = NULL; + + drm_vma_offset_lock_lookup(&bdev->vma_manager); + + node = drm_vma_offset_lookup_locked(&bdev->vma_manager, offset, pages); + if (likely(node)) { + bo = container_of(node, struct ttm_buffer_object, vma_node); + if (!kref_get_unless_zero(&bo->kref)) + bo = NULL; + } + + drm_vma_offset_unlock_lookup(&bdev->vma_manager); + + if (!bo) + pr_err("Could not find buffer object to map\n"); + + return bo; +} + struct uvm_object * ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev) { @@ -308,14 +279,9 @@ ttm_bo_mmap(voff_t off, vsize_t size, struct ttm_bo_device *bdev) struct ttm_buffer_object *bo; int ret; - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT); - if (likely(bo != NULL) && !kref_get_unless_zero(&bo->kref)) - bo = NULL; - read_unlock(&bdev->vm_lock); - - if (unlikely(bo == NULL)) { - pr_err("Could not find buffer object to map\n"); + bo = ttm_bo_vm_lookup(bdev, off >> PAGE_SHIFT, size >> PAGE_SHIFT); + if (unlikely(!bo)) { + ret = -EINVAL; return NULL; } @@ -383,12 +349,7 @@ ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, bool no_wait = false; bool dummy; - read_lock(&bdev->vm_lock); - bo = ttm_bo_vm_lookup_rb(bdev, dev_offset, 1); - if (likely(bo != NULL)) - ttm_bo_reference(bo); - read_unlock(&bdev->vm_lock); - + bo = ttm_bo_vm_lookup(bdev, dev_offset, 1); if (unlikely(bo == NULL)) return -EFAULT; @@ -402,7 +363,7 @@ ssize_t ttm_bo_io(struct ttm_bo_device *bdev, struct file *filp, if (unlikely(ret != 0)) goto out_unref; - kmap_offset = dev_offset - bo->vm_node->start; + kmap_offset = dev_offset - drm_vma_node_start(&bo->vma_node); if (unlikely(kmap_offset >= bo->num_pages)) { ret = -EFBIG; goto out_unref;