Commit 02208441 authored by Felix Kuehling's avatar Felix Kuehling Committed by Alex Deucher

drm/amdgpu: Add PASID management

Allows assigning a PASID to a VM for identifying VMs involved in page
faults. The global PASID manager is also exported in the KFD
interface so that AMDGPU and KFD can share the PASID space.

PASIDs of different sizes can be requested. On APUs, the PASID size
is deterined by the capabilities of the IOMMU. So KFD must be able
to allocate PASIDs in a smaller range.
Signed-off-by: default avatarFelix Kuehling <Felix.Kuehling@amd.com>
Acked-by: default avatarAlex Deucher <alexander.deucher@amd.com>
Reviewed-by: default avatarOded Gabbay <oded.gabbay@gmail.com>
Signed-off-by: default avatarAlex Deucher <alexander.deucher@amd.com>
parent ca290da8
...@@ -169,6 +169,8 @@ static const struct kfd2kgd_calls kfd2kgd = { ...@@ -169,6 +169,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
.get_vmem_size = get_vmem_size, .get_vmem_size = get_vmem_size,
.get_gpu_clock_counter = get_gpu_clock_counter, .get_gpu_clock_counter = get_gpu_clock_counter,
.get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
.alloc_pasid = amdgpu_vm_alloc_pasid,
.free_pasid = amdgpu_vm_free_pasid,
.program_sh_mem_settings = kgd_program_sh_mem_settings, .program_sh_mem_settings = kgd_program_sh_mem_settings,
.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
.init_pipeline = kgd_init_pipeline, .init_pipeline = kgd_init_pipeline,
......
...@@ -128,6 +128,8 @@ static const struct kfd2kgd_calls kfd2kgd = { ...@@ -128,6 +128,8 @@ static const struct kfd2kgd_calls kfd2kgd = {
.get_vmem_size = get_vmem_size, .get_vmem_size = get_vmem_size,
.get_gpu_clock_counter = get_gpu_clock_counter, .get_gpu_clock_counter = get_gpu_clock_counter,
.get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz, .get_max_engine_clock_in_mhz = get_max_engine_clock_in_mhz,
.alloc_pasid = amdgpu_vm_alloc_pasid,
.free_pasid = amdgpu_vm_free_pasid,
.program_sh_mem_settings = kgd_program_sh_mem_settings, .program_sh_mem_settings = kgd_program_sh_mem_settings,
.set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping, .set_pasid_vmid_mapping = kgd_set_pasid_vmid_mapping,
.init_pipeline = kgd_init_pipeline, .init_pipeline = kgd_init_pipeline,
......
...@@ -825,7 +825,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv) ...@@ -825,7 +825,7 @@ int amdgpu_driver_open_kms(struct drm_device *dev, struct drm_file *file_priv)
} }
r = amdgpu_vm_init(adev, &fpriv->vm, r = amdgpu_vm_init(adev, &fpriv->vm,
AMDGPU_VM_CONTEXT_GFX); AMDGPU_VM_CONTEXT_GFX, 0);
if (r) { if (r) {
kfree(fpriv); kfree(fpriv);
goto out_suspend; goto out_suspend;
......
...@@ -27,11 +27,58 @@ ...@@ -27,11 +27,58 @@
*/ */
#include <linux/dma-fence-array.h> #include <linux/dma-fence-array.h>
#include <linux/interval_tree_generic.h> #include <linux/interval_tree_generic.h>
#include <linux/idr.h>
#include <drm/drmP.h> #include <drm/drmP.h>
#include <drm/amdgpu_drm.h> #include <drm/amdgpu_drm.h>
#include "amdgpu.h" #include "amdgpu.h"
#include "amdgpu_trace.h" #include "amdgpu_trace.h"
/*
* PASID manager
*
* PASIDs are global address space identifiers that can be shared
* between the GPU, an IOMMU and the driver. VMs on different devices
* may use the same PASID if they share the same address
* space. Therefore PASIDs are allocated using a global IDA. VMs are
* looked up from the PASID per amdgpu_device.
*/
static DEFINE_IDA(amdgpu_vm_pasid_ida);
/**
* amdgpu_vm_alloc_pasid - Allocate a PASID
* @bits: Maximum width of the PASID in bits, must be at least 1
*
* Allocates a PASID of the given width while keeping smaller PASIDs
* available if possible.
*
* Returns a positive integer on success. Returns %-EINVAL if bits==0.
* Returns %-ENOSPC if no PASID was available. Returns %-ENOMEM on
* memory allocation failure.
*/
int amdgpu_vm_alloc_pasid(unsigned int bits)
{
int pasid = -EINVAL;
for (bits = min(bits, 31U); bits > 0; bits--) {
pasid = ida_simple_get(&amdgpu_vm_pasid_ida,
1U << (bits - 1), 1U << bits,
GFP_KERNEL);
if (pasid != -ENOSPC)
break;
}
return pasid;
}
/**
* amdgpu_vm_free_pasid - Free a PASID
* @pasid: PASID to free
*/
void amdgpu_vm_free_pasid(unsigned int pasid)
{
ida_simple_remove(&amdgpu_vm_pasid_ida, pasid);
}
/* /*
* GPUVM * GPUVM
* GPUVM is similar to the legacy gart on older asics, however * GPUVM is similar to the legacy gart on older asics, however
...@@ -2539,7 +2586,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_ ...@@ -2539,7 +2586,7 @@ void amdgpu_vm_adjust_size(struct amdgpu_device *adev, uint64_t vm_size, uint32_
* Init @vm fields. * Init @vm fields.
*/ */
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int vm_context) int vm_context, unsigned int pasid)
{ {
const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE, const unsigned align = min(AMDGPU_VM_PTB_ALIGN_SIZE,
AMDGPU_VM_PTE_COUNT(adev) * 8); AMDGPU_VM_PTE_COUNT(adev) * 8);
...@@ -2620,6 +2667,19 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, ...@@ -2620,6 +2667,19 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
goto error_free_root; goto error_free_root;
} }
if (pasid) {
unsigned long flags;
spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
r = idr_alloc(&adev->vm_manager.pasid_idr, vm, pasid, pasid + 1,
GFP_ATOMIC);
spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
if (r < 0)
goto error_free_root;
vm->pasid = pasid;
}
return 0; return 0;
error_free_root: error_free_root:
...@@ -2673,6 +2733,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm) ...@@ -2673,6 +2733,14 @@ void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm)
bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt; bool prt_fini_needed = !!adev->gart.gart_funcs->set_prt;
int i; int i;
if (vm->pasid) {
unsigned long flags;
spin_lock_irqsave(&adev->vm_manager.pasid_lock, flags);
idr_remove(&adev->vm_manager.pasid_idr, vm->pasid);
spin_unlock_irqrestore(&adev->vm_manager.pasid_lock, flags);
}
amd_sched_entity_fini(vm->entity.sched, &vm->entity); amd_sched_entity_fini(vm->entity.sched, &vm->entity);
if (!RB_EMPTY_ROOT(&vm->va)) { if (!RB_EMPTY_ROOT(&vm->va)) {
...@@ -2752,6 +2820,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev) ...@@ -2752,6 +2820,8 @@ void amdgpu_vm_manager_init(struct amdgpu_device *adev)
adev->vm_manager.vm_update_mode = 0; adev->vm_manager.vm_update_mode = 0;
#endif #endif
idr_init(&adev->vm_manager.pasid_idr);
spin_lock_init(&adev->vm_manager.pasid_lock);
} }
/** /**
...@@ -2765,6 +2835,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev) ...@@ -2765,6 +2835,9 @@ void amdgpu_vm_manager_fini(struct amdgpu_device *adev)
{ {
unsigned i, j; unsigned i, j;
WARN_ON(!idr_is_empty(&adev->vm_manager.pasid_idr));
idr_destroy(&adev->vm_manager.pasid_idr);
for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) { for (i = 0; i < AMDGPU_MAX_VMHUBS; ++i) {
struct amdgpu_vm_id_manager *id_mgr = struct amdgpu_vm_id_manager *id_mgr =
&adev->vm_manager.id_mgr[i]; &adev->vm_manager.id_mgr[i];
......
...@@ -25,6 +25,7 @@ ...@@ -25,6 +25,7 @@
#define __AMDGPU_VM_H__ #define __AMDGPU_VM_H__
#include <linux/rbtree.h> #include <linux/rbtree.h>
#include <linux/idr.h>
#include "gpu_scheduler.h" #include "gpu_scheduler.h"
#include "amdgpu_sync.h" #include "amdgpu_sync.h"
...@@ -148,8 +149,9 @@ struct amdgpu_vm { ...@@ -148,8 +149,9 @@ struct amdgpu_vm {
/* Scheduler entity for page table updates */ /* Scheduler entity for page table updates */
struct amd_sched_entity entity; struct amd_sched_entity entity;
/* client id */ /* client id and PASID (TODO: replace client_id with PASID) */
u64 client_id; u64 client_id;
unsigned int pasid;
/* dedicated to vm */ /* dedicated to vm */
struct amdgpu_vm_id *reserved_vmid[AMDGPU_MAX_VMHUBS]; struct amdgpu_vm_id *reserved_vmid[AMDGPU_MAX_VMHUBS];
...@@ -220,12 +222,20 @@ struct amdgpu_vm_manager { ...@@ -220,12 +222,20 @@ struct amdgpu_vm_manager {
* BIT1[= 0] Compute updated by SDMA [= 1] by CPU * BIT1[= 0] Compute updated by SDMA [= 1] by CPU
*/ */
int vm_update_mode; int vm_update_mode;
/* PASID to VM mapping, will be used in interrupt context to
* look up VM of a page fault
*/
struct idr pasid_idr;
spinlock_t pasid_lock;
}; };
int amdgpu_vm_alloc_pasid(unsigned int bits);
void amdgpu_vm_free_pasid(unsigned int pasid);
void amdgpu_vm_manager_init(struct amdgpu_device *adev); void amdgpu_vm_manager_init(struct amdgpu_device *adev);
void amdgpu_vm_manager_fini(struct amdgpu_device *adev); void amdgpu_vm_manager_fini(struct amdgpu_device *adev);
int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm, int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
int vm_context); int vm_context, unsigned int pasid);
void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm); void amdgpu_vm_fini(struct amdgpu_device *adev, struct amdgpu_vm *vm);
void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm, void amdgpu_vm_get_pd_bo(struct amdgpu_vm *vm,
struct list_head *validated, struct list_head *validated,
......
...@@ -112,6 +112,9 @@ struct tile_config { ...@@ -112,6 +112,9 @@ struct tile_config {
* *
* @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz * @get_max_engine_clock_in_mhz: Retrieves maximum GPU clock in MHz
* *
* @alloc_pasid: Allocate a PASID
* @free_pasid: Free a PASID
*
* @program_sh_mem_settings: A function that should initiate the memory * @program_sh_mem_settings: A function that should initiate the memory
* properties such as main aperture memory type (cache / non cached) and * properties such as main aperture memory type (cache / non cached) and
* secondary aperture base address, size and memory type. * secondary aperture base address, size and memory type.
...@@ -160,6 +163,9 @@ struct kfd2kgd_calls { ...@@ -160,6 +163,9 @@ struct kfd2kgd_calls {
uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd); uint32_t (*get_max_engine_clock_in_mhz)(struct kgd_dev *kgd);
int (*alloc_pasid)(unsigned int bits);
void (*free_pasid)(unsigned int pasid);
/* Register access functions */ /* Register access functions */
void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid, void (*program_sh_mem_settings)(struct kgd_dev *kgd, uint32_t vmid,
uint32_t sh_mem_config, uint32_t sh_mem_ape1_base, uint32_t sh_mem_config, uint32_t sh_mem_ape1_base,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment