1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
|
From a54a155b4b30d2154e0c32ef32eb528fc0fc54be Mon Sep 17 00:00:00 2001
From: Felix Kuehling <Felix.Kuehling@amd.com>
Date: Thu, 4 Jan 2018 20:14:54 -0500
Subject: [PATCH 3130/4131] drm/amdgpu: Update page diretories after page
tables
With an upcoming amdgpu change, huge page handling during page table
updates can invalidate page directory entries again. Therefore update
page directories only after updating page tables.
Change-Id: I3de514abac89ef2864808fc4813c4248ca7bebe0
Signed-off-by: Felix Kuehling <Felix.Kuehling@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 97 +++++++++++++++++-------
1 file changed, 69 insertions(+), 28 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 606677a..e386fb3 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -377,9 +377,10 @@ static int amdgpu_amdkfd_validate(void *param, struct amdgpu_bo *bo)
/* vm_validate_pt_pd_bos - Validate page table and directory BOs
*
- * Also updates page directory entries so we don't need to do this
- * again later until the page directory is validated again (e.g. after
- * an eviction or allocating new page tables).
+ * Page directories are not updated here because huge page handling
+ * during page table updates can invalidate page directory entries
+ * again. Page directories are only updated after updating page
+ * tables.
*/
static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
{
@@ -411,11 +412,37 @@ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
}
}
+ return 0;
+}
+
+static int sync_vm_fence(struct amdgpu_device *adev, struct amdgpu_sync *sync,
+ struct dma_fence *f)
+{
+ int ret = amdgpu_sync_fence(adev, sync, f, false);
+
+ /* Sync objects can't handle multiple GPUs (contexts) updating
+ * sync->last_vm_update. Fortunately we don't need it for
+ * KFD's purposes, so we can just drop that fence.
+ */
+ if (sync->last_vm_update) {
+ dma_fence_put(sync->last_vm_update);
+ sync->last_vm_update = NULL;
+ }
+
+ return ret;
+}
+
+static int vm_update_pds(struct amdgpu_vm *vm, struct amdgpu_sync *sync)
+{
+ struct amdgpu_bo *pd = vm->root.base.bo;
+ struct amdgpu_device *adev = amdgpu_ttm_adev(pd->tbo.bdev);
+ int ret;
+
ret = amdgpu_vm_update_directories(adev, vm);
- if (ret != 0)
+ if (ret)
return ret;
- return 0;
+ return sync_vm_fence(adev, sync, vm->last_update);
}
/* add_bo_to_vm - Add a BO to a VM
@@ -428,7 +455,7 @@ static int vm_validate_pt_pd_bos(struct amdgpu_vm *vm)
* 2. Add BO to the VM
* 3. Determine ASIC-specific PTE flags
* 4. Alloc page tables and directories if needed
- * 4a. Validate new page tables and directories and update directories
+ * 4a. Validate new page tables and directories
*/
static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_vm *avm, bool is_aql,
@@ -939,16 +966,7 @@ static int unmap_bo_from_gpuvm(struct amdgpu_device *adev,
/* Add the eviction fence back */
amdgpu_bo_fence(pd, &kvm->process_info->eviction_fence->base, true);
- amdgpu_sync_fence(adev, sync, bo_va->last_pt_update, false);
-
- /* Sync objects can't handle multiple GPUs (contexts) updating
- * sync->last_vm_update. Fortunately we don't need it for
- * KFD's purposes, so we can just drop that fence.
- */
- if (sync->last_vm_update) {
- dma_fence_put(sync->last_vm_update);
- sync->last_vm_update = NULL;
- }
+ sync_vm_fence(adev, sync, bo_va->last_pt_update);
return 0;
}
@@ -973,18 +991,7 @@ static int update_gpuvm_pte(struct amdgpu_device *adev,
return ret;
}
- amdgpu_sync_fence(adev, sync, bo_va->last_pt_update, false);
-
- /* Sync objects can't handle multiple GPUs (contexts) updating
- * sync->last_vm_update. Fortunately we don't need it for
- * KFD's purposes, so we can just drop that fence.
- */
- if (sync->last_vm_update) {
- dma_fence_put(sync->last_vm_update);
- sync->last_vm_update = NULL;
- }
-
- return 0;
+ return sync_vm_fence(adev, sync, bo_va->last_pt_update);
}
static int map_bo_to_gpuvm(struct amdgpu_device *adev,
@@ -1316,6 +1323,13 @@ int amdgpu_amdkfd_gpuvm_map_memory_to_gpu(
pr_err("Failed to map radeon bo to gpuvm\n");
goto map_bo_to_gpuvm_failed;
}
+
+ ret = vm_update_pds(vm, ctx.sync);
+ if (ret) {
+ pr_err("Failed to update page directories\n");
+ goto map_bo_to_gpuvm_failed;
+ }
+
entry->is_mapped = true;
mem->mapped_to_gpu_memory++;
pr_debug("\t INC mapping count %d\n",
@@ -1914,6 +1928,22 @@ static int process_validate_vms(struct amdkfd_process_info *process_info)
return 0;
}
+static int process_update_pds(struct amdkfd_process_info *process_info,
+ struct amdgpu_sync *sync)
+{
+ struct amdkfd_vm *peer_vm;
+ int ret;
+
+ list_for_each_entry(peer_vm, &process_info->vm_list_head,
+ vm_list_node) {
+ ret = vm_update_pds(&peer_vm->base, sync);
+ if (ret)
+ return ret;
+ }
+
+ return 0;
+}
+
/* Evict a userptr BO by stopping the queues if necessary
*
* Runs in MMU notifier, may be in RECLAIM_FS context. This means it
@@ -2165,6 +2195,10 @@ static int validate_invalid_user_pages(struct amdkfd_process_info *process_info)
}
}
}
+
+ /* Update page directories */
+ ret = process_update_pds(process_info, &sync);
+
unreserve_out:
list_for_each_entry(peer_vm, &process_info->vm_list_head,
vm_list_node)
@@ -2358,6 +2392,13 @@ int amdgpu_amdkfd_gpuvm_restore_process_bos(void *info, struct dma_fence **ef)
}
}
+ /* Update page directories */
+ ret = process_update_pds(process_info, ctx.sync);
+ if (ret) {
+ pr_debug("Memory eviction: update PDs failed. Try again\n");
+ goto validate_map_fail;
+ }
+
amdgpu_sync_wait(ctx.sync, false);
/* Release old eviction fence and create new one, because fence only
--
2.7.4
|
