1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
|
From c1d658ee6665242e1e8a2ea6c250e3f8c48ce47a Mon Sep 17 00:00:00 2001
From: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
Date: Fri, 18 Nov 2016 16:58:38 -0500
Subject: [PATCH 1557/4131] drm/amdgpu: Remove / Add eviction fences from BO
KFD BOs could be shared (IPC, Graphics Interop).
During BO validate temporarily remove other processes eviction fences.
During BO free remove only the eviction fence of the current process.
Change-Id: Id2d74e6a8edd781181f28eff7b25e9d79338ab8a
Signed-off-by: Harish Kasiviswanathan <Harish.Kasiviswanathan@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c | 170 ++++++++++++++++++++++-
1 file changed, 163 insertions(+), 7 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
index 45749a5..0926000 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_amdkfd_gpuvm.c
@@ -156,6 +156,145 @@ void amdgpu_amdkfd_unreserve_system_memory_limit(struct amdgpu_bo *bo)
spin_unlock(&kfd_mem_limit.mem_limit_lock);
}
+
+/* amdgpu_amdkfd_remove_eviction_fence - Removes eviction fence(s) from BO's
+ * reservation object.
+ *
+ * @bo: [IN] Remove eviction fence(s) from this BO
+ * @ef: [IN] If ef is specified, then this eviction fence is removed if it
+ * is present in the shared list.
+ * @ef_list: [OUT] Returns list of eviction fences. These fences are removed
+ * from BO's reservation object shared list.
+ * @ef_count: [OUT] Number of fences in ef_list.
+ *
+ * NOTE: If called with ef_list, then amdgpu_amdkfd_add_eviction_fence must be
+ * called to restore the eviction fences and to avoid memory leak. This is
+ * useful for shared BOs.
+ * NOTE: Must be called with BO reserved i.e. bo->tbo.resv->lock held.
+ */
+static int amdgpu_amdkfd_remove_eviction_fence(struct amdgpu_bo *bo,
+ struct amdgpu_amdkfd_fence *ef,
+ struct amdgpu_amdkfd_fence ***ef_list,
+ unsigned int *ef_count)
+{
+ struct reservation_object_list *fobj;
+ struct reservation_object *resv;
+ unsigned int i = 0, j = 0, k = 0, shared_count;
+ unsigned int count = 0;
+ struct amdgpu_amdkfd_fence **fence_list;
+
+ if (!ef && !ef_list)
+ return -EINVAL;
+
+ if (ef_list) {
+ *ef_list = NULL;
+ *ef_count = 0;
+ }
+
+ resv = bo->tbo.resv;
+ fobj = reservation_object_get_list(resv);
+
+ if (!fobj)
+ return 0;
+
+ preempt_disable();
+ write_seqcount_begin(&resv->seq);
+
+ /* Go through all the shared fences in the resevation object. If
+ * ef is specified and it exists in the list, remove it and reduce the
+ * count. If ef is not specified, then get the count of eviction fences
+ * present.
+ */
+ shared_count = fobj->shared_count;
+ for (i = 0; i < shared_count; ++i) {
+ struct fence *f;
+
+ f = rcu_dereference_protected(fobj->shared[i],
+ reservation_object_held(resv));
+
+ if (ef) {
+ if (f->context == ef->base.context) {
+ fence_put(f);
+ fobj->shared_count--;
+ } else
+ RCU_INIT_POINTER(fobj->shared[j++], f);
+
+ } else if (to_amdgpu_amdkfd_fence(f))
+ count++;
+ }
+ write_seqcount_end(&resv->seq);
+ preempt_enable();
+
+ if (ef || !count)
+ return 0;
+
+ /* Alloc memory for count number of eviction fence pointers. Fill the
+ * ef_list array and ef_count
+ */
+
+ fence_list = kcalloc(count, sizeof(struct amdgpu_amdkfd_fence *),
+ GFP_KERNEL);
+ if (!fence_list)
+ return -ENOMEM;
+
+ preempt_disable();
+ write_seqcount_begin(&resv->seq);
+
+ j = 0;
+ for (i = 0; i < shared_count; ++i) {
+ struct fence *f;
+ struct amdgpu_amdkfd_fence *efence;
+
+ f = rcu_dereference_protected(fobj->shared[i],
+ reservation_object_held(resv));
+
+ efence = to_amdgpu_amdkfd_fence(f);
+ if (efence) {
+ fence_list[k++] = efence;
+ fobj->shared_count--;
+ } else
+ RCU_INIT_POINTER(fobj->shared[j++], f);
+ }
+
+ write_seqcount_end(&resv->seq);
+ preempt_enable();
+
+ *ef_list = fence_list;
+ *ef_count = k;
+
+ return 0;
+}
+
+/* amdgpu_amdkfd_add_eviction_fence - Adds eviction fence(s) back into BO's
+ * reservation object.
+ *
+ * @bo: [IN] Add eviction fences to this BO
+ * @ef_list: [IN] List of eviction fences to be added
+ * @ef_count: [IN] Number of fences in ef_list.
+ *
+ * NOTE: Must call amdgpu_amdkfd_remove_eviction_fence before calling this
+ * function.
+ */
+static void amdgpu_amdkfd_add_eviction_fence(struct amdgpu_bo *bo,
+ struct amdgpu_amdkfd_fence **ef_list,
+ unsigned int ef_count)
+{
+ int i;
+
+ if (!ef_list || !ef_count)
+ return;
+
+ for (i = 0; i < ef_count; i++) {
+ amdgpu_bo_fence(bo, &ef_list[i]->base, true);
+ /* Readding the fence takes an additional reference. Drop that
+ * reference.
+ */
+ fence_put(&ef_list[i]->base);
+ }
+
+ kfree(ef_list);
+}
+
static int add_bo_to_vm(struct amdgpu_device *adev, struct kgd_mem *mem,
struct amdgpu_vm *avm, bool is_aql,
struct kfd_bo_va_list **p_bo_va_entry)
@@ -216,12 +355,25 @@ static int amdgpu_amdkfd_bo_validate(struct amdgpu_bo *bo, uint32_t domain,
if (!amdgpu_ttm_tt_get_usermm(bo->tbo.ttm)) {
amdgpu_ttm_placement_from_domain(bo, domain);
+
ret = ttm_bo_validate(&bo->tbo, &bo->placement,
false, false);
if (ret)
goto validate_fail;
- if (wait)
- ret = ttm_bo_wait(&bo->tbo, false, false);
+ if (wait) {
+ struct amdgpu_amdkfd_fence **ef_list;
+ unsigned int ef_count;
+
+ ret = amdgpu_amdkfd_remove_eviction_fence(bo, NULL,
+ &ef_list,
+ &ef_count);
+ if (ret)
+ goto validate_fail;
+
+ ttm_bo_wait(&bo->tbo, false, false);
+ amdgpu_amdkfd_add_eviction_fence(bo, ef_list,
+ ef_count);
+ }
} else {
/* Userptrs are not pinned. Therefore we can use the
* bo->pin_count for our version of pinning without conflict.
@@ -270,7 +422,8 @@ static int validate_pt_pd_bos(struct amdgpu_vm *vm)
struct amdkfd_vm *kvm = container_of(vm, struct amdkfd_vm, base);
/* Remove eviction fence so that validate can wait on move fences */
- amdgpu_bo_fence(pd, NULL, false);
+ amdgpu_amdkfd_remove_eviction_fence(pd, kvm->eviction_fence,
+ NULL, NULL);
/* PTs share same reservation object as PD. So only fence PD */
for (i = 0; i <= vm->max_pde_used; ++i) {
@@ -1026,7 +1179,8 @@ int amdgpu_amdkfd_gpuvm_free_memory_of_gpu(
* TODO: Log an error condition if the bo still has the eviction fence
* attached
*/
- amdgpu_bo_fence(mem->bo, NULL, false);
+ amdgpu_amdkfd_remove_eviction_fence(mem->bo, master_vm->eviction_fence,
+ NULL, NULL);
pr_debug("Releasing BO with VA 0x%llx, size %lu bytes\n",
mem->va,
mem->bo->tbo.mem.size);
@@ -1337,12 +1491,14 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
unsigned mapped_before;
int ret = 0;
struct bo_vm_reservation_context ctx;
+ struct amdkfd_vm *master_vm;
int num_to_resume = 0;
BUG_ON(kgd == NULL);
BUG_ON(mem == NULL);
adev = (struct amdgpu_device *) kgd;
+ master_vm = ((struct amdkfd_vm *)vm)->master;
mutex_lock(&mem->lock);
@@ -1401,11 +1557,11 @@ int amdgpu_amdkfd_gpuvm_unmap_memory_from_gpu(
/* If BO is unmapped from all VMs, unfence it. It can be evicted if
* required.
- * TODO: For interop this will remove fences added by graphics driver.
- * Remove only KFD eviction fence
*/
if (mem->mapped_to_gpu_memory == 0)
- amdgpu_bo_fence(mem->bo, NULL, false);
+ amdgpu_amdkfd_remove_eviction_fence(mem->bo,
+ master_vm->eviction_fence,
+ NULL, NULL);
if (mapped_before == mem->mapped_to_gpu_memory) {
pr_debug("BO size %lu bytes at va 0x%llx is not mapped on GPU %x:%x.%x\n",
--
2.7.4
|
