1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
|
From 78113fe6d925f0bbb2875d69813a6de48be56711 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Christian=20K=C3=B6nig?= <christian.koenig@amd.com>
Date: Sat, 1 Sep 2018 13:25:31 +0200
Subject: [PATCH 0319/2940] drm/amdgpu: separate per VM BOs from normal in the
moved state
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Allows us to avoid taking the spinlock in more places.
Change-Id: I54c3dcb21ce28b3c5dde39c8f77af8016fbbd8a4
Signed-off-by: Christian König <christian.koenig@amd.com>
Reviewed-by: Junwei Zhang <Jerry.Zhang@amd.com>
Signed-off-by: Kalyan Alle <kalyan.alle@amd.com>
---
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c | 71 +++++++++++++-------------
drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h | 7 ++-
2 files changed, 40 insertions(+), 38 deletions(-)
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
index 0a334a04e8c9..cfe946f01b93 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.c
@@ -342,9 +342,7 @@ int amdgpu_vm_validate_pt_bos(struct amdgpu_device *adev, struct amdgpu_vm *vm,
break;
if (bo->tbo.type != ttm_bo_type_kernel) {
- spin_lock(&vm->moved_lock);
list_move(&bo_base->vm_status, &vm->moved);
- spin_unlock(&vm->moved_lock);
} else {
if (vm->use_cpu_for_update)
r = amdgpu_bo_kmap(bo, NULL);
@@ -1758,10 +1756,6 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
amdgpu_asic_flush_hdp(adev, NULL);
}
- spin_lock(&vm->moved_lock);
- list_del_init(&bo_va->base.vm_status);
- spin_unlock(&vm->moved_lock);
-
/* If the BO is not in its preferred location add it back to
* the evicted list so that it gets validated again on the
* next command submission.
@@ -1770,9 +1764,13 @@ int amdgpu_vm_bo_update(struct amdgpu_device *adev,
uint32_t mem_type = bo->tbo.mem.mem_type;
if (!(bo->preferred_domains & amdgpu_mem_type_to_domain(mem_type)))
- list_add_tail(&bo_va->base.vm_status, &vm->evicted);
+ list_move_tail(&bo_va->base.vm_status, &vm->evicted);
else
- list_add(&bo_va->base.vm_status, &vm->idle);
+ list_move(&bo_va->base.vm_status, &vm->idle);
+ } else {
+ spin_lock(&vm->invalidated_lock);
+ list_del_init(&bo_va->base.vm_status);
+ spin_unlock(&vm->invalidated_lock);
}
list_splice_init(&bo_va->invalids, &bo_va->valids);
@@ -1998,41 +1996,41 @@ int amdgpu_vm_handle_moved(struct amdgpu_device *adev,
struct amdgpu_vm *vm)
{
struct amdgpu_bo_va *bo_va, *tmp;
- struct list_head moved;
+ struct reservation_object *resv;
bool clear;
int r;
- INIT_LIST_HEAD(&moved);
- spin_lock(&vm->moved_lock);
- list_splice_init(&vm->moved, &moved);
- spin_unlock(&vm->moved_lock);
-
- list_for_each_entry_safe(bo_va, tmp, &moved, base.vm_status) {
- struct reservation_object *resv = bo_va->base.bo->tbo.resv;
-
+ list_for_each_entry_safe(bo_va, tmp, &vm->moved, base.vm_status) {
/* Per VM BOs never need to bo cleared in the page tables */
- if (resv == vm->root.base.bo->tbo.resv)
- clear = false;
+ r = amdgpu_vm_bo_update(adev, bo_va, false);
+ if (r)
+ return r;
+ }
+
+ spin_lock(&vm->invalidated_lock);
+ while (!list_empty(&vm->invalidated)) {
+ bo_va = list_first_entry(&vm->invalidated, struct amdgpu_bo_va,
+ base.vm_status);
+ resv = bo_va->base.bo->tbo.resv;
+ spin_unlock(&vm->invalidated_lock);
+
/* Try to reserve the BO to avoid clearing its ptes */
- else if (!amdgpu_vm_debug && reservation_object_trylock(resv))
+ if (!amdgpu_vm_debug && reservation_object_trylock(resv))
clear = false;
/* Somebody else is using the BO right now */
else
clear = true;
r = amdgpu_vm_bo_update(adev, bo_va, clear);
- if (r) {
- spin_lock(&vm->moved_lock);
- list_splice(&moved, &vm->moved);
- spin_unlock(&vm->moved_lock);
+ if (r)
return r;
- }
- if (!clear && resv != vm->root.base.bo->tbo.resv)
+ if (!clear)
reservation_object_unlock(resv);
-
+ spin_lock(&vm->invalidated_lock);
}
-
+ spin_unlock(&vm->invalidated_lock);
+
return 0;
}
@@ -2096,9 +2094,7 @@ static void amdgpu_vm_bo_insert_map(struct amdgpu_device *adev,
if (bo && bo->tbo.resv == vm->root.base.bo->tbo.resv &&
!bo_va->base.moved) {
- spin_lock(&vm->moved_lock);
list_move(&bo_va->base.vm_status, &vm->moved);
- spin_unlock(&vm->moved_lock);
}
trace_amdgpu_vm_bo_map(bo_va, mapping);
}
@@ -2458,9 +2454,9 @@ void amdgpu_vm_bo_rmv(struct amdgpu_device *adev,
list_del(&bo_va->base.bo_list);
- spin_lock(&vm->moved_lock);
+ spin_lock(&vm->invalidated_lock);
list_del(&bo_va->base.vm_status);
- spin_unlock(&vm->moved_lock);
+ spin_unlock(&vm->invalidated_lock);
list_for_each_entry_safe(mapping, next, &bo_va->valids, list) {
list_del(&mapping->list);
@@ -2517,10 +2513,12 @@ void amdgpu_vm_bo_invalidate(struct amdgpu_device *adev,
if (bo->tbo.type == ttm_bo_type_kernel) {
list_move(&bo_base->vm_status, &vm->relocated);
- } else {
- spin_lock(&bo_base->vm->moved_lock);
+ } else if (bo->tbo.resv == vm->root.base.bo->tbo.resv) {
list_move(&bo_base->vm_status, &vm->moved);
- spin_unlock(&bo_base->vm->moved_lock);
+ } else {
+ spin_lock(&vm->invalidated_lock);
+ list_move(&bo_base->vm_status, &vm->invalidated);
+ spin_unlock(&vm->invalidated_lock);
}
}
}
@@ -2665,9 +2663,10 @@ int amdgpu_vm_init(struct amdgpu_device *adev, struct amdgpu_vm *vm,
vm->reserved_vmid[i] = NULL;
INIT_LIST_HEAD(&vm->evicted);
INIT_LIST_HEAD(&vm->relocated);
- spin_lock_init(&vm->moved_lock);
INIT_LIST_HEAD(&vm->moved);
INIT_LIST_HEAD(&vm->idle);
+ INIT_LIST_HEAD(&vm->invalidated);
+ spin_lock_init(&vm->invalidated_lock);
INIT_LIST_HEAD(&vm->freed);
/* create scheduler entity for page table updates */
diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
index 94fe47890adf..3787c1973f91 100644
--- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
+++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vm.h
@@ -201,13 +201,16 @@ struct amdgpu_vm {
/* PT BOs which relocated and their parent need an update */
struct list_head relocated;
- /* BOs moved, but not yet updated in the PT */
+ /* per VM BOs moved, but not yet updated in the PT */
struct list_head moved;
- spinlock_t moved_lock;
/* All BOs of this VM not currently in the state machine */
struct list_head idle;
+ /* regular invalidated BOs, but not yet updated in the PT */
+ struct list_head invalidated;
+ spinlock_t invalidated_lock;
+
/* BO mappings freed, but not yet updated in the PT */
struct list_head freed;
--
2.17.1
|
