1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
|
// SPDX-License-Identifier: MIT
/*
* Copyright © 2022 Intel Corporation
*/
#include "xe_sa.h"
#include <linux/kernel.h>
#include <drm/drm_managed.h>
#include "xe_bo.h"
#include "xe_device_types.h"
#include "xe_map.h"
static void xe_sa_bo_manager_fini(struct drm_device *drm, void *arg)
{
struct xe_sa_manager *sa_manager = arg;
struct xe_bo *bo = sa_manager->bo;
if (!bo) {
drm_err(drm, "no bo for sa manager\n");
return;
}
drm_suballoc_manager_fini(&sa_manager->base);
if (sa_manager->is_iomem)
kvfree(sa_manager->cpu_ptr);
sa_manager->bo = NULL;
sa_manager->shadow = NULL;
}
/**
* __xe_sa_bo_manager_init() - Create and initialize the suballocator
* @tile: the &xe_tile where allocate
* @size: number of bytes to allocate
* @guard: number of bytes to exclude from suballocations
* @align: alignment for each suballocated chunk
* @flags: flags for suballocator
*
* Prepares the suballocation manager for suballocations.
*
* Return: a pointer to the &xe_sa_manager or an ERR_PTR on failure.
*/
struct xe_sa_manager *__xe_sa_bo_manager_init(struct xe_tile *tile, u32 size,
u32 guard, u32 align, u32 flags)
{
struct xe_device *xe = tile_to_xe(tile);
struct xe_sa_manager *sa_manager;
u32 managed_size;
struct xe_bo *bo;
int ret;
xe_tile_assert(tile, size > guard);
managed_size = size - guard;
sa_manager = drmm_kzalloc(&xe->drm, sizeof(*sa_manager), GFP_KERNEL);
if (!sa_manager)
return ERR_PTR(-ENOMEM);
bo = xe_managed_bo_create_pin_map(xe, tile, size,
XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
if (IS_ERR(bo)) {
drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n",
size / SZ_1K, bo);
return ERR_CAST(bo);
}
sa_manager->bo = bo;
sa_manager->is_iomem = bo->vmap.is_iomem;
if (bo->vmap.is_iomem) {
sa_manager->cpu_ptr = kvzalloc(managed_size, GFP_KERNEL);
if (!sa_manager->cpu_ptr)
return ERR_PTR(-ENOMEM);
} else {
sa_manager->cpu_ptr = bo->vmap.vaddr;
memset(sa_manager->cpu_ptr, 0, bo->ttm.base.size);
}
if (flags & XE_SA_BO_MANAGER_FLAG_SHADOW) {
struct xe_bo *shadow;
ret = drmm_mutex_init(&xe->drm, &sa_manager->swap_guard);
if (ret)
return ERR_PTR(ret);
shadow = xe_managed_bo_create_pin_map(xe, tile, size,
XE_BO_FLAG_VRAM_IF_DGFX(tile) |
XE_BO_FLAG_GGTT |
XE_BO_FLAG_GGTT_INVALIDATE |
XE_BO_FLAG_PINNED_NORESTORE);
if (IS_ERR(shadow)) {
drm_err(&xe->drm, "Failed to prepare %uKiB BO for SA manager (%pe)\n",
size / SZ_1K, shadow);
return ERR_CAST(shadow);
}
sa_manager->shadow = shadow;
}
drm_suballoc_manager_init(&sa_manager->base, managed_size, align);
ret = drmm_add_action_or_reset(&xe->drm, xe_sa_bo_manager_fini,
sa_manager);
if (ret)
return ERR_PTR(ret);
return sa_manager;
}
/**
* xe_sa_bo_swap_shadow() - Swap the SA BO with shadow BO.
* @sa_manager: the XE sub allocator manager
*
* Swaps the sub-allocator primary buffer object with shadow buffer object.
*
* Return: None.
*/
void xe_sa_bo_swap_shadow(struct xe_sa_manager *sa_manager)
{
struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
xe_assert(xe, sa_manager->shadow);
lockdep_assert_held(&sa_manager->swap_guard);
swap(sa_manager->bo, sa_manager->shadow);
if (!sa_manager->bo->vmap.is_iomem)
sa_manager->cpu_ptr = sa_manager->bo->vmap.vaddr;
}
/**
* xe_sa_bo_sync_shadow() - Sync the SA Shadow BO with primary BO.
* @sa_bo: the sub-allocator buffer object.
*
* Synchronize sub-allocator shadow buffer object with primary buffer object.
*
* Return: None.
*/
void xe_sa_bo_sync_shadow(struct drm_suballoc *sa_bo)
{
struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
xe_assert(xe, sa_manager->shadow);
lockdep_assert_held(&sa_manager->swap_guard);
xe_map_memcpy_to(xe, &sa_manager->shadow->vmap,
drm_suballoc_soffset(sa_bo),
xe_sa_bo_cpu_addr(sa_bo),
drm_suballoc_size(sa_bo));
}
/**
* __xe_sa_bo_new() - Make a suballocation but use custom gfp flags.
* @sa_manager: the &xe_sa_manager
* @size: number of bytes we want to suballocate
* @gfp: gfp flags used for memory allocation. Typically GFP_KERNEL.
*
* Try to make a suballocation of size @size.
*
* Return: a &drm_suballoc, or an ERR_PTR.
*/
struct drm_suballoc *__xe_sa_bo_new(struct xe_sa_manager *sa_manager, u32 size, gfp_t gfp)
{
/*
* BB to large, return -ENOBUFS indicating user should split
* array of binds into smaller chunks.
*/
if (size > sa_manager->base.size)
return ERR_PTR(-ENOBUFS);
return drm_suballoc_new(&sa_manager->base, size, gfp, true, 0);
}
/**
* xe_sa_bo_flush_write() - Copy the data from the sub-allocation to the GPU memory.
* @sa_bo: the &drm_suballoc to flush
*/
void xe_sa_bo_flush_write(struct drm_suballoc *sa_bo)
{
struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
if (!sa_manager->bo->vmap.is_iomem)
return;
xe_map_memcpy_to(xe, &sa_manager->bo->vmap, drm_suballoc_soffset(sa_bo),
xe_sa_bo_cpu_addr(sa_bo),
drm_suballoc_size(sa_bo));
}
/**
* xe_sa_bo_sync_read() - Copy the data from GPU memory to the sub-allocation.
* @sa_bo: the &drm_suballoc to sync
*/
void xe_sa_bo_sync_read(struct drm_suballoc *sa_bo)
{
struct xe_sa_manager *sa_manager = to_xe_sa_manager(sa_bo->manager);
struct xe_device *xe = tile_to_xe(sa_manager->bo->tile);
if (!sa_manager->bo->vmap.is_iomem)
return;
xe_map_memcpy_from(xe, xe_sa_bo_cpu_addr(sa_bo), &sa_manager->bo->vmap,
drm_suballoc_soffset(sa_bo),
drm_suballoc_size(sa_bo));
}
void xe_sa_bo_free(struct drm_suballoc *sa_bo,
struct dma_fence *fence)
{
drm_suballoc_free(sa_bo, fence);
}
|
