1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
|
// SPDX-License-Identifier: GPL-2.0
#include "kublk.h"
static enum io_uring_op ublk_to_uring_op(const struct ublksrv_io_desc *iod, int zc)
{
unsigned ublk_op = ublksrv_get_op(iod);
if (ublk_op == UBLK_IO_OP_READ)
return zc ? IORING_OP_READ_FIXED : IORING_OP_READ;
else if (ublk_op == UBLK_IO_OP_WRITE)
return zc ? IORING_OP_WRITE_FIXED : IORING_OP_WRITE;
ublk_assert(0);
}
static int loop_queue_flush_io(struct ublk_thread *t, struct ublk_queue *q,
const struct ublksrv_io_desc *iod, int tag)
{
unsigned ublk_op = ublksrv_get_op(iod);
struct io_uring_sqe *sqe[1];
ublk_io_alloc_sqes(t, sqe, 1);
io_uring_prep_fsync(sqe[0], ublk_get_registered_fd(q, 1) /*fds[1]*/, IORING_FSYNC_DATASYNC);
io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);
/* bit63 marks us as tgt io */
sqe[0]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);
return 1;
}
static int loop_queue_tgt_rw_io(struct ublk_thread *t, struct ublk_queue *q,
const struct ublksrv_io_desc *iod, int tag)
{
unsigned ublk_op = ublksrv_get_op(iod);
unsigned zc = ublk_queue_use_zc(q);
unsigned auto_zc = ublk_queue_use_auto_zc(q);
enum io_uring_op op = ublk_to_uring_op(iod, zc | auto_zc);
struct ublk_io *io = ublk_get_io(q, tag);
__u64 offset = iod->start_sector << 9;
__u32 len = iod->nr_sectors << 9;
struct io_uring_sqe *sqe[3];
void *addr = io->buf_addr;
unsigned short buf_index = ublk_io_buf_idx(t, q, tag);
if (iod->op_flags & UBLK_IO_F_INTEGRITY) {
ublk_io_alloc_sqes(t, sqe, 1);
/* Use second backing file for integrity data */
io_uring_prep_rw(op, sqe[0], ublk_get_registered_fd(q, 2),
io->integrity_buf,
ublk_integrity_len(q, len),
ublk_integrity_len(q, offset));
sqe[0]->flags = IOSQE_FIXED_FILE;
/* tgt_data = 1 indicates integrity I/O */
sqe[0]->user_data = build_user_data(tag, ublk_op, 1, q->q_id, 1);
}
if (!zc || auto_zc) {
ublk_io_alloc_sqes(t, sqe, 1);
if (!sqe[0])
return -ENOMEM;
io_uring_prep_rw(op, sqe[0], ublk_get_registered_fd(q, 1) /*fds[1]*/,
addr,
len,
offset);
if (auto_zc)
sqe[0]->buf_index = buf_index;
io_uring_sqe_set_flags(sqe[0], IOSQE_FIXED_FILE);
/* bit63 marks us as tgt io */
sqe[0]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);
return !!(iod->op_flags & UBLK_IO_F_INTEGRITY) + 1;
}
ublk_io_alloc_sqes(t, sqe, 3);
io_uring_prep_buf_register(sqe[0], q, tag, q->q_id, buf_index);
sqe[0]->flags |= IOSQE_CQE_SKIP_SUCCESS | IOSQE_IO_HARDLINK;
sqe[0]->user_data = build_user_data(tag,
ublk_cmd_op_nr(sqe[0]->cmd_op), 0, q->q_id, 1);
io_uring_prep_rw(op, sqe[1], ublk_get_registered_fd(q, 1) /*fds[1]*/, 0,
len,
offset);
sqe[1]->buf_index = buf_index;
sqe[1]->flags |= IOSQE_FIXED_FILE | IOSQE_IO_HARDLINK;
sqe[1]->user_data = build_user_data(tag, ublk_op, 0, q->q_id, 1);
io_uring_prep_buf_unregister(sqe[2], q, tag, q->q_id, buf_index);
sqe[2]->user_data = build_user_data(tag, ublk_cmd_op_nr(sqe[2]->cmd_op), 0, q->q_id, 1);
return !!(iod->op_flags & UBLK_IO_F_INTEGRITY) + 2;
}
static int loop_queue_tgt_io(struct ublk_thread *t, struct ublk_queue *q, int tag)
{
const struct ublksrv_io_desc *iod = ublk_get_iod(q, tag);
unsigned ublk_op = ublksrv_get_op(iod);
int ret;
switch (ublk_op) {
case UBLK_IO_OP_FLUSH:
ret = loop_queue_flush_io(t, q, iod, tag);
break;
case UBLK_IO_OP_WRITE_ZEROES:
case UBLK_IO_OP_DISCARD:
ret = -ENOTSUP;
break;
case UBLK_IO_OP_READ:
case UBLK_IO_OP_WRITE:
ret = loop_queue_tgt_rw_io(t, q, iod, tag);
break;
default:
ret = -EINVAL;
break;
}
ublk_dbg(UBLK_DBG_IO, "%s: tag %d ublk io %x %llx %u\n", __func__, tag,
iod->op_flags, iod->start_sector, iod->nr_sectors << 9);
return ret;
}
static int ublk_loop_queue_io(struct ublk_thread *t, struct ublk_queue *q,
int tag)
{
int queued = loop_queue_tgt_io(t, q, tag);
ublk_queued_tgt_io(t, q, tag, queued);
return 0;
}
static void ublk_loop_io_done(struct ublk_thread *t, struct ublk_queue *q,
const struct io_uring_cqe *cqe)
{
unsigned tag = user_data_to_tag(cqe->user_data);
unsigned op = user_data_to_op(cqe->user_data);
struct ublk_io *io = ublk_get_io(q, tag);
if (cqe->res < 0) {
io->result = cqe->res;
ublk_err("%s: io failed op %x user_data %lx\n",
__func__, op, cqe->user_data);
} else if (op != ublk_cmd_op_nr(UBLK_U_IO_UNREGISTER_IO_BUF)) {
__s32 data_len = user_data_to_tgt_data(cqe->user_data)
? ublk_integrity_data_len(q, cqe->res)
: cqe->res;
if (!io->result || data_len < io->result)
io->result = data_len;
}
/* buffer register op is IOSQE_CQE_SKIP_SUCCESS */
if (op == ublk_cmd_op_nr(UBLK_U_IO_REGISTER_IO_BUF))
io->tgt_ios += 1;
if (ublk_completed_tgt_io(t, q, tag))
ublk_complete_io(t, q, tag, io->result);
}
static int ublk_loop_memset_file(int fd, __u8 byte, size_t len)
{
off_t offset = 0;
__u8 buf[4096];
memset(buf, byte, sizeof(buf));
while (len) {
int ret = pwrite(fd, buf, min(len, sizeof(buf)), offset);
if (ret < 0)
return -errno;
if (!ret)
return -EIO;
len -= ret;
offset += ret;
}
return 0;
}
static int ublk_loop_tgt_init(const struct dev_ctx *ctx, struct ublk_dev *dev)
{
unsigned long long bytes;
unsigned long blocks;
int ret;
struct ublk_params p = {
.types = UBLK_PARAM_TYPE_BASIC | UBLK_PARAM_TYPE_DMA_ALIGN,
.basic = {
.attrs = UBLK_ATTR_VOLATILE_CACHE,
.logical_bs_shift = 9,
.physical_bs_shift = 12,
.io_opt_shift = 12,
.io_min_shift = 9,
.max_sectors = dev->dev_info.max_io_buf_bytes >> 9,
},
.dma = {
.alignment = 511,
},
};
ublk_set_integrity_params(ctx, &p);
if (ctx->auto_zc_fallback) {
ublk_err("%s: not support auto_zc_fallback\n", __func__);
return -EINVAL;
}
/* Use O_DIRECT only for data file */
ret = backing_file_tgt_init(dev, 1);
if (ret)
return ret;
/* Expect a second file for integrity data */
if (dev->tgt.nr_backing_files != 1 + !!ctx->metadata_size)
return -EINVAL;
blocks = dev->tgt.backing_file_size[0] >> p.basic.logical_bs_shift;
if (ctx->metadata_size) {
unsigned long metadata_blocks =
dev->tgt.backing_file_size[1] / ctx->metadata_size;
unsigned long integrity_len;
/* Ensure both data and integrity data fit in backing files */
blocks = min(blocks, metadata_blocks);
integrity_len = blocks * ctx->metadata_size;
/*
* Initialize PI app tag and ref tag to 0xFF
* to disable bio-integrity-auto checks
*/
ret = ublk_loop_memset_file(dev->fds[2], 0xFF, integrity_len);
if (ret)
return ret;
}
bytes = blocks << p.basic.logical_bs_shift;
dev->tgt.dev_size = bytes;
p.basic.dev_sectors = bytes >> 9;
dev->tgt.params = p;
return 0;
}
const struct ublk_tgt_ops loop_tgt_ops = {
.name = "loop",
.init_tgt = ublk_loop_tgt_init,
.deinit_tgt = backing_file_tgt_deinit,
.queue_io = ublk_loop_queue_io,
.tgt_io_done = ublk_loop_io_done,
};
|
