blob: 9b66593735955a60d725dffd3b3e664fbc4afad0 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
|
#pragma once
#include <cstdint>
#include <drivers/disk.hpp>
#include <generic/hhdm.hpp>
#include <generic/lock/spinlock.hpp>
#include <generic/lock/mutex.hpp>
#include <generic/arch.hpp>
//#define NVME_ORANGE_TRACE
#define NVME_REG_CAP 0x00 // Controller Capabilities
#define NVME_REG_VS 0x08 // Version
#define NVME_REG_CC 0x14 // Controller Configuration
#define NVME_REG_CSTS 0x1C // Controller Status
#define NVME_REG_AQA 0x24 // Admin Queue Attributes
#define NVME_REG_ASQ 0x28 // Admin Submission Queue Base Address
#define NVME_REG_ACQ 0x30 // Admin Completion Queue Base Address
#define NVME_CAP_MQES_MASK 0xFFFF // Maximum Queue Entries Supported
#define NVME_CAP_CQR (1ULL << 16) // Contiguous Queues Required
#define NVME_CAP_DSTRD_MASK (0xFULL << 32) // Doorbell Stride
#define NVME_CAP_CSS_MASK (0xFFULL << 37) // Command Sets Supported
#define NVME_CAP_MPSMIN_MASK (0xFULL << 48) // Memory Page Size Minimum
#define NVME_CAP_MPSMAX_MASK (0xFULL << 52) // Memory Page Size Maximum
#define NVME_CC_EN (1 << 0) // Enable
#define NVME_CC_CSS_NVM (0 << 4) // NVM Command Set
#define NVME_CC_SHN_NORMAL (1 << 14) // Normal shutdown
#define NVME_CSTS_RDY (1 << 0) // Ready
#define NVME_CSTS_CFS (1 << 1) // Controller Fatal Status
#define NVME_MAX_IO_QUEUES (65535)
#define NVME_MAX_QUEUE_SIZE 64
#define NVME_RESET_TIMEOUT_MS 30000
#define NVME_ENABLE_TIMEOUT_MS 30000
struct nvme_lbaf {
std::uint16_t ms;
std::uint8_t ds;
std::uint8_t rp;
};
struct nvme_command {
uint32_t cdw0;
uint32_t nsid;
uint32_t reserved[2];
uint64_t metadata;
uint64_t prp1;
uint64_t prp2;
uint32_t cdw10;
uint32_t cdw11_15[5];
};
struct nvme_completion {
uint32_t result;
uint32_t reserved;
uint16_t sq_head;
uint16_t sq_id;
uint16_t command_id;
uint16_t status;
};
struct nvme_id_ns {
std::uint64_t nsze; // Namespace Size
std::uint64_t ncap; // Namespace Capacity
std::uint64_t nuse; // Namespace Utilization
std::uint8_t nsfeat; // Namespace Features
std::uint8_t nlbaf; // Number of LBA Formats
std::uint8_t flbas; // Formatted LBA Size
std::uint8_t mc; // Metadata Capabilities
std::uint8_t dpc; // End-to-end Data Protection Capabilities
std::uint8_t dps; // End-to-end Data Protection Type Settings
std::uint8_t nmic; // Namespace Multi-path I/O and Namespace Sharing Capabilities
std::uint8_t rescap; // Reservation Capabilities
std::uint8_t fpi; // Format Progress Indicator
std::uint8_t dlfeat; // Deallocate Logical Block Features
std::uint16_t nawun; // Namespace Atomic Write Unit Normal
std::uint16_t nawupf; // Namespace Atomic Write Unit Power Fail
std::uint16_t nacwu; // Namespace Atomic Compare & Write Unit
std::uint16_t nabsn; // Namespace Atomic Boundary Size Normal
std::uint16_t nabo; // Namespace Atomic Boundary Offset
std::uint16_t nabspf; // Namespace Atomic Boundary Size Power Fail
std::uint16_t noiob; // Namespace Optimal I/O Boundary
std::uint8_t nvmcap[16]; // NVM Capacity
std::uint16_t npwg; // Namespace Preferred Write Granularity
std::uint16_t npwa; // Namespace Preferred Write Alignment
std::uint16_t npdg; // Namespace Preferred Deallocate Granularity
std::uint16_t npda; // Namespace Preferred Deallocate Alignment
std::uint16_t nows; // Namespace Optimal Write Size
std::uint16_t mssrl; // Maximum Single Source Range Length
std::uint32_t mcl; // Maximum Copy Length
std::uint8_t msrc; // Maximum Source Range Count
std::uint8_t rsvd81[11];
std::uint32_t anagrpid; // ANA Group Identifier
std::uint8_t rsvd96[3];
std::uint8_t nsattr; // Namespace Attributes
std::uint16_t nvmsetid; // NVM Set Identifier
std::uint16_t endgid; // Endurance Group Identifier
std::uint8_t nguid[16]; // Namespace Globally Unique Identifier
std::uint8_t eui64[8]; // IEEE Extended Unique Identifier
nvme_lbaf lbaf[16]; // LBA Format Support
std::uint8_t rsvd192[192];
std::uint8_t vs[3712]; // Vendor Specific
};
static_assert(sizeof(nvme_id_ns) == 4096, "nvme_id_ns must be exactly 4096 bytes");
struct nvme_queue {
std::uint64_t address;
std::size_t size;
};
struct nvme_namespace {
uint32_t nsid;
uint64_t size;
uint32_t lba_size;
uint16_t lba_shift;
bool valid;
struct nvme_id_ns* ns_data;
};
struct nvme_pair_queue {
nvme_queue sq;
nvme_queue cq;
std::uint64_t sq_tail;
std::uint64_t cq_head;
std::uint8_t phase;
std::uint8_t qid;
volatile std::uint32_t* sq_doorbell;
volatile std::uint32_t* cq_doorbell;
locks::mutex lock;
};
struct nvme_controller {
void* bar0;
uint32_t stride;
uint32_t page_size;
uint16_t max_queue_entries;
nvme_queue admin_sq;
nvme_queue admin_cq;
std::uint64_t admin_sq_tail;
std::uint64_t admin_cq_head;
std::uint8_t admin_phase;
nvme_pair_queue* main_io_queue;
nvme_namespace namespaces[256];
uint32_t num_namespaces;
uint8_t mpsmin;
void* identify;
};
struct nvme_arg_disk {
nvme_controller* ctrl;
std::uint32_t nsid;
};
namespace drivers {
namespace nvme {
static inline std::uint32_t read32(struct nvme_controller* ctrl, std::uint32_t offset) {
volatile std::uint32_t* reg = (volatile std::uint32_t*)((std::uint8_t*)ctrl->bar0 + offset);
std::uint32_t value = *reg;
arch::memory_barrier();
return value;
}
static inline void write32(struct nvme_controller* ctrl, std::uint32_t offset, std::uint32_t value) {
volatile std::uint32_t* reg = (volatile std::uint32_t*)((std::uint8_t*)ctrl->bar0 + offset);
*reg = value;
arch::memory_barrier();
}
static inline std::uint64_t read64(struct nvme_controller* ctrl, uint32_t offset) {
volatile std::uint64_t* reg = (volatile std::uint64_t*)((std::uint8_t*)ctrl->bar0 + offset);
std::uint64_t value = *reg;
arch::memory_barrier();
return value;
}
static inline void write64(struct nvme_controller* ctrl, std::uint32_t offset, std::uint32_t value) {
volatile std::uint64_t* reg = (volatile std::uint64_t*)((std::uint8_t*)ctrl->bar0 + offset);
*reg = value;
arch::memory_barrier();
}
void init();
void disable();
};
};
|
