10#include <l4/cxx/ref_ptr>
12#include <l4/cxx/utils>
13#include <l4/sys/cache.h>
17#include <l4/l4virtio/server/virtio-block>
19#include <l4/libblock-device/debug.h>
20#include <l4/libblock-device/device.h>
21#include <l4/libblock-device/types.h>
22#include <l4/libblock-device/request.h>
24namespace Block_device {
26template <
typename DEV>
32 class Generic_pending_request :
public Pending_request
35 int check_error(
int result)
37 if (result < 0 && result != -
L4_EBUSY)
38 client->handle_request_error(result,
this);
44 explicit Generic_pending_request(Virtio_client *c, cxx::unique_ptr<Request> &&req)
45 : request(
cxx::move(req)), client(c)
48 void fail_request()
override
53 cxx::unique_ptr<Request> request;
54 Virtio_client *client;
57 struct Pending_inout_request :
public Generic_pending_request
62 explicit Pending_inout_request(Virtio_client *c,
63 cxx::unique_ptr<Request> &&req)
64 : Generic_pending_request(c,
cxx::move(req))
71 ~Pending_inout_request()
override
73 this->client->release_dma(
this);
76 int handle_request()
override
77 {
return this->check_error(this->client->inout_request(
this)); }
80 struct Pending_flush_request :
public Generic_pending_request
82 using Generic_pending_request::Generic_pending_request;
84 int handle_request()
override
85 {
return this->check_error(this->client->flush_request(
this)); }
88 struct Pending_cmd_request :
public Generic_pending_request
92 using Generic_pending_request::Generic_pending_request;
94 int handle_request()
override
96 return this->check_error(this->client->discard_cmd_request(
this, 0));
101 using Device_type = DEV;
113 dev->capacity() >> 9,
116 _client_invalidate_cb(nullptr),
117 _client_idle_cb(nullptr),
123 init_discard_info(0);
129 void reset_device()
override
131 if (_client_invalidate_cb)
132 _client_invalidate_cb(
false);
134 _negotiated_features.raw = 0;
147 _shutdown_state = Shutdown_type::Running;
148 _negotiated_features.raw = 0;
151 bool queue_stopped()
override
152 {
return _shutdown_state == Shutdown_type::Client_gone; }
159 void set_client_invalidate_cb(std::function<
void(
bool)> &&cb)
161 _client_invalidate_cb = cb;
164 void set_client_idle_cb(std::function<
void()> &&cb)
166 _client_idle_cb = cb;
172 _device_notify_irq = irq;
177 return _device_notify_irq;
185 cxx::unique_ptr<Pending_request> start_request(cxx::unique_ptr<Request> &&req)
187 auto trace = Dbg::trace(
"virtio");
189 cxx::unique_ptr<Pending_request> pending;
191 if (_shutdown_state != Shutdown_type::Running)
193 trace.printf(
"Failing requests as the client is shutting down\n");
198 trace.printf(
"request received: type 0x%x, sector 0x%llx\n",
199 req->header().type, req->header().sector);
200 switch (req->header().type)
205 auto p = cxx::make_unique<Pending_inout_request>(
this, cxx::move(req));
206 int ret = build_inout_blocks(p.get());
208 pending.reset(p.release());
210 handle_request_error(ret, p.get());
215 auto p = cxx::make_unique<Pending_flush_request>(
this, cxx::move(req));
216 int ret = check_flush_request(p.get());
218 pending.reset(p.release());
220 handle_request_error(ret, p.get());
226 auto p = cxx::make_unique<Pending_cmd_request>(
this, cxx::move(req));
227 int ret = build_discard_cmd_blocks(p.get());
229 pending.reset(p.release());
231 handle_request_error(ret, p.get());
242 void task_finished(Generic_pending_request *preq,
int error,
l4_size_t sz)
249 if (_shutdown_state != Client_gone)
257 cxx::unique_ptr<Pending_request> ureq(preq);
263 void shutdown_event(Shutdown_type type)
269 if (_shutdown_state == Client_gone)
274 l4_assert(_shutdown_state != System_shutdown);
277 l4_assert(_shutdown_state != System_suspend
278 || type == Shutdown_type::Running);
281 _shutdown_state = type;
283 if (type == Shutdown_type::Client_shutdown)
288 l4_assert(_shutdown_state == Shutdown_type::Running);
291 if (type != Shutdown_type::Running)
293 if (_client_invalidate_cb)
294 _client_invalidate_cb(type != Shutdown_type::Client_gone);
312 char const *service = 0)
342 return _in_flight != 0;
345 Notification_domain
const *notification_domain()
const
346 {
return _device->notification_domain(); }
355 void release_dma(Pending_inout_request *req)
358 Inout_block *cur = &req->blocks;
361 if (cur->num_sectors)
362 _device->dma_unmap(cur->dma_addr, cur->num_sectors, req->dir);
363 cur = cur->next.get();
367 int build_inout_blocks(Pending_inout_request *preq)
369 auto *req = preq->request.get();
370 l4_size_t sps = _device->sector_size() >> 9;
371 l4_uint64_t current_sector = req->header().sector / sps;
372 l4_uint64_t sectors = _device->capacity() / _device->sector_size();
373 auto dir = preq->dir;
379 if (device_features().ro())
383 if (_negotiated_features.config_wce())
386 flags = Block_device::Inout_f_wb;
388 else if (_negotiated_features.flush())
389 flags = Block_device::Inout_f_wb;
393 if (current_sector * sps != req->header().sector)
396 Inout_block *last_blk =
nullptr;
400 while (req->has_more())
402 Request::Data_block b;
404 if (++seg > _device->max_segments())
409 b = req->next_block();
413 Dbg::warn().printf(
"Descriptor error: %s\n", e.
message());
420 l4_size_t sz = b.len / _device->sector_size();
422 if (sz * _device->sector_size() != b.len)
424 Dbg::warn().printf(
"Bad block size 0x%x\n", b.len);
431 if (current_sector > sectors - sz)
437 last_blk->next = cxx::make_unique<Inout_block>();
438 blk = last_blk->next.get();
444 long ret = _device->dma_map(b.mem, off, sz, dir, &phys);
448 blk->dma_addr = phys;
449 blk->virt_addr = (
void *) ((
l4_addr_t)b.mem->local_base() + off);
450 blk->num_sectors = sz;
451 current_sector += sz;
460 void maintain_cache_before_req(Pending_inout_request
const *preq)
464 for (Inout_block
const *cur = &preq->blocks; cur; cur = cur->next.get())
469 l4_size_t vsize = cur->num_sectors * _device->sector_size();
480 void maintain_cache_after_req(Pending_inout_request
const *preq)
484 for (Inout_block
const *cur = &preq->blocks; cur; cur = cur->next.get())
489 l4_size_t vsize = cur->num_sectors * _device->sector_size();
496 int inout_request(Pending_inout_request *preq)
498 auto *req = preq->request.get();
499 l4_uint64_t sector = req->header().sector / (_device->sector_size() >> 9);
501 maintain_cache_before_req(preq);
502 int res = _device->inout_data(
503 sector, preq->blocks,
505 maintain_cache_after_req(preq);
506 task_finished(preq, error, sz);
517 int check_flush_request(Pending_flush_request *preq)
519 if (!_negotiated_features.flush())
522 auto *req = preq->request.get();
525 if (req->header().sector)
531 int flush_request(Pending_flush_request *preq)
533 int res = _device->flush([
this, preq](
int error,
l4_size_t sz) {
534 task_finished(preq, error, sz);
544 bool check_features(
void)
override
546 _negotiated_features = negotiated_features();
550 template <
typename T = Device_type>
551 void init_discard_info(
long) {}
553 template <
typename T = Device_type>
554 auto init_discard_info(
int)
555 ->
decltype(((T*)0)->discard_info(), void())
557 _di = _device->discard_info();
560 size_t sps = _device->sector_size() >> 9;
561 if (_di.max_discard_sectors)
562 set_discard(_di.max_discard_sectors * sps, _di.max_discard_seg,
563 _di.discard_sector_alignment * sps);
564 if (_di.max_write_zeroes_sectors)
566 _di.max_write_zeroes_seg, _di.write_zeroes_may_unmap);
569 int build_discard_cmd_blocks(Pending_cmd_request *preq)
571 auto *req = preq->request.get();
574 if (this->device_features().ro())
578 if (req->header().sector)
583 if (!_negotiated_features.discard())
588 if (!_negotiated_features.write_zeroes())
592 auto *d = _device.get();
595 size_t max_seg = discard ? _di.max_discard_seg : _di.max_write_zeroes_seg;
598 l4_uint64_t sectors = d->capacity() / d->sector_size();
600 Inout_block *last_blk =
nullptr;
602 while (req->has_more())
604 Request::Data_block b;
608 b = req->next_block();
612 Dbg::warn().printf(
"Descriptor error: %s\n", e.
message());
618 size_t items = b.len /
sizeof(payload[0]);
619 if (items *
sizeof(payload[0]) != b.len)
622 if (seg + items > max_seg)
626 for (
auto i = 0u; i < items; i++)
628 auto p = cxx::access_once<l4virtio_block_discard_t>(&payload[i]);
632 if (p.sector % sps != 0)
634 if (p.num_sectors % sps != 0)
639 p.num_sectors /= sps;
642 if (p.num_sectors > sectors)
644 if (p.sector > sectors - p.num_sectors)
647 if (p.flags & L4VIRTIO_BLOCK_DISCARD_F_RESERVED)
653 last_blk->next = cxx::make_unique<Inout_block>();
654 blk = last_blk->next.get();
659 blk->sector = p.sector;
660 blk->num_sectors = p.num_sectors;
664 if (p.flags & L4VIRTIO_BLOCK_DISCARD_F_UNMAP)
666 if (p.num_sectors > _di.max_discard_sectors)
671 if (p.flags & L4VIRTIO_BLOCK_DISCARD_F_UNMAP
672 && _di.write_zeroes_may_unmap)
673 blk->flags = Inout_f_unmap;
674 if (p.num_sectors > _di.max_write_zeroes_sectors)
685 template <
typename T = Device_type>
686 int discard_cmd_request(Pending_cmd_request *,
long)
689 template <
typename T = Device_type>
690 auto discard_cmd_request(Pending_cmd_request *preq,
int)
691 ->
decltype(((T*)0)->discard_info(), int())
693 auto *req = preq->request.get();
696 int res = _device->discard(
698 [
this, preq](
int error,
l4_size_t sz) { task_finished(preq, error, sz); },
709 void handle_request_error(
int error, Generic_pending_request *pending)
711 auto trace = Dbg::trace(
"virtio");
715 trace.printf(
"Unsupported operation.\n");
721 trace.printf(
"Got IO error: %d\n", error);
728 std::function<void(
bool)> _client_invalidate_cb;
729 std::function<void()> _client_idle_cb;
731 Shutdown_type _shutdown_state;
733 Device_discard_feature::Discard_info _di;
735 L4virtio::Svr::Block_features _negotiated_features;
l4_uint64_t Dma_addr
Data type for DMA addresses.
Direction
Direction of the DMA transfers.
@ To_device
device reads the memory
@ None
device is coherently connected to the memory
@ From_device
device writes to the memory
C++ interface for capabilities.
Interface for server-loop related functions.
Abstract interface for object registries.
virtual void unregister_obj(L4::Epiface *o, bool unmap=true)=0
Unregister the given object o from the server.
virtual L4::Cap< void > register_obj(L4::Epiface *o, char const *service)=0
Register an L4::Epiface for an IPC gate available in the applications environment under the name serv...
Base class for virtio block devices.
void set_write_zeroes(l4_uint32_t max_write_zeroes_sectors, l4_uint32_t max_write_zeroes_seg, l4_uint8_t write_zeroes_may_unmap)
Sets constraints for and enables the write zeroes command.
l4_uint8_t get_writeback()
Get the writeback field from the configuration space.
cxx::unique_ptr< Request > get_request()
Return one request if available.
void set_config_wce(l4_uint8_t writeback)
Sets cache mode and enables the writeback toggle.
Block_dev_base(l4_uint32_t vendor, unsigned queue_size, l4_uint64_t capacity, bool read_only)
Create a new virtio block device.
void set_flush()
Enables the flush command.
void set_size_max(l4_uint32_t sz)
Sets the maximum size of any single segment reported to client.
void set_discard(l4_uint32_t max_discard_sectors, l4_uint32_t max_discard_seg, l4_uint32_t discard_sector_alignment)
Sets constraints for and enables the discard command.
void set_seg_max(l4_uint32_t sz)
Sets the maximum number of segments in a request that is reported to client.
void finalize_request(cxx::unique_ptr< Request > req, unsigned sz, l4_uint8_t status=L4VIRTIO_BLOCK_S_OK)
Releases resources related to a request and notifies the client.
void init_mem_info(unsigned num)
Initialize the memory region list to the given maximum.
virtual L4::Cap< L4::Irq > device_notify_irq(unsigned idx)
Callback to gather the device notification IRQ (multi IRQ).
A reference-counting pointer with automatic cleanup.
unsigned int l4_size_t
Unsigned size type.
unsigned long l4_addr_t
Address type.
unsigned int l4_uint32_t
Unsigned 32bit value.
unsigned long long l4_uint64_t
Unsigned 64bit value.
int l4_cache_flush_data(unsigned long start, unsigned long end) L4_NOTHROW
Cache flush a range; writes back to PoC.
int l4_cache_clean_data(unsigned long start, unsigned long end) L4_NOTHROW
Cache clean a range in D-cache; writes back to PoC.
int l4_cache_inv_data(unsigned long start, unsigned long end) L4_NOTHROW
Cache invalidate a range; might write back to PoC.
@ L4_EBUSY
Object currently busy, try later.
@ L4VIRTIO_BLOCK_T_DISCARD
Discard a range of sectors.
@ L4VIRTIO_BLOCK_T_FLUSH
Flush data to disk.
@ L4VIRTIO_BLOCK_T_IN
Read from device.
@ L4VIRTIO_BLOCK_T_OUT
Write to device.
@ L4VIRTIO_BLOCK_T_WRITE_ZEROES
Write zeroes to a range of sectors.
@ L4VIRTIO_BLOCK_S_IOERR
IO error on device.
@ L4VIRTIO_BLOCK_S_UNSUPP
Operation is not supported.
T chkcap(T &&cap, char const *extra="", long err=-L4_ENOMEM)
Check for valid capability or raise C++ exception.
L4-VIRTIO Transport C++ API.
Epiface implementation for Kobject-based interface implementations.
Server_iface * server_iface() const
Get pointer to server interface at which the object is currently registered.
Exception used by Queue to indicate descriptor errors.
char const * message() const
Get a human readable description of the error code.
Structure used for the write zeroes and discard commands.
#define l4_assert(expr)
Low-level assert.
Common task related definitions.
Implementation of a list of unique-ptr-managed objects.