GitLab

Duncan Coutts pushed to branch wip/dcoutts/io-manager-uring at Glasgow Haskell Compiler / GHC

Commits:

102b80f6
by Duncan Coutts at 2026-08-26T11:16:05+01:00
```
Hacking on io_uring I/O manager
```

16 changed files:

compiler/GHC/Builtin/primops.txt.pp
compiler/GHC/StgToCmm/Prim.hs
hadrian/src/Oracles/Flag.hs
rts/IOManager.c
rts/IOManager.h
rts/IOManagerInternals.h
rts/PrimOps.cmm
rts/configure.ac
rts/include/rts/Constants.h
rts/include/rts/Flags.h
rts/include/rts/storage/Closures.h
rts/posix/Poll.c
+ rts/posix/URing.c
+ rts/posix/URing.h
+ rts/posix/URing.svg
rts/rts.cabal

Changes:

compiler/GHC/Builtin/primops.txt.pp

@@ -3185,6 +3185,128 @@ primop  WaitWriteOp "waitWrite#" GenPrimOp
     effect = ReadWriteEffect
     out_of_line      = True
 +------------------------------------------------------------------------
 +section "synchronous I/O operations"
 +  {These primops read up to n bytes from an open file into memory, or write up
 +   to n bytes from memory to an open file.
++
 +   There are I\/O primops for all combinations of:
++
 +   * read or write to open files
 +   * pinned byte array or raw pointer for the memory buffer
 +   * I/O at the current file pointer or at a given file offset
++
 +   All of these operations are synchronous with respect to the calling
 +   thread. It is implementation defined whether these operations block just
 +   the calling thread or block all other Haskell threads running on the same
 +   capability. In practice this depends on the I\/O manager being used.
++
 +   The result is the number of bytes transferred (if non-negative), or an
 +   error code if negative. The error code is a system error code which,
 +   depending on the platform, is either a C/Posix style errno, or a Win32 error
 +   code.
++
 +   Note that partial\/short reads\/writes are possible. These are common with
 +   sockets and character devices, and rare with disk block devices, but they
 +   can happen and Posix and Win32 APIs say they can happen, so they must be
 +   handled.
++
 +   The source\/destination buffer is either specified by a pointer, or by a
 +   (mutable, pinned) byte array and an offset within the array. This area is
 +   required to be at least n bytes large. This cannot be checked for the
 +   pointer variants but it is checked for the byte array variants. For the
 +   byte array variants, the array must be pinned (so that the location of the
 +   buffer is stable for the duration of the I/O operation) and this is
 +   checked.
++
 +   The buffer must remain live for the duration of the I/O operation (until
 +   the I/O completes or the I/O is cancelled by an asynchronous exception).
 +   For the primop variants that use byte array buffers this is done
 +   automatically: the byte array is kept live for the duration of the
 +   operation. For the pointer primop variants it is the caller's
 +   responsibility to keep the buffer live for the duration of the I/O
 +   operation. For these synchronous primops, callers can rely on the guarantee
 +   that the I/O operation is complete or cancelled by the time the I/O primop
 +   returns.
++
 +   For the primop variants that do I/O at the current file pointer, the
 +   current file pointer constitutes shared mutable state and should be treated
 +   appropriately. Concurrent reads or writes are possible, and will happen in
 +   some sequentially consistent order, but the order is not deterministic.
++
 +   For the primop variants that do I/O at given offsets, note that this is
 +   only supported on seekable files, which in practice means disk files. There
 +   is no mutation to a shared file pointer in this case so it is safer to
 +   use concurrent reads and writes. Callers are however responsible for
 +   avoiding concurrent write operations to overlapping ranges (either
 +   write\/write or read\/write), as these would have unspecified results.
 +  }
 +------------------------------------------------------------------------
++
 +primop  SyncIOReadAddrOp "syncIOReadAddr#" GenPrimOp
 +  Int# -> Addr# -> Word# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, byte count}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOReadByteArrayOp "syncIOReadByteArray#" GenPrimOp
 +  Int# -> MutableByteArray# RealWorld -> Word# -> Word# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, buf offset, byte count}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOReadAddrAtOp "syncIOReadAddrAt#" GenPrimOp
 +  Int# -> Addr# -> Word# -> Int64# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, byte count, file offset}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOReadByteArrayAtOp "syncIOReadByteArrayAt#" GenPrimOp
 +  Int# -> MutableByteArray# RealWorld -> Word# -> Word# -> Int64# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, buf offset, byte count, file offset}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOWriteAddrOp "syncIOWriteAddr#" GenPrimOp
 +  Int# -> Addr# -> Word# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, byte count}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOWriteByteArrayOp "syncIOWriteByteArray#" GenPrimOp
 +  Int# -> ByteArray# -> Word# -> Word# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, buf offset, byte count}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOWriteAddrAtOp "syncIOWriteAddrAt#" GenPrimOp
 +  Int# -> Addr# -> Word# -> Int64# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, byte count, file offset}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
 +primop  SyncIOWriteByteArrayAtOp "syncIOWriteByteArrayAt#" GenPrimOp
 +  Int# -> ByteArray# -> Word# -> Word# -> Int64# ->
 +  State# RealWorld -> (# State# RealWorld, Int# #)
 +  {Args: fd, buf, buf offset, byte count, file offset}
 +  with
 +  effect = ReadWriteEffect
 +  out_of_line = True
++
  ------------------------------------------------------------------------
  section "Concurrency primitives"
  ------------------------------------------------------------------------

compiler/GHC/StgToCmm/Prim.hs

@@ -1714,6 +1714,14 @@ emitPrimOp cfg primop =
    DelayOp -> alwaysExternal
    WaitReadOp -> alwaysExternal
    WaitWriteOp -> alwaysExternal
 +  SyncIOReadAddrOp -> alwaysExternal
 +  SyncIOReadByteArrayOp -> alwaysExternal
 +  SyncIOReadAddrAtOp -> alwaysExternal
 +  SyncIOReadByteArrayAtOp -> alwaysExternal
 +  SyncIOWriteAddrOp -> alwaysExternal
 +  SyncIOWriteByteArrayOp -> alwaysExternal
 +  SyncIOWriteAddrAtOp -> alwaysExternal
 +  SyncIOWriteByteArrayAtOp -> alwaysExternal
    ForkOp -> alwaysExternal
    ForkOnOp -> alwaysExternal
    KillThreadOp -> alwaysExternal

hadrian/src/Oracles/Flag.hs

@@ -37,6 +37,7 @@ data Flag = CrossCompiling
            | UseLibdl
            | UseLibbfd
            | UseLibpthread
 +          | UseLiburing
            | NeedLibatomic
            | UseGhcToolchain
@@ -61,6 +62,7 @@ flag f = do
              UseLibdl             -> "use-lib-dl"
              UseLibbfd            -> "use-lib-bfd"
              UseLibpthread        -> "use-lib-pthread"
 +            UseLiburing          -> "use-lib-uring"
              NeedLibatomic        -> "need-libatomic"
              UseGhcToolchain      -> "use-ghc-toolchain"
      value <- lookupSystemConfig key

rts/IOManager.c

@@ -38,6 +38,11 @@
  #include "posix/Timeout.h"
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +#include "posix/URing.h"
 +#include "posix/Timeout.h"
 +#endif
++
  #if defined(IOMGR_ENABLED_MIO_POSIX)
  #include "posix/Signals.h"
  #include "Prelude.h"
@@ -114,6 +119,14 @@ parseIOManagerFlag(const char *iomgrstr, IO_MANAGER_FLAG *flag)
          return IOManagerAvailable;
  #else
          return IOManagerUnavailable;
 +#endif
 +    }
 +    else if (strcmp("uring", iomgrstr) == 0) {
 +#if defined(IOMGR_ENABLED_URING)
 +        *flag = IO_MNGR_FLAG_URING;
 +        return IOManagerAvailable;
 +#else
 +        return IOManagerUnavailable;
  #endif
+     }
      else if (strcmp("mio", iomgrstr) == 0) {
@@ -218,6 +231,8 @@ void selectIOManager(void)
              iomgr_type = IO_MANAGER_SELECT;
  #elif defined(IOMGR_DEFAULT_NON_THREADED_POLL)
              iomgr_type = IO_MANAGER_POLL;
 +#elif defined(IOMGR_DEFAULT_NON_THREADED_URING)
 +            iomgr_type = IO_MANAGER_URING;
  #elif defined(IOMGR_DEFAULT_NON_THREADED_WINIO)
              iomgr_type = IO_MANAGER_WINIO;
  #elif defined(IOMGR_DEFAULT_NON_THREADED_WIN32_LEGACY)
@@ -240,6 +255,12 @@ void selectIOManager(void)
              break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MNGR_FLAG_URING:
 +            iomgr_type = IO_MANAGER_URING;
 +            break;
 +#endif
++
  #if defined(IOMGR_ENABLED_MIO_POSIX)
          case IO_MNGR_FLAG_MIO:
              iomgr_type = IO_MANAGER_MIO_POSIX;
@@ -282,6 +303,10 @@ char * showIOManager(void)
          case IO_MANAGER_POLL:
              return "poll";
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            return "uring";
 +#endif
  #if defined(IOMGR_ENABLED_MIO_POSIX)
          case IO_MANAGER_MIO_POSIX:
              return "mio";
@@ -335,6 +360,12 @@ void initCapabilityIOManager(Capability *cap)
              break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            initCapabilityIOManagerURing(cap, iomgr);
 +            break;
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
          case IO_MANAGER_WIN32_LEGACY:
              iomgr->blocked_queue_hd = END_TSO_QUEUE;
@@ -378,6 +409,11 @@ void initIOManager(void)
              break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            break;
 +#endif
++
  #if defined(IOMGR_ENABLED_MIO_POSIX)
          case IO_MANAGER_MIO_POSIX:
              /* Posix implementation in posix/Signals.c
@@ -441,6 +477,18 @@ initIOManagerAfterFork(Capability **pcap)
               */
              ioManagerStartCap(pcap);
              break;
 +#endif
 +#if defined(IOMGR_ENABLED_URING)
 +        //TODO: So currently there's no per-cap re-initialisation
 +        // except for cap0.
 +        case IO_MANAGER_URING:
 +        barf("IOManager.c:initIOManagerAfterFork:URing:TODO");
 +        {
 +            for (unsigned int i = 0; i < getNumCapabilities(); i++) {
 +                Capability *cap = getCapability(i);
 +                initCapabilityIOManagerAfterForkURing(cap, cap->iomgr);
 +            }
 +        }
  #endif
          /* The IO_MANAGER_SELECT needs no initialisation */
          /* The IO_MANAGER_POLL needs no initialisation */
@@ -452,7 +500,13 @@ initIOManagerAfterFork(Capability **pcap)
+ }
 -/* Called from setNumCapabilities.
 +/* Called from setNumCapabilities, after all other per-capability changes have
 + * been made. When the scheduler increases the number of capabilities, it
 + * (indirectly) calls initCapabilityIOManager, for each new capability. So this
 + * notification is only needed by I/O managers that need a global hook (not
 + * per-cap), and/or need to be notified of there being fewer (as well as more)
 + * capabilities. There is no per-capability notification for disabling a
 + * capability (which occurs when the number of capabilities is reduced).
   */
  void notifyIOManagerCapabilitiesChanged(Capability **pcap)
+ {
@@ -583,6 +637,18 @@ void markCapabilityIOManager(evac_fn evac, void *user, Capability *cap)
+         }
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +        {
 +            CapIOManager *iomgr = cap->iomgr;
 +            markClosureTable(evac, user, &iomgr->aiop_table);
 +            evac(user, (StgClosure **)(void *)&iomgr->overflow_tso_q_hd);
 +            evac(user, (StgClosure **)(void *)&iomgr->overflow_tso_q_tl);
 +            evac(user, (StgClosure **)(void *)&iomgr->timeout_queue);
 +            break;
 +        }
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
          case IO_MANAGER_WIN32_LEGACY:
+         {
@@ -608,8 +674,15 @@ void scavengeTSOIOManager(StgTSO *tso)
               * both of these are not GC pointers, so there is nothing to do.
               */
 +#if defined(IOMGR_ENABLED_POLL) \
 + || defined(IOMGR_ENABLED_URING)
++
  #if defined(IOMGR_ENABLED_POLL)
          case IO_MANAGER_POLL:
 +#endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +#endif
              /* BlockedOn{Read,Write} uses block_info.aiop
               * BlockedOnDelay        uses block_info.timeout
               * both of these are heap allocated, so we can do the same in all
@@ -669,6 +742,12 @@ bool anyPendingTimeoutsOrIO(Capability *cap)
              return anyPendingTimeoutsOrIOPoll(cap->iomgr);
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            return anyPendingTimeoutsOrIOURing(cap->iomgr);
 +        //TODO: do we want to share code with poll here?
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
          case IO_MANAGER_WIN32_LEGACY:
+         {
@@ -732,6 +811,12 @@ void pollCompletedTimeoutsOrIO(Capability *cap)
            break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +          pollCompletedTimeoutsOrIOURing(cap);
 +          break;
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY) || \
     (defined(IOMGR_ENABLED_WINIO) && !defined(THREADED_RTS))
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
@@ -765,6 +850,12 @@ void awaitCompletedTimeoutsOrIO(Capability *cap)
            break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +          awaitCompletedTimeoutsOrIOURing(cap);
 +          break;
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY) || \
     (defined(IOMGR_ENABLED_WINIO) && !defined(THREADED_RTS))
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
@@ -805,8 +896,11 @@ int syncIOWaitReady(Capability   *cap,
  #endif
  #if defined(IOMGR_ENABLED_POLL)
          case IO_MANAGER_POLL:
 -            ASSERT(tso->why_blocked == NotBlocked);
              return syncIOWaitReadyPoll(cap, tso, rw, fd);
 +#endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            return syncIOWaitReadyURing(cap, tso, rw, fd);
  #endif
          default:
              barf("waitRead# / waitWrite# not available for current I/O manager");
@@ -814,6 +908,91 @@ int syncIOWaitReady(Capability   *cap,
+ }
 +int syncIOReadWrite(Capability *cap, StgTSO *tso,
 +                    IOReadOrWrite rw, HsInt fd,
 +                    StgClosure *live, void *buf,
 +                    HsWord len)
 +{
 +    debugTrace(DEBUG_iomanager,
 +               "thread %ld %s fd %d", (long) tso->id,
 +               rw == IORead ? "reading from" : "writing to", (int) fd);
 +    ASSERT(tso->why_blocked == NotBlocked);
 +    switch (iomgr_type) {
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            return syncIOReadWriteURing(cap, tso, rw, (int)fd, live, buf,
 +                                        (size_t)len, (off_t)(-1));
 +            /* off_t = -1 means use and update the file pointer. */
 +#endif
 +/*
 +#if defined(IOMGR_ENABLED_SELECT) \
 + || defined(IOMGR_ENABLED_POLL)
 +#if defined(IOMGR_ENABLED_SELECT)
 +        case IO_MANAGER_SELECT:
 +#endif
 +#if defined(IOMGR_ENABLED_POLL)
 +        case IO_MANAGER_POLL:
 +#endif
 +        {
 +            if (rw == IORead) {
 +                read((int) fd, buf, (size_t) len);
 +            } else {
 +                write((int) fd, buf, (size_t) len);
 +            }
 +            //TODO: Ugg! We need to tell the primop to return synchonrously!
 +            //Need to change the return type. Should use an out arg for the
 +            //GC alloc retry.
 +        }
 +#endif
 +*/
 +        default:
 +            barf("syncIORead/Write# not available for current I/O manager");
 +    }
 +}
++
 +int syncIOReadWriteAt(Capability *cap, StgTSO *tso,
 +                      IOReadOrWrite rw, HsInt fd,
 +                      StgClosure *live, void *buf,
 +                      HsWord len, HsInt64 off)
 +{
 +    debugTrace(DEBUG_iomanager,
 +               "thread %ld %s fd %d", (long) tso->id,
 +               rw == IORead ? "reading from" : "writing to", (int) fd);
 +    ASSERT(tso->why_blocked == NotBlocked);
 +    switch (iomgr_type) {
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            return syncIOReadWriteURing(cap, tso, rw,
 +                                        (int)fd, live, buf,
 +                                        (size_t)len, (off_t)off);
 +#endif
 +/*
 +#if defined(IOMGR_ENABLED_SELECT) \
 + || defined(IOMGR_ENABLED_POLL)
 +#if defined(IOMGR_ENABLED_SELECT)
 +        case IO_MANAGER_SELECT:
 +#endif
 +#if defined(IOMGR_ENABLED_POLL)
 +        case IO_MANAGER_POLL:
 +#endif
 +        {
 +            if (rw == IORead) {
 +                pread((int)fd, buf, (size_t)len, (off_t)off);
 +            } else {
 +                pwrite((int)fd, buf, (size_t)len, (off_t)off);
 +            }
 +            //TODO: Ugg! We need to tell the primop to return synchonrously!
 +            //Need to change the return type. Should use an out arg for the
 +            //GC alloc retry.
 +        }
 +#endif
 +*/
 +        default:
 +            barf("syncIORead/Write# not available for current I/O manager");
 +    }
 +}
++
++
  void syncIOCancel(Capability *cap, StgTSO *tso)
+ {
      debugTrace(DEBUG_iomanager, "cancelling I/O for thread %ld", (long) tso->id);
@@ -829,6 +1008,11 @@ void syncIOCancel(Capability *cap, StgTSO *tso)
              syncIOCancelPoll(cap, tso);
              break;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +            syncIOCancelURing(cap, tso);
 +            break;
 +#endif
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
          case IO_MANAGER_WIN32_LEGACY:
              removeThreadFromDeQueue(cap, &cap->iomgr->blocked_queue_hd,
@@ -862,8 +1046,15 @@ int syncDelay(Capability *cap, StgTSO *tso, HsInt us_delay)
              return 0;
+         }
  #endif
 +#if defined(IOMGR_ENABLED_POLL) \
 + || defined(IOMGR_ENABLED_URING)
++
  #if defined(IOMGR_ENABLED_POLL)
          case IO_MANAGER_POLL:
 +#endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +#endif
              return syncDelayTimeout(cap, tso, us_delay);
  #endif
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
@@ -903,8 +1094,15 @@ void syncDelayCancel(Capability *cap, StgTSO *tso)
              removeThreadFromQueue(cap, &cap->iomgr->sleeping_queue, tso);
              break;
  #endif
 +#if defined(IOMGR_ENABLED_POLL) \
 + || defined(IOMGR_ENABLED_URING)
++
  #if defined(IOMGR_ENABLED_POLL)
          case IO_MANAGER_POLL:
 +#endif
 +#if defined(IOMGR_ENABLED_URING)
 +        case IO_MANAGER_URING:
 +#endif
              syncDelayCancelTimeout(cap, tso);
              break;
  #endif

rts/IOManager.h

@@ -46,6 +46,9 @@
  #if defined(IOMGR_BUILD_POLL) && !defined(THREADED_RTS)
      #define IOMGR_ENABLED_POLL
  #endif
 +#if defined(IOMGR_BUILD_URING) && !defined(THREADED_RTS)
 +    #define IOMGR_ENABLED_URING
 +#endif
  #if defined(IOMGR_BUILD_MIO) && defined(THREADED_RTS)
  /* For MIO, it is really two separate I/O manager implementations: one for
   * Windows and one for non-Windows. This is clear from both the C code on the
@@ -110,6 +113,11 @@
  #else
      #define IOMGR_ENABLED_STR_POLL ""
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +    #define IOMGR_ENABLED_STR_URING " uring"
 +#else
 +    #define IOMGR_ENABLED_STR_URING ""
 +#endif
  #if defined(IOMGR_ENABLED_MIO_POSIX) || defined(IOMGR_ENABLED_MIO_WIN32)
      #define IOMGR_ENABLED_STR_MIO " mio"
  #else
@@ -128,6 +136,7 @@
  #define IOMGRS_ENABLED_STR \
            IOMGR_ENABLED_STR_SELECT \
            IOMGR_ENABLED_STR_POLL \
 +          IOMGR_ENABLED_STR_URING \
            IOMGR_ENABLED_STR_MIO \
            IOMGR_ENABLED_STR_WINIO \
            IOMGR_ENABLED_STR_WIN32_LEGACY
@@ -143,6 +152,9 @@ typedef enum {
  #if defined(IOMGR_ENABLED_POLL)
      IO_MANAGER_POLL,
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +    IO_MANAGER_URING,
 +#endif
  #if defined(IOMGR_ENABLED_MIO_POSIX)
      IO_MANAGER_MIO_POSIX,
  #endif
@@ -298,7 +310,7 @@ void scavengeTSOIOManager(StgTSO *tso);
  /* Several code paths are almost identical between read and write paths. In
   * such cases we use a shared code path with an enum to say which we're doing.
   */
 -typedef enum { IORead, IOWrite } IOReadOrWrite;
 +typedef enum { IORead = 0, IOWrite = 1 } IOReadOrWrite;
  /* Synchronous operations: I/O and delays. As synchronous operations they
   * necessarily operate on threads. The thread is suspended until the operation
@@ -310,7 +322,18 @@ typedef enum { IORead, IOWrite } IOReadOrWrite;
   * GC to free up at least n words and then retry the operation.
   */
 -int syncIOWaitReady(Capability *cap, StgTSO *tso, IOReadOrWrite rw, HsInt fd);
 +int syncIOWaitReady(Capability *cap, StgTSO *tso,
 +                    IOReadOrWrite rw, HsInt fd);
++
 +int syncIOReadWrite(Capability *cap, StgTSO *tso,
 +                    IOReadOrWrite rw, HsInt fd,
 +                    StgClosure *live, void *buf,
 +                    HsWord len);
++
 +int syncIOReadWriteAt(Capability *cap, StgTSO *tso,
 +                      IOReadOrWrite rw, HsInt fd,
 +                      StgClosure *live, void *buf,
 +                      HsWord len, HsInt64 off);
  void syncIOCancel(Capability *cap, StgTSO *tso);

rts/IOManagerInternals.h

@@ -42,7 +42,8 @@ struct _CapIOManager {
      StgTSO *sleeping_queue;
  #endif
 -#if defined(IOMGR_ENABLED_POLL)
 +#if defined(IOMGR_ENABLED_POLL) \
 + || defined(IOMGR_ENABLED_URING)
      /* AIOP and timeout collections shared by several I/O manager impls */
      ClosureTable     aiop_table;
      StgTimeoutQueue *timeout_queue;
@@ -53,6 +54,54 @@ struct _CapIOManager {
      struct pollfd *aiop_poll_table;
  #endif
 +#if defined(IOMGR_ENABLED_URING)
 +    /* io_uring library structure */
 +    struct io_uring *uring;
++
 +    /* The number of operations submitted (by Haskell threads to the I/O
 +       manager) and not yet notified of completion. */
 +    int n_submitted_b;  /* for blocking operations */
 +    int n_submitted_nb; /* for non-blocking operations */
++
 +    /* The number of operations pending in the submission queue, but not yet
 +       submitted to the kernel (so not in-flight). */
 +    int n_prepared_b;  /* for blocking operations */
 +    int n_prepared_nb; /* for non-blocking operations */
++
 +    /* The number of operations submitted to the kernel but where the
 +       corresponding completion has not yet been processed. */
 +    int n_inflight_b;  /* for blocking operations */
 +    int n_inflight_nb; /* for non-blocking operations */
++
 +    /* The limit on the number of operations we allow to be in-flight */
 +    int limit_inflight_b;  /* for blocking operations */
 +    int limit_inflight_nb; /* for non-blocking operations */
++
 +    /* The number of operations pending in the overflow queue (so not in the
 +       submission queue or in flight) */
 +    /* no overflow for blocking operations */
 +    int n_overflow_nb;  /* for non-blocking operations */
++
 +    /* Invariants:
 +         n_submitted_b  = n_prepared_b  + n_inflight_b
 +         n_submitted_nb = n_prepared_nb + n_inflight_nb + n_overflow_nb
 +         n_prepared_b + n_prepared_nb <= size of submission queue
 +     */
++
 +    /* A queue of threads blocked on I/O submission and a parallel queue of
 +     * their corresponding SQEs. This is only used when there are more pending
 +     * (non-blocking) I/O operations than the inflight limit.
 +     */
 +    StgTSO *overflow_tso_q_hd, *overflow_tso_q_tl;
 +    struct overflow_sqe_q_t {
 +        struct io_uring_sqe     *sqe;
 +        struct overflow_sqe_q_t *next;
 +#if defined(DEBUG)
 +        StgThreadID              tid;
 +#endif
 +    } *overflow_sqe_q_hd, *overflow_sqe_q_tl;
 +#endif
++
  #if defined(IOMGR_ENABLED_WIN32_LEGACY)
      /* Thread queue for threads blocked on I/O completion. */
      StgTSO *blocked_queue_hd;

rts/PrimOps.cmm

@@ -2602,6 +2602,108 @@ stg_delayzh ( W_ us_delay )
+     }
+ }
 +/*
 +int syncIOReadWrite(Capability *cap, StgTSO *tso,
 +                    IOReadOrWrite rw, HsInt fd,
 +                    StgClosure *live, void *buf,
 +                    HsWord len);
++
 +int syncIOReadWriteAt(Capability *cap, StgTSO *tso,
 +                      IOReadOrWrite rw, HsInt fd,
 +                      StgClosure *live, void *buf,
 +                      HsWord len, HsInt64 off);
 +*/
++
 +#define SYNCIO_BLOCK_OR_GC(fail)                                        \
 +    if (fail == 0) (likely: True) {                                     \
 +        jump stg_block_noregs();                                        \
 +    } else {                                                            \
 +        /* TODO: should invoke GC, requesting 'fail' words, then retry  \
 +           see: https://gitlab.haskell.org/ghc/ghc/-/issues/24105 */    \
 +        jump stg_raisezh(ghczminternal_GHCziInternalziIOziException_heapOverflow_closure); \
 +    }
 +    //TODO: need to handle returning synchonrously, for non-uring I/O managers.
 +    //or if uring optimises the read case.
++
 +stg_syncIOReadAddrzh ( W_ fd, W_ buf, W_ len )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   0::I32 /* IORead */, fd,
 +                                   stg_ASYNCIO_LIVE0_closure, buf,
 +                                   len);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOReadByteArrayzh ( W_ fd, P_ buf, W_ boff, W_ len )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   0::I32 /* IORead */, fd,
 +                                   buf, buf + SIZEOF_StgArrBytes + boff,
 +                                   len);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOReadAddrAtzh ( W_ fd, W_ buf, W_ len, I64 foff )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWriteAt(MyCapability() "ptr", CurrentTSO "ptr",
 +                                     0::I32 /* IORead */, fd,
 +                                     stg_ASYNCIO_LIVE0_closure, buf,
 +                                     len, foff);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOReadByteArrayAtzh ( W_ fd, P_ buf, W_ boff, W_ len, I64 foff )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   0::I32 /* IORead */, fd,
 +                                   buf, buf + SIZEOF_StgArrBytes + boff,
 +                                   len, foff);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOWriteAddrzh ( W_ fd, W_ buf, W_ len )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   1::I32 /* IOWrite */, fd,
 +                                   stg_ASYNCIO_LIVE0_closure, buf,
 +                                   len);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOWriteByteArrayzh ( W_ fd, P_ buf, W_ boff, W_ len )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   1::I32 /* IOWrite */, fd,
 +                                   buf, buf + SIZEOF_StgArrBytes + boff,
 +                                   len);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOWriteAddrAtzh ( W_ fd, W_ buf, W_ len, I64 foff )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWriteAt(MyCapability() "ptr", CurrentTSO "ptr",
 +                                     1::I32 /* IOWrite */, fd,
 +                                     stg_ASYNCIO_LIVE0_closure, buf,
 +                                     len, foff);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
++
 +stg_syncIOWriteByteArrayAtzh ( W_ fd, P_ buf, W_ boff, W_ len, I64 foff )
 +{
 +    W_ fail; /* Request this many words on heap alloc failure. */
 +    (fail) = ccall syncIOReadWrite(MyCapability() "ptr", CurrentTSO "ptr",
 +                                   1::I32 /* IOWrite */, fd,
 +                                   buf, buf + SIZEOF_StgArrBytes + boff,
 +                                   len, foff);
 +    SYNCIO_BLOCK_OR_GC(fail);
 +}
  #if defined(mingw32_HOST_OS)
  stg_asyncReadzh ( W_ fd, W_ is_sock, W_ len, W_ buf )

rts/configure.ac

@@ -397,6 +397,23 @@ GHC_IOMANAGER_ENABLE([poll], [EnableIOManagerPoll], [IOMGR_BUILD_POLL],
              #include <poll.h>])
     fi])
 +GHC_IOMANAGER_ENABLE([uring], [EnableIOManagerURing], [IOMGR_BUILD_URING],
 +  [if test "$HostOS" = "linux"; then
 +       AC_CHECK_HEADER([liburing.h], [HaveLiburingH=YES],
 +           [AC_MSG_WARN([liburing.h is required by the uring I/O manager])],[])
 +       AC_CHECK_LIB([uring], [io_uring_check_version], [HaveLiburing=YES],
 +           [AC_MSG_WARN([liburing is required by the uring I/O manager])], [])
 +       if test "$HaveLiburingH" = "YES" && test "$HaveLiburing" = YES; then
 +           EnableIOManagerURing=YES
 +           LinkLiburing=uring
 +       else
 +           EnableIOManagerURing=NO
 +       fi
 +   else
 +       EnableIOManagerURing=NO
 +   fi])
 +AC_SUBST(LinkLiburing)
++
  GHC_IOMANAGER_ENABLE([mio], [EnableIOManagerMIO], [IOMGR_BUILD_MIO],
    [EnableIOManagerMIO=YES])
@@ -415,6 +432,7 @@ if test "$HostOS" = "mingw32"; then
    GHC_IOMANAGER_DEFAULT_SELECT([IOManagerThreadedDefault], [winio], [EnableIOManagerWinIO])
    GHC_IOMANAGER_DEFAULT_SELECT([IOManagerThreadedDefault], [mio], [EnableIOManagerMIO])
  else
 +  GHC_IOMANAGER_DEFAULT_SELECT([IOManagerNonThreadedDefault], [uring], [EnableIOManagerURing])
    GHC_IOMANAGER_DEFAULT_SELECT([IOManagerNonThreadedDefault], [select], [EnableIOManagerSelect])
    GHC_IOMANAGER_DEFAULT_SELECT([IOManagerNonThreadedDefault], [poll], [EnableIOManagerPoll])
    GHC_IOMANAGER_DEFAULT_SELECT([IOManagerThreadedDefault], [mio], [EnableIOManagerMIO])
@@ -432,6 +450,9 @@ GHC_IOMANAGER_DEFAULT_AC_DEFINE([IOManagerNonThreadedDefault], [non-threaded],
  GHC_IOMANAGER_DEFAULT_AC_DEFINE([IOManagerNonThreadedDefault], [non-threaded],
                                  [poll], [IOMGR_DEFAULT_NON_THREADED_POLL])
 +GHC_IOMANAGER_DEFAULT_AC_DEFINE([IOManagerNonThreadedDefault], [non-threaded],
 +                                [uring], [IOMGR_DEFAULT_NON_THREADED_URING])
++
  GHC_IOMANAGER_DEFAULT_AC_DEFINE([IOManagerNonThreadedDefault], [non-threaded],
                                  [winio], [IOMGR_DEFAULT_NON_THREADED_WINIO])

rts/include/rts/Constants.h

@@ -271,7 +271,9 @@
     by tryWakeupThread() */
  #define ThreadMigrating     13
 -/* Next number is 15.  */
 +#define BlockedOnIOSubmission 15
++
 +/* Next number is 16.  */
  /*
   * These constants are returned to the scheduler by a thread that has

rts/include/rts/Flags.h

@@ -247,6 +247,7 @@ typedef enum _IO_MANAGER_FLAG {
      /* All other choices pick only the requested one, with no fallback. */
      IO_MNGR_FLAG_SELECT,          /* Unix only,    non-threaded RTS only */
      IO_MNGR_FLAG_POLL,            /* Unix only,    non-threaded RTS only */
 +    IO_MNGR_FLAG_URING,           /* Linux only,   non-threaded RTS only */
      IO_MNGR_FLAG_MIO,             /* cross-platform,   threaded RTS only */
      IO_MNGR_FLAG_WINIO,           /* Windows only                        */
      IO_MNGR_FLAG_WIN32_LEGACY,    /* Windows only, non-threaded RTS only */
@@ -272,6 +273,8 @@ typedef struct _MISC_FLAGS {
                                    * for the linker, NULL ==> off */
      IO_MANAGER_FLAG ioManager;   /* The I/O manager to use.  */
      uint32_t numIoWorkerThreads; /* Number of I/O worker threads to use.  */
 +    uint32_t io_uring_sq_entries; /* io_uring submission queue size */
 +    uint32_t io_uring_cq_entries; /* io_uring completion queue size */
  } MISC_FLAGS;
  /* See Note [Synchronization of flags and base APIs] */

rts/include/rts/storage/Closures.h

@@ -713,7 +713,13 @@ union NotifyCompletion {
  enum NotifyCompletionType {
      NotifyTSO  = 0,
      NotifyMVar = 1,
 -    NotifyTVar = 2
 +    NotifyTVar = 2,
 +    NotifyNone = 3
 +    /* If a TSO receives an async exception while it's waiting on I/O then
 +     * the TSO stops waiting but the I/O may still be outstanding, and we
 +     * need the corresponding StgAsyncIOOp until the I/O completes. We use
 +     * NotifyNone in this case to avoid disturbing the original TSO.
 +     */
  };
  /* A node in the leftist heap. */
@@ -786,9 +792,9 @@ typedef struct {
        // to know which capability an aiop is on.
      uint16_t capno;
 -      // Tells us which thing the notify union above contains.
 -      // This is a value from enum IONotify but we don't use the enum type
 -      // here due to portability concerns for this size C enum bitfield.
 +      // Tells us which thing the notify union above contains. This is a value
 +      // from enum NotifyCompletionType but we don't use the enum type here
 +      // due to portability concerns for this size C enum bitfield.
      uint16_t notify_type: 2;
        // The outcome:
@@ -816,3 +822,15 @@ typedef struct {
        // We handle this in the INFO_TABLE_CONSTR decl for stg_ASYNCIOOP using
        // Either32Or64Bit(4,2) for the non-pointer words.
  } StgAsyncIOOp;
++
 +struct io_uring_sqe;
 +typedef struct {
 +    StgHeader header;
++
 +      // Any heap object to keep alive for the duration of the I/O operation,
 +      // for example I/O buffers.
 +    StgClosure *live;
++
 +    struct io_uring_sqe *sqe;
 +} StgAsyncURingSQE;
++

rts/posix/Poll.c

@@ -347,7 +347,7 @@ void pollCompletedTimeoutsOrIOPoll(Capability *cap)
  #if defined(HAVE_DECL_PPOLL) && HAVE_DECL_PPOLL == 1
          /* We could use poll here, since we use no timeout, but for
             consistency we use the same syscall as at the other call site. */
 -        struct timespec tv = (struct timespec) { .tv_sec = 0, .tv_nsec = 0 };
 +        struct timespec tv = { .tv_sec = 0, .tv_nsec = 0 };
          int res = ppoll(iomgr->aiop_poll_table, nfds, &tv, NULL);
          debugTrace(DEBUG_iomanager,

rts/posix/URing.c

 +/* -----------------------------------------------------------------------------
 + *
 + * (c) The GHC Team 2021-2023
 + *
 + * An I/O manager based on the Linux io_uring API.
 + *
 + * ---------------------------------------------------------------------------*/
++
 +#include "rts/PosixSource.h"
 +#include "Rts.h"
++
 +#include "IOManager.h" // defines IOMGR_ENABLED_URING
++
 +#if defined(IOMGR_ENABLED_URING)
++
 +#include "Capability.h"
 +#include "Threads.h"
 +#include "Schedule.h"
 +#include "Prelude.h"
 +#include "RtsUtils.h"
 +#include "rts/Time.h"
 +#include "RaiseAsync.h"
++
 +#include "URing.h"
 +#include "Signals.h"
++
 +#include <liburing.h>
 +#include <poll.h> // for poll() flags POLLIN POLLOUT
 +#include <limits.h>
 +#include <errno.h>
 +#include <fcntl.h>
++
 +#include "IOManagerInternals.h"
 +#include "Timeout.h"
++
 +/******************************************************************************
++
 +This I/O manager is based on the Linux io_uring API. We rely on the liburing
 +library, rather than using the system calls directly.
++
 +Introduction
 +============
++
 +The io_uring API is an _almost_ generic mechanism for performing Linux syscalls
 +asynchronously. It supports a range of I/O related operations, including
 +ordinary file read and write, and waiting for I/O readiness. It works using a
 +queue to submit I/O operations, and another queue to receive I/O completions.
 +The io_uring documentation calls these the "submission queue", abbreviated SQ,
 +and "completion queue", abbreviated CQ. The corresponding queue entries are
 +abbreviated as SQEs and CQEs.
++
 +There is a single system call to both submit operations and/or wait for I/O
 +completion (or a timeout). It is also possible to poll for new entries in the
 +completion queue without using a system call at all. This fits the RTS
 +scheduler design quite well. Every time round the scheduler loop we have to
 +do a non-blocking check for I/O completion, and it is only when there are no
 +runnable threads that we want to block and wait for I/O (or timers). So in busy
 +applications there are a lot more non-blocking than blocking checks for I/O
 +completion. So being able to do the non-blocking poll without needing a system
 +call should save significantly on system calls, compared to other APIs.
++
 +We use the liburing C library, rather than the system calls directly. This
 +provides a degree of convenience and portability across kernel versions.
++
 +Synchronous I/O
 +---------------
++
 +Classically, asynchronous I/O APIs are slower than ordinary synchronous I/O
 +APIs for the case of buffered I/O reads where the requested data is already in
 +the OS page cache (if they support buffered I/O at all). This requires
 +applications that use asynchronous I/O (and thus care about performance) to use
 +it only in some cases, and use synchronous I/O when that is expected to be
 +faster. This is complex, and thus relatively few applications use asynchronous
 +I/O. Furthermore, knowing whether data is in the page cache is something that
 +only the kernel knows reliably. User space can only make educated guesses.
++
 +A major selling point of io_uring is that it (mostly) solves this problem. It
 +is designed work for buffered I/O (as well as direct I/O). Operations are
 +submitted by putting entries (SQEs) into the submission queue (SQ) and then
 +entering the kernel (io_uring_enter). The kernel starts work on all the
 +operations. If any of them can complete synchronously then the kernel can place
 +the completion entries (CQEs) into the completion queue (CQ). Thus the
 +synchronous operations complete synchronously. Furthermore, the decision about
 +whether it can complete synchronously is made dynamically by the kernel (e.g.
 +based on whether the data is available in the page cache). The result is that a
 +simple file read where the data is available in the page cache can be as quick
 +as a normal synchronous file read system call.
++
 +Overall, this allows the same API to be used for all I/O, without having to
 +guess about synchronous vs asynchronous completions. Furthermore, io_uring
 +supports both disk I/O and sockets I/O, whereas older APIs only supported one
 +or the other (see e.g. epoll for sockets and Linux/Posix AIO for disk files).
 +Overall this allows a less complex solution, by using the same relatively
 +uniform API for everything (even through the API itself is somewhat more
 +complex than other APIs).
++
 +General strategy
 +----------------
++
 +Our I/O strategy with io_uring is to prepare but not submit I/O operations in
 +the I/O primops, and then submit the operations in the scheduler. That is, in
 +the I/O primops we put the I/O operations (SQEs) into the submission queue (SQ),
 +but we don't yet call into the kernel to inform it of the new operations.
 +Instead we do that in the scheduler. At this point in the scheduler, we can
 +both submit any pending I/O operations and handle any completions. This lets us
 +submit and collect I/O with a single system call. Handling I/O completions
 +typically results in waking up threads, which the scheduler can then deal with.
++
 +Consider the important example of simple disk reads that complete synchronously.
 +In this case the completion is available immediately after io_uring_enter
 +returns and the scheduler can reschedule the thread that submitted the I/O. So
 +the end result _should_ be as fast as a normal synchronous blocking read()
 +system call (but benchmarks are needed to verify this).
++
 +Important fast-path cases
 +-------------------------
++
 +There are a few important special fast-path cases to keep in mind:
++
 +1. Common case: as above, a Haskell thread has pending I/O, so we submit
 +   it. This requires a system call. Afterwards we also process any
 +   completions. This means any I/O operations that complete synchronously
 +   get handled immediately and with a single system call. This is crucial
 +   for performance of ordinary buffered I/O in the common case that the data
 +   is available in the page cache.
++
 +2. Common case: no pending I/O to submit, but there is outstanding I/O
 +   that we need to see if it has completed. This does not require a
 +   system call. We just need to look at the completion ring.
++
 +3. Rare case: we have pending I/O to submit but the kernel refuses to
 +   accept more I/O because the completion queue is full. In this case
 +   we process the completion queue first, and then retry. This ends up
 +   using two system calls.
++
 +I/O submission overload
 +-----------------------
++
 +Asynchronous I/O APIs enable I/O to be submitted without waiting and thus they
 +must deal with the problem of having too much I/O in flight at once for the
 +resources available.
++
 +For example, epoll_ctl will return ENOMEM or ENOSPC if it cannot allocate the
 +necessary memory or hits a resource limit. The epoll backend for MIO turns
 +these failures into Haskell exceptions. This works ok in practice for epoll
 +because epoll only supports one kind of async I/O operation: waiting for I/O
 +readiness. This has relatively low resource use and so systems generally hit
 +other resource limits first, e.g. file descriptors.
++
 +On the other hand io_uring supports a variety of I/O operations with different
 +behaviour and resource use. They can roughly be divided into two categories:
 +1. cheap blocking operations; and
 +2. expensive non-blocking operations.
++
 +The cheap blocking operations include waiting for I/O readiness on pipes and
 +sockets, as with epoll, but it also includes blocking reads/write/send/recv on
 +pipes and sockets. These operations tend to have very low resource use and
 +correspondingly the kernel can support very many outstanding operations. This
 +is needed for some large scale networking use cases where there can be very
 +many sockets in use at once.
++
 +The comparatively expensive non-blocking operations include read and write on
 +disk files and various file and file system operations. These operations tend
 +to have higher resource use and so the kernel cannot support too many of them
 +in progress at once. Furthermore, for non-blocking operations there isn't much
 +need to have huge numbers in progress at once: the performance benefit is
 +limited by the hardware concurrency (CPUs, SSD queue depth etc).
++
 +Unfortunately, the limits on the number of concurrent operations of each kind
 +is not known, or at least not reported by the kernel in advance. When trying
 +to submit new operations, the kernel can report EAGAIN if it is out of
 +resources. This reporting mechanism is awkward because of the two classes of
 +operations. The way we would like to respond to hitting a limit is also
 +different for blocking vs non-blocking operations.
++
 +For non-blocking operations, we can simply wait for some operations to complete
 +and then submit more. This works because non-blocking operations will
 +eventually complete, and indeed typically complete fairly promptly.
++
 +For blocking operations on the other hand, there's not a lot we can do if we
 +hit a resource limit. Waiting may not help. Blocking operations can block
 +indefinitely. It is also possible to deadlock if not all I/O readiness
 +notifications are active simultaneously. So if we hit a resource limit for
 +these operations then we can't do much better than throw exceptions to the
 +Haskell threads submitting the operations. This is of course what the MIO epoll
 +backend does.
++
 +We will make the assumption that the kernel does _not_ have separate limits for
 +these two classes of operations, but assume that it is a common limit based on
 +the common resource of kernel memory. As noted above, there is no great benefit
 +to having excessive concurrency of non-blocking operations. We can limit the
 +concurrency (using a queue) with no change in semantics of the application. On
 +the other hand an application that wants to wait on zillions of blocking
 +operations cannot have that concurrency reduced without it having a semantic
 +effect on the application. We either have to support it or (hopefully
 +gracefully) fail as resources run out.
++
 +So we take the approach of trying to limit the concurrency and thus resource
 +use of the non-blocking operations so that the remaining kernel memory can
 +maximise the number of blocking operations we can support. We do that by
 +imposing a "reasonable" limit on the number of concurrent non-blocking
 +operations. We also impose a separate larger limit on the number of concurrent
 +blocking operations. The high level idea is that we treat the non-blocking
 +limit like a semaphore (with that concurrency limit) on the threads submitting
 +non-blocking I/O operations. This means we can handle overload transparently.
 +On the other hand the limit on blocking operations is a hard limit and if we
 +hit that then we have to throw exceptions (to the threads submitting the I/O).
++
 +In the typical application use cases this strategy should avoid encountering
 +EAGAIN in the first place, however it can still happen and we must handle it
 +somehow.
++
 +Before considering EAGAIN, consider a more normal scenario with lots of
 +concurrent I/O -- both blocking and non-blocking operations -- where we do hit
 +the limit on non-blocking operations. We track the number of blocking and
 +non-blocking operations that are currently "in flight" using the counters
 +iomgr->uring_inflight_{non}blocking_aiops. These are incremented by the number
 +of prepared operations when the submission queue is successfully flushed, and
 +decremented for each completion processed. Hitting the limit means that when a
 +thread submits a new non-blocking operation we find the count of inflight
 +non-blocking operations is already at the limit, and thus a new operation would
 +be over the limit.
++
 +Normally when a thread uses a primop to submit an I/O operation we would grab
 +an existing SQE from the submission queue (SQ) and fill in the SQE. If however
 +we would go over the inflight limit, then instead we allocate a fresh SQE (on
 +the C heap using malloc) and then fill in the SQE as normal. Then we put the
 +SQE and the TSO for the thread that submitted it onto (the end of) a pair of
 +overflow queues: one for the SQEs and one for the corresponding threads. We use
 +a pair of queues rather than a queue of pairs because the TSOs are allocated on
 +the GC heap but the SQEs are on the C heap and there is existing infrastructure
 +for handling TSO queues, including tracing them for GC. In the scheduler, when
 +we process the completion of a blocking operation, if there are entries on the
 +overflow queues then we can dequeue an SQE and corresponding TSO and copy the
 +SQE into the SQ and reschedule the TSO. This means the SQE is ready to go and
 +the TSO is unblocked. If we are in this overflow situation and the completions
 +for several non-blocking operations are processed in one go then will will add
 +several SQEs to be added to the SQ in one batch. This should naturally lead to
 +batching in the overflow situation and amortise the syscall overheads.
++
 +While all of this is going on, other threads submitting _blocking_ operations
 +can proceed as normal, preparing their operations into the submission queue.
++
 +Now if we do encounter EAGAIN, we assume that it's the blocking operations
 +that are at fault.
++
 +Approaches to batching
 +----------------------
++
 +We currently do no batching, but this section discusses plausible approaches.
++
 +With this design we have the opportunity to try to accumulate multiple pending
 +I/O operations and then submit them in one batch, which could improve
 +performance in I/O intensive applications by reducing the number of system
 +calls. Ordinary synchronous I/O primops like threadWaitRead# block the Haskell
 +thread and return to the scheduler, which means we only accumulate a single
 +pending I/O operation. There are a couple ways we could achieve batching.
++
 +One approach is to allow individual Haskell threads to prepare multiple I/O
 +operations by providing asynchronous I/O primops. This could be quite effective
 +at generating a lot of I/O, but it would likely see relatively little use
 +because it requires changing application designs.
++
 +Another approach is by having the scheduler not always submit pending I/O, but
 +instead let it run other threads in the hope that they will produce more
 +pending I/O. It would wait until either a time limit or a pending count limit
 +before submitting the I/O. Such an approach would increase I/O latency:
 +consider the scheduler running a thread that creates a pending I/O operation,
 +followed by running a CPU-bound thread for a whole 20 millisecond timeslice,
 +after which the scheduler submits the I/O. Thus this approach could only be
 +used for low priority I/O, which itself would require introducing a notion of
 +I/O priority.
++
 +One can imagine variations on this design such as a more sophisticated
 +scheduler predicting if the next thread to run is likely to be I/O or CPU bound
 +and using that to decide whether to flush pending I/O or to speculate on
 +accumulating more. Or the scheduler could set a timer to interrupt CPU bound
 +threads sooner if it's speculating on gathering more I/O. This could reduce
 +the cost on average, and bound latency, but there would still be a latency vs
 +throughput tradeoff, which would almost certainly require some notion of I/O
 +priority.
++
++
 +io_uring features we use
 +------------------------
++
 +IORING_FEAT_NODROP
++
 +io_uring features we could use but don't (yet)
 +----------------------------------------------
++
 +io_uring features we cannot use
 +-------------------------------
++
 +NOTES: cannot use registered ring fd, due to multiple worker threads, even
 +though it's protected by the lock. Same for registered/direct fds.
++
++
 +Tracking counters
 +-----------------
++
 +We track the number of operations submitted (by Haskell threads to the I/O
 +manager) and not yet notified of completion:
++
 +> int n_submitted_b;
 +> int n_submitted_nb;
++
 +These are incremented when the primop submits I/O to the I/O manager, and are
 +decremented when the I/O completion is processed. We track blocking and
 +non-blocking operations separately.
++
 +We track the number of operations that are prepared in the submission queue,
 +but not yet submitted to the kernel.
++
 +> int n_prepared_b;
 +> int n_prepared_nb;
++
 +These are incremented when an operation is prepared in the submission queue
 +and decremented when operations are submitted to the kernel.
++
 +We track the number of operations submitted to the kernel and where the
 +completion has not yet been processed. We call these "in-flight" operations.
++
 +> int n_inflight_b;
 +> int n_inflight_nb;
++
 +These are incremented when operations are submitted to the kernel, and
 +decremented when I/O completions are processed.
++
 +We track the limit on the number of operations the I/O manager will allow to be
 +in-flight with the kernel.
++
 +> int limit_inflight_b;
 +> int limit_inflight_nb;
++
 +These are typically set on RTS startup and then rarely changed. The
 +limit_inflight_b can be reduced dynamically if the I/O manager encounters
 +EAGAIN when submitting operations.
++
 +We track the number of non-blocking operations that are in the overflow queue.
 +These are operations that have been submitted, and could not be put into the
 +submission queue (because it would exceed the limit_inflight_nb), and so go
 +into the overflow queue instead.
++
 +> int n_overflow_nb;
++
 +This is incremented (instead of n_prepared_nb) if the number of n_inflight_nb
 +plus n_prepared_nb is at or above the limit_inflight_nb, in which case the
 +operation (and submitting thread) is put into the overflow queue. It is
 +decremented when there is space available within the limit_inflight_nb and the
 +operation can be put into the submission queue (and thus also incrementing the
 +n_prepared_nb).
++
 +We maintain two invariants:
++
 +1. n_submitted_b  = n_prepared_b                  + n_inflight_b
 +2. n_submitted_nb = n_prepared_nb + n_overflow_nb + n_inflight_nb
++
 +The implication is that set of prepared, overflow and inflight operations are
 +distinct from each other and their union is equal to the submitted set. Another
 +way to look at it is that there distinct states for submitted operations:
 +prepared, overflow (for non-blocking) and inflight.
++
 +******************************************************************************/
++
 +/* Forward declarations */
 +static int  enlargeTables(Capability *cap, CapIOManager *iomgr);
 +static void notifyIOCompletion(Capability *cap, StgAsyncIOOp *aiop);
 +static void enqueueOverflowQueue(Capability *cap, CapIOManager *iomgr,
 +                                 StgTSO *tso, struct io_uring_sqe *sqe);
 +static void dequeueOverflowQueue(CapIOManager *iomgr,
 +                                 StgTSO **ptso, struct io_uring_sqe **psqe);
++
 +/* Constants */
++
 +/* A couple tags we add to the sqe->user_data to tell us about this operation
 + * when we process the completion. Currently we distinguish:
 + * 1. non-blocking vs blocking operations, just so we can update our counters
 + *    which count these separately.
 + * 2. cancellation operations
 + */
 +const uint64_t AIOP_TAG_CANCEL = 0x80lu << 56;  /* bit 63 */
 +const uint64_t AIOP_TAG_OP_NB  = 0x40lu << 56;  /* bit 62 */
 +const uint64_t AIOP_TAG_MASK   = 0xc0lu << 56;  /* bit 62 | 63 */
++
++
 +void initCapabilityIOManagerURing(Capability *cap, CapIOManager *iomgr)
 +{
 +    initClosureTable(&iomgr->aiop_table, ClosureTableCompact);
 +    iomgr->timeout_queue = emptyTimeoutQueue();
++
 +    int sq_entries = RtsFlags.MiscFlags.io_uring_sq_entries;
++
 +    iomgr->n_submitted_b     = 0;
 +    iomgr->n_submitted_nb    = 0;
 +    iomgr->n_prepared_b      = 0;
 +    iomgr->n_prepared_nb     = 0;
 +    iomgr->n_inflight_b      = 0;
 +    iomgr->n_inflight_nb     = 0;
 +    iomgr->limit_inflight_b  = INT_MAX;
 +    iomgr->limit_inflight_nb = 4 * sq_entries;
 +    iomgr->n_overflow_nb     = 0;
 +    iomgr->overflow_tso_q_hd = END_TSO_QUEUE;
 +    iomgr->overflow_tso_q_tl = END_TSO_QUEUE;
 +    iomgr->overflow_sqe_q_hd = NULL;
 +    iomgr->overflow_sqe_q_tl = NULL;
++
 +    /* Set the uring params: we want to use independent sizes of submission
 +     * and completion queues. We typically want a bigger completion queue than
 +     * a submission queue.
 +     */
 +    struct io_uring_params params = {
 +             .flags      = IORING_SETUP_CQSIZE
 +                         | IORING_SETUP_CLAMP
 +                         | IORING_SETUP_SUBMIT_ALL,
 +             .cq_entries = RtsFlags.MiscFlags.io_uring_cq_entries
 +           };
 +    //TODO: what happens if we use flags that are not recognised by the kernel
 +    // version, e.g. IORING_SETUP_SUBMIT_ALL prior to 5.18?
++
 +    /* TODO: see if we want to support IORING_SETUP_SQPOLL. With
 +      IORING_FEAT_NATIVE_WORKERS, it doesn't need any priviledges. */
++
 +    /* Share the same kernel work-queue between the urings for each capability.
 +     * Do this by using the IORING_SETUP_ATTACH_WQ flag for capabilities > 0,
 +     * and pass the uring fd for cap 0 (the main capability).
 +     */
 +    if (cap->no > 0) {
 +        params.flags |= IORING_SETUP_ATTACH_WQ;
 +        params.wq_fd  = MainCapability.iomgr->uring->ring_fd;
 +    }
++
 +    /* Try to initialise the uring */
 +    struct io_uring *uring = stgMallocBytes(sizeof (struct io_uring),
 +                                            "initCapabilityIOManagerUring");
 +    int res = io_uring_queue_init_params(sq_entries, uring, &params);
 +    if (res < 0) goto fail;
++
 +    /* Check for features we require. */
 +    unsigned required =
 +        /* Needed for simple handling of ring sizes and limits.
 +         * TODO: we might be able to support kernels without this
 +         * by setting the limits to be the same as the ring sizes. */
 +        IORING_FEAT_NODROP
++
 +        /* TODO: explain why */
 +      | IORING_FEAT_SUBMIT_STABLE
++
 +        /* Needed for read/write that updates the file pos. */
 +      | IORING_FEAT_RW_CUR_POS
++
 +        /* Needed for corner cases like reading from /proc/self,
 +         * or signalfd */
 +      | IORING_FEAT_NATIVE_WORKERS;
++
 +    if ((uring->features & required) != required) goto fail;
++
 +    /* Arrange for the uring (fd and mmap'ed queues) not to be inherited. */
 +    res = fcntl(uring->ring_fd, F_SETFD, FD_CLOEXEC);
 +    if (res < 0) goto fail;
 +    res = io_uring_ring_dontfork(uring);
 +    if (res < 0) goto fail;
++
 +    /* Success. Save what we need. */
 +    iomgr->uring = uring;
 +    return;
++
 +    /* Failure. Clean up. */
 +fail:
 +    stgFree(uring);
 +    barf("uring iomgr: initialisation failed");
 +    //TODO: we should add support to fail and use a fallback I/O manager
 +}
++
++
 +void initCapabilityIOManagerAfterForkURing(Capability *cap STG_UNUSED,
 +                                           CapIOManager *iomgr STG_UNUSED)
 +{
 +    //TODO: ugg, do we need to shutdown all the other caps? Or does that
 +    // happen automagically? Need to look into forkProcess and shutting
 +    // down the I/O managers.
 +}
++
 +/******************************************************************************
 + * Common prologues and epilogues for primops for I/O operations.
 + *
 + * There are different common prologues/epilogues depending on:
 + *  - synchronous or asynchronous primops
 + *  - blocking or non-blocking I/O operations
 + */
++
++
 +/* The common prologue for for all synchronous primops for both blocking and
 + * non-blocking I/O operations.
 + */
 +static int prologueSyncIOOp(Capability *cap, StgTSO *tso,
 +                            int why_blocked, StgAsyncIOOp **paiop)
 +{
 +    StgAsyncIOOp *aiop;
 +    aiop = (StgAsyncIOOp *)allocateMightFail(cap, sizeofW(StgAsyncIOOp));
 +    if (RTS_UNLIKELY(aiop == NULL)) { return (sizeof(StgAsyncIOOp)); }
 +    SET_HDR(aiop, &stg_ASYNCIOOP_info, CCS_SYSTEM); //TODO: get CCCS
 +    aiop->notify_type    = NotifyTSO;
 +    aiop->notify.tso     = tso;
 +    tso->why_blocked     = why_blocked;
 +    tso->block_info.aiop = aiop;
 +    *paiop               = aiop;
 +    return 0;
 +}
++
++
 +/* The common prologue for all non-blocking I/O operations.
 + *
 + * Allocate a table index
 + * Fill in some of the aiop fields
 + * Allocate an SQE, either on the ring or on the heap if we're in overflow.
 + */
 +static int prologueNonBlockingIOOp(Capability *cap,
 +                                   StgTSO *tso, StgAsyncIOOp *aiop,
 +                                   struct io_uring_sqe **psqe,
 +                                   bool *tso_block)
 +{
 +    CapIOManager *iomgr = cap->iomgr;
 +    if (RTS_UNLIKELY(isFullClosureTable(&iomgr->aiop_table))) {
 +        int fail = enlargeTables(cap, iomgr);
 +        if (RTS_UNLIKELY(fail)) return fail;
 +    }
++
 +    int index = insertClosureTable(cap, &iomgr->aiop_table, aiop);
++
 +    aiop->capno = cap->no;
 +    aiop->index = index;
++
 +    struct io_uring_sqe *sqe;
++
 +    if (iomgr->n_inflight_nb + iomgr->n_prepared_nb
 +          < iomgr->limit_inflight_nb) {
 +        /* The typical case. Allocate an SQE from the ring */
 +        sqe = io_uring_get_sqe(iomgr->uring);
 +        ASSERT(sqe); /* Otherwise we counted wrong */
 +        iomgr->n_submitted_nb++;
 +        iomgr->n_prepared_nb++;
 +        *tso_block = false;
 +    } else {
 +        /* We're going to have to block the submitting thread.
 +         * Allocate an SQE on the heap and suspend the calling TSO.
 +         */
 +        sqe = stgMallocBytes(sizeof (struct io_uring_sqe), "uring iomgr");
 +        enqueueOverflowQueue(cap, iomgr, tso, sqe);
 +        iomgr->n_submitted_nb++;
 +        iomgr->n_overflow_nb++;
 +        *tso_block = true;
 +    }
 +    io_uring_sqe_set_data64(sqe, index);
 +    *psqe = sqe;
 +    return 0;
 +}
++
++
 +/* The common prologue for all blocking I/O operations.
 + *
 + * Allocate a table index
 + * Fill in some of the aiop fields
 + * Allocate an SQE, or return failure if we're at the limit.
 + */
 +static int prologueBlockingIOOp(Capability *cap,
 +                                StgTSO *tso, StgAsyncIOOp *aiop,
 +                                struct io_uring_sqe **psqe)
 +{
 +    CapIOManager *iomgr = cap->iomgr;
++
 +    if (iomgr->n_inflight_b + iomgr->n_prepared_b >= iomgr->limit_inflight_b) {
 +        /* If we reach the limit we fail and throw an exception */
 +        raiseAsync(cap, tso, (StgClosure *)blockedOnBadFD_closure
 +                           /*TODO: use ioopResourcesExhausted_closure */,
 +                           false, NULL);
 +        return -1;
 +        //TODO: review this
 +        //TODO: current return value is for memory alloc failure, not for
 +        // other failures. Need error result separate from alloc.
 +    }
++
 +    if (RTS_UNLIKELY(isFullClosureTable(&iomgr->aiop_table))) {
 +        int fail = enlargeTables(cap, iomgr);
 +        if (RTS_UNLIKELY(fail)) return fail;
 +    }
++
 +    int index = insertClosureTable(cap, &iomgr->aiop_table, aiop);
++
 +    aiop->capno = cap->no;
 +    aiop->index = index;
++
 +    /* Allocate an SQE on the ring */
 +    struct io_uring_sqe *sqe = io_uring_get_sqe(iomgr->uring);
 +    ASSERT(sqe); /* Otherwise we counted wrong */
 +    iomgr->n_submitted_b++;
 +    iomgr->n_prepared_b++;
 +    io_uring_sqe_set_data64(sqe, index);
 +    *psqe = sqe;
 +    return 0;
 +}
++
++
 +/* The common epilogue for all async non-blocking I/O operations.
 + */
 +static int epilogueAsyncNonBlockingIOOp(CapIOManager *iomgr,
 +                                        StgTSO *tso, StgAsyncIOOp *aiop,
 +                                        bool tso_block)
 +{
 +    if (tso_block) {
 +        tso->why_blocked     = BlockedOnIOSubmission;
 +        tso->block_info.aiop = aiop;
 +        return -1;
 +    } else if (io_uring_sq_space_left(iomgr->uring) == 0) {
 +        return -1;
 +    } else {
 +      return 0;
 +    }
 +}
++
++
 +/* Common epilogue for all async blocking I/O operations.
 + */
 +static int epilogueAsyncBlockingIOOp(CapIOManager *iomgr)
 +{
 +    return io_uring_sq_space_left(iomgr->uring) == 0 ? -1 : 0;
 +}
++
++
 +/******************************************************************************
 + * Non-blocking I/O operations. This includes read/write on files (not sockets).
 + *
 + * The code is organised as common I/O preparation functions and then individual
 + * primops (sync and async). They also rely on the common prologue and epilogue
 + * functions above.
 + */
++
 +static int prepareIOReadWrite(Capability *cap, StgTSO *tso, bool *tso_block,
 +                              StgAsyncIOOp *aiop,
 +                              IOReadOrWrite rw, int fd,
 +                              StgClosure *live, void *buf,
 +                              size_t len, off_t off) {
 +    struct io_uring_sqe *sqe;
 +    int fail = prologueNonBlockingIOOp(cap, tso, aiop, &sqe, tso_block);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    aiop->live = live;
 +    if (rw == IORead) {
 +        io_uring_prep_read(sqe, fd, buf, len, off);
 +    } else {
 +        io_uring_prep_write(sqe, fd, buf, len, off);
 +    }
 +    return 0;
 +}
++
++
 +int asyncIOReadWriteURing(Capability *cap, StgTSO *tso, StgAsyncIOOp *aiop,
 +                          IOReadOrWrite rw, int fd,
 +                          StgClosure *live, void *buf,
 +                          size_t len, off_t off)
 +{
 +    bool tso_block;
 +    int fail = prepareIOReadWrite(cap, tso, &tso_block, aiop,
 +                                  rw, fd, live, buf, len, off);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    return epilogueAsyncNonBlockingIOOp(cap->iomgr, tso, aiop, tso_block);
 +}
++
++
 +int syncIOReadWriteURing(Capability *cap, StgTSO *tso,
 +                         IOReadOrWrite rw, int fd,
 +                         StgClosure *live, void *buf,
 +                         size_t len, off_t off)
 +{
 +    StgAsyncIOOp *aiop;
 +    int why_blocked = rw == IORead ? BlockedOnRead : BlockedOnWrite;
 +    int fail = prologueSyncIOOp(cap, tso, why_blocked, &aiop);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    bool unused;
 +    fail = prepareIOReadWrite(cap, tso, &unused, aiop,
 +                              rw, fd, live, buf, len, off);
 +    if (RTS_UNLIKELY(fail)) return fail;
 +    return 0;
 +}
++
++
 +/******************************************************************************
 + * Blocking I/O operations. This includes waiting for I/O readiness on sockets,
 + * pipes etc.
 + *
 + * The code is organised as common I/O preparation functions and then individual
 + * primops (sync and async). They also rely on the common prologue and epilogue
 + * functions above.
 + */
++
++
 +static void prepareIOWaitReady(StgAsyncIOOp *aiop, struct io_uring_sqe *sqe,
 +                               IOReadOrWrite rw, int fd) {
 +    aiop->live = &stg_ASYNCIO_LIVE0_closure;
 +    io_uring_prep_poll_add(sqe, fd, rw == IORead ? POLLIN : POLLOUT);
 +}
++
++
 +int asyncIOWaitReadyURing(Capability *cap, StgTSO *tso, StgAsyncIOOp *aiop,
 +                          IOReadOrWrite rw, int fd)
 +{
 +    struct io_uring_sqe *sqe;
 +    int fail = prologueBlockingIOOp(cap, tso, aiop, &sqe);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    prepareIOWaitReady(aiop,sqe, rw, fd);
++
 +    return epilogueAsyncBlockingIOOp(cap->iomgr);
 +}
++
++
 +int syncIOWaitReadyURing(Capability *cap, StgTSO *tso,
 +                         IOReadOrWrite rw, int fd)
 +{
 +    StgAsyncIOOp *aiop;
 +    int why_blocked = rw == IORead ? BlockedOnRead : BlockedOnWrite;
 +    int fail = prologueSyncIOOp(cap, tso, why_blocked, &aiop);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    struct io_uring_sqe *sqe;
 +    fail = prologueBlockingIOOp(cap, tso, aiop, &sqe);
 +    if (RTS_UNLIKELY(fail)) return fail;
++
 +    prepareIOWaitReady(aiop, sqe, rw, fd);
++
 +    return 0;
 +}
++
++
 +/******************************************************************************
 + * Actions to cancel outstanding I/O operations. Also support cancelling any
 + * outstanding I/O on an fd prior to it being closed.
 + *
 + * This covers both synchronous and asynchronous operations.
 + */
++
++
 +static void ioCancel(Capability *cap, StgAsyncIOOp *aiop);
++
++
 +void syncIOCancelURing(Capability *cap, StgTSO *tso)
 +{
 +    StgAsyncIOOp *aiop  = tso->block_info.aiop;
 +    ASSERT(aiop->notify_type == NotifyTSO);
 +    ASSERT(indexClosureTable(&cap->iomgr->aiop_table, aiop->index) == aiop);
 +    ioCancel(cap, aiop);
 +    /* We cannot use the normal notifyIOCompletion here. We are in the context
 +     * of throwTo, interrupting a thread blocked on IO via an async exception.
 +     * We don't put the TSO back on the run queue or change the why_blocked
 +     * status, as that is done by removeFromQueues (in the throwTo* functions).
 +     */
 +    tso->block_info.closure = (StgClosure *)END_TSO_QUEUE;
++
 +    //TODO: Synchronous cancellation from throwTo seems to be pretty keen for
 +    // the thread to be unblocked immediately so it can start unwinding the
 +    // stack. Perhaps it is ok to continue the cancellation in the background.
 +    // But if so then we will need to adjust the notify type to be none /
 +    // cancelled!
 +    /* Cancelling thread-synchronous I/O happens from throwTo, which is very
 +     * keen for the thread to be unblocked immediately so it can unwind the
 +     * stack and schedule the thread to run an exception handler. This demand
 +     * to be synchronous is a bit tricky to arrange because cancelling the I/O
 +     * operation the thread is blocked on is potentially asynchronous.
 +     */
 +    aiop->notify_type = NotifyNone;
 +    aiop->notify.tso  = END_TSO_QUEUE;
 +}
++
++
 +void asyncIOCancelURing(Capability *cap, StgAsyncIOOp *aiop)
 +{
 +    /* We can reliably determine if the aiop is still in progress by checking
 +     * if the aiop_table still points to this aiop object. This is reliable
 +     * because each aiop is GC heap allocated, so cannot be recycled until it
 +     * is no longer retained by the application.
 +     */
 +    ASSERT(aiop->notify_type != NotifyTSO);
 +    if (indexClosureTable(&cap->iomgr->aiop_table, aiop->index) == aiop) {
 +        ioCancel(cap, aiop);
 +        notifyIOCompletion(cap, aiop);
 +    }
 +}
++
++
 +static void ioCancel(Capability *cap, StgAsyncIOOp *aiop)
 +{
 +    barf("URing.c:ioCancel:TODO");
++
 +    CapIOManager *iomgr = cap->iomgr;
++
 +    int ix = aiop->index;
 +    struct io_uring_sqe *sqe = io_uring_get_sqe(iomgr->uring);
 +    //TODO: return status to indicate if we need to return to the scheduler
 +    // to flush a full submission queue
++
 +    /* io_uring lets us include one word into submission queue entries (SQEs),
 +     * which come back in the corresponding completion queue entry (CQE). We
 +     * use this feature to identify the aiop so that we will be able to
 +     * process the completion properly, e.g. waking up the right thread.
 +     * We can't use a direct pointer to the aiop because the aiops are heap
 +     * allocated and GC pointers are not stable. We use the index in the
 +     * ClosureTable, because this is stable. Indeed the raison d'être of the
 +     * ClosureTable is to provide stable pointers for thus purpose.
 +     //TODO: the above is a helpful comment but it belongs elsewhere, e.g.
 +     //in the intro.
 +     */
 +    uint64_t sqe_data = ix;
++
 +    /* Although IORING_OP_POLL_ADD has a special separate cancellation using
 +     * IORING_OP_POLL_REMOVE, apparently it can also be cancelled using the
 +     * generic IORING_OP_ASYNC_CANCEL, which is good so we don't need to
 +     * distinguish.
 +     */
 +    io_uring_prep_cancel64(sqe, sqe_data, 0 /*flags*/);
 +    //TODO: if we use other tags in the sqe_data we'll need to reconstruct
 +    // them here so we can find the right item. e.g. if we use AIOP_TAG_OP_NB
 +    // we'd need to distinguish in the aiop->flags for example.
++
 +    /* Cancelling is itself a new uring I/O operation which will have a
 +     * corresponding completion. Set a high bit to mark this as a cancel
 +     * operation, but still knowing the index of the original operation.
 +     */
 +    io_uring_sqe_set_data64(sqe, sqe_data | AIOP_TAG_CANCEL);
++
 +    //TODO: verify for IORING_OP_POLL_ADD, if we do a successful
 +    // IORING_OP_POLL_REMOVE then which completions do we get?
 +    // Do we get a completion for the IORING_OP_POLL_ADD, and if so with
 +    // what result? Or does a successful IORING_OP_POLL_REMOVE mean we
 +    // only get a completion for the remove and not the add?
++
 +    //TODO: verify similar for normal I/O cancel, e.g. a read on a pipe.
 +    // Which completions do we get if we
++
 +    //TODO: is it ok cancel I/O asynchronously here? Do we need to submit the
 +    // cancel op? The cancel op will complete synchronously but the cancellation
 +    // may only complete later. This might confuse resource cleanup, e.g. because
 +    // a file will not get closed until the cancel finishes.
++
 +                /* This is an CQE for a cancellation. After posting a
 +                 * cancellation SQE then we expect to get _both_
 +                 * a cancellation CQE and a CQE for the original operation that
 +                 * was the target of cancellation. This means that (provided
 +                 * it's not an error) we can ignore the cancellation SQE and
 +                 * just process the normal SQE for the target operation.
 +                 *
 +                 * The target operation will either be cancelled successfully
 +                 * immediately (in which case the cancellation cqe_data->res == 0) or if the target operation
 +                 * is in progress and cannot be cancelled, then we'll get
 +                 * cqe_data->res == -EALREADY for the cancellation
 +                 * the
 +                 * result of the operation (which may still be an interrupted
 +                 * outcome. If the cancellation fails, we
 +                 */
 +                if (cqe_data->res != 0 && cqe_data->res != -EALREADY)
 +                    sysErrorBelch("uring:processIOCompletions");
 +                }
++
 +}
++
++
++
++
 +/* TODO: need to add support for closing properly.
 + * Unfortunately, uring's behaviour for closing a fd when there are outstanding
 + * poll (or other async I/O) operations on that fd is unhelpful. The poll
 + * operation itself keeps a reference to the file open. Thus the close will
 + * not in fact interrupt and cancel the poll.
 + * So the I/O manager needs to be notified of fd close, so that we can do
 + * something. Fortunately we can use io_uring_prep_cancel_fd to cancel all
 + * operations on an fd.
 + *
 + * I think cancelled ops _do_ generate CQEs. So we should be able to do the
 + * appropriate notifications by waiting for the original CQEs. We should
 + * probably issue the cancellation
 + *
 + */
++
++
 +/******************************************************************************
 + * The functions called from the scheduler to poll or wait for pending I/O,
 + * and process any I/O completions.
 + */
++
++
 +bool anyPendingTimeoutsOrIOURing(CapIOManager *iomgr)
 +{
 +    return !isEmptyTimeoutQueue(iomgr->timeout_queue)
 +        || !isEmptyClosureTable(&iomgr->aiop_table);
 +}
++
++
 +static void notifyIOCompletion(Capability *cap, StgAsyncIOOp *aiop)
 +{
 +    switch (aiop->notify_type) {
 +        case NotifyTSO:
 +        {
 +            if (aiop->outcome == IOOpOutcomeFailed && aiop->error == EBADF) {
 +                /* The fd is invalid: raise an IOError exception in the blocked
 +                 * thread. (See bug #4934 for what happens without this.)
 +                 */
 +                StgTSO *tso = aiop->notify.tso;
 +                debugTrace(DEBUG_iomanager,
 +                           "Raising exception in thread %" FMT_StgThreadID
 +                           " blocked on an invalid fd", tso->id);
 +                raiseAsync(cap, tso, (StgClosure *)blockedOnBadFD_closure,
 +                           false, NULL);
 +                break;
 +            } else {
 +                /* We should be guaranteed that the tso is still on the same
 +                 * cap because the tso was not on the run queue of any cap and
 +                 * so is not subject to thread migration.
 +                 */
 +                StgTSO *tso      = aiop->notify.tso;
 +                tso->why_blocked = NotBlocked;
 +                tso->_link       = END_TSO_QUEUE;
 +                pushOnRunQueue(cap, tso);
 +            }
 +            break;
 +        }
 +        case NotifyMVar:
 +            performTryPutMVar(cap, aiop->notify.mvar, Unit_closure);
 +            break;
++
 +        case NotifyTVar:
 +            barf("uring iomgr: TVar notification not yet supported");
 +            break;
 +    }
 +}
++
++
 +/* Process all the I/O completions that are currently available without
 + * blocking.
 + *
 + * This will correctly deal with completion queue overflow: if the completions
 + * queue is empty but has the overflow bit set then io_uring_peek_batch_cqe
 + * will do another non-blocking uring enter to refill the completion queue.
 + */
 +static void processIOCompletions(Capability *cap, CapIOManager *iomgr)
 +{
 +    struct io_uring *uring = iomgr->uring;
 +    while (1) {
 +        unsigned head, count = 0;
 +        struct io_uring_cqe *cqe;
 +        io_uring_for_each_cqe(uring, head, cqe) {
 +            uint64_t cqe_data = io_uring_cqe_get_data64(cqe);
 +            if (RTS_UNLIKELY(cqe_data & AIOP_TAG_CANCEL)) {
++
 +            } else {
 +                int ix = cqe_data;
 +                StgAsyncIOOp *aiop = indexClosureTable(&iomgr->aiop_table, ix);
 +                removeClosureTable(cap, &iomgr->aiop_table, ix);
 +                aiop->result = cqe->res;
 +                //TODO: if we use these tags in the sqe_data we'll need to reconstruct
 +                // them for cancel so we can find the right item. e.g. if we use AIOP_TAG_OP_NB
 +                // we'd need to distinguish in the aiop->flags for example.
 +                if (cqe_data & AIOP_TAG_OP_NB) {
 +                    iomgr->n_inflight_nb--;
 +                    iomgr->n_submitted_nb--;
 +                } else {
 +                    iomgr->n_inflight_b--;
 +                    iomgr->n_submitted_b--;
 +                }
 +                notifyIOCompletion(cap, aiop);
 +            }
 +            count++;
 +        }
 +        io_uring_cq_advance(uring, count);
 +        if (RTS_UNLIKELY(io_uring_cq_has_overflow(uring))) {
 +            if (io_uring_get_events(uring) < 0) {
 +                sysErrorBelch("io_uring_enter");
 +                stg_exit(EXIT_FAILURE);
 +            }
 +            continue;
 +        } else {
 +            break;
 +        }
 +    }
 +}
++
++
 +/* Check invariants that must hold on entry to and exit from the scheduler.
 + * Used before/after {poll,await}CompletedTimeoutsOrIOURing which are called
 + * from the scheduler.
 + */
 +static void assertURingSchedulerInvariants(CapIOManager *iomgr)
 +{
 +    struct io_uring *uring = iomgr->uring;
++
 +    // That our tracking counters are consistent.
 +    ASSERT(iomgr->n_submitted_b == iomgr->n_prepared_b
 +                                 + iomgr->n_inflight_b);
 +    ASSERT(iomgr->n_submitted_nb == iomgr->n_prepared_nb
 +                                  + iomgr->n_overflow_nb
 +                                  + iomgr->n_inflight_nb);
++
 +    // That we are within limits
 +    ASSERT(iomgr->n_inflight_b  <= iomgr->limit_inflight_b);
 +    ASSERT(iomgr->n_inflight_nb <= iomgr->limit_inflight_nb);
++
 +    // That we correctly track the submission queue size.
 +    ASSERT((int)io_uring_sq_ready(uring) == iomgr->n_prepared_b
 +                                          + iomgr->n_prepared_nb);
++
 +    // That our overflow queue is consistent with the overflow counter.
 +    ASSERT(iomgr->n_overflow_nb > 0
 +         ? iomgr->overflow_sqe_q_hd == NULL &&
 +           iomgr->overflow_sqe_q_tl == NULL &&
 +           iomgr->overflow_tso_q_hd == END_TSO_QUEUE &&
 +           iomgr->overflow_tso_q_tl == END_TSO_QUEUE
 +         : iomgr->n_overflow_nb == 0 &&
 +           iomgr->overflow_sqe_q_hd != NULL &&
 +           iomgr->overflow_sqe_q_tl != NULL &&
 +           iomgr->overflow_tso_q_hd != END_TSO_QUEUE &&
 +           iomgr->overflow_tso_q_tl != END_TSO_QUEUE);
 +}
++
 +/* If there are any completed I/O operations or expired timers, process the
 + * completions as appropriate. If there are none, return without waiting.
 + *
 + * This is the non-blocking variant. See awaitCompletedTimeoutsOrIOURing
 + * for the potentially-blocking variant.
 + */
 +void pollCompletedTimeoutsOrIOURing(Capability *cap)
 +{
 +    CapIOManager    *iomgr = cap->iomgr;
 +    struct io_uring *uring = iomgr->uring;
++
 +    assertURingSchedulerInvariants(iomgr);
++
 +    /* Process timeouts, if any, but don't immediately return to the scheduler,
 +     * since we should submit I/O and reap any completions too.
 +     */
 +    if (!isEmptyTimeoutQueue(iomgr->timeout_queue)) {
 +        Time now = getProcessElapsedTime();
 +        processTimeoutCompletions(cap, now);
 +    }
++
 +    /* Submit I/O if needed */
 +    if (io_uring_sq_ready(uring)) {
 +        int res = io_uring_submit_and_get_events(uring);
++
 +        if (RTS_UNLIKELY(res < 0)) {
 +            if (res == -EBUSY) {
 +                /* This is an odd one. According to the doc:
 +                 * If the IORING_FEAT_NODROP feature flag is set, then EBUSY
 +                 * will be returned if there were overflow entries,
 +                 * IORING_ENTER_GETEVENTS flag is set and not all of the
 +                 * overflow entries were able to be flushed to the CQ ring.
 +                 *
 +                 * So it's not really an error at all. It just means we will
 +                 * have to do multiple iterations in processIOCompletions()
 +                 * to collect all the completions.
 +                 *
 +                 * Thus EBUSY should imply that there are entries in the CQ.
 +                 */
 +                ASSERT(io_uring_cq_ready(uring) > 0);
 +            }
 +        } else {
 +            ASSERT(res == iomgr->n_prepared_b + iomgr->n_prepared_nb);
 +            /* We're using IORING_SETUP_SUBMIT_ALL so we should expect to have
 +             * all of them submitted, or an error.
 +             * https://github.com/axboe/liburing/issues/186
 +             * Alternatively, we could loop and submit the remainder.
 +             */
 +           iomgr->n_inflight_b  += iomgr->n_prepared_b;
 +           iomgr->n_inflight_nb += iomgr->n_prepared_nb;
 +           iomgr->n_prepared_b  = 0;
 +           iomgr->n_prepared_nb = 0;
 +        }
 +    }
++
 +    if (io_uring_cq_ready(uring)) {
 +        processIOCompletions(cap, iomgr);
 +    }
 +    //TODO: now we need to check if we have any items in our overflow queue
 +    //and if so, we need to copy some of those into the SQ and submit them.
 +    //copy in up to either the SQ limit or in-flight limit.
++
 +    assertURingSchedulerInvariants(iomgr);
 +}
++
++
 +/* If there are any completed I/O operations or expired timers, process the
 + * completions as appropriate. If there are none, wait until I/O or a timer
 + * does complete (or we get a signal with a handler) and process the
 + * completions as appropriate.
 + *
 + * This is the potentially-blocking variant. See pollCompletedTimeoutsOrIOURing
 + * for the non-blocking variant.
 + */
 +void awaitCompletedTimeoutsOrIOURing(Capability *cap)
 +{
 +    CapIOManager    *iomgr = cap->iomgr;
 +    struct io_uring *uring = iomgr->uring;
++
 +    assertURingSchedulerInvariants(iomgr);
++
 +    do {
++
 +        /* We're being asked (by the scheduler) to block if there's no
 +         * immediate timer or I/O completions. So there had better be
 +         * some pending I/O or pending timers, or we'd deadlock.
 +         */
 +        ASSERT(!isEmptyTimeoutQueue(iomgr->timeout_queue) ||
 +               !isEmptyClosureTable(&iomgr->aiop_table));
++
 +        Time now = getProcessElapsedTime();
 +        processTimeoutCompletions(cap, now);
++
 +        /* If we didn't wake any threads due to expiring timeouts, then we need
 +         * to wait on I/O. Or to put it another way, even if we did wake some
 +         * threads, we'll still poll (but not wait) for I/O. This is to ensure
 +         * we avoid starving threads blocked on I/O.
 +         */
 +        bool wait = emptyRunQueue(cap);
++
 +        /* There are four possible cases:
 +         * 1. non-blocking check for I/O completion with I/O submission
 +         * 2. non-blocking check for I/O completion with no I/O submission
 +         * 3. blocking wait for I/O completion with a timeout
 +         * 4. blocking wait for I/O completion without a timeout
 +         *
 +         * If we woke any threads due to timeouts we're in the first or second
 +         * case.
 +         *
 +         * Otherwise we're in one of the blocking cases. We will use a timeout
 +         * if the timeout queue is non-empty.
 +         */
++
 +        int res;
 +        if (!wait && io_uring_sq_ready(uring)) {
 +            /* Case 1 (as above) */
 +            res = io_uring_submit_and_get_events(uring);
 +        } else if (!wait) {
 +            /* Case 2 (as above) */
 +            res = io_uring_get_events(uring);
 +        } else {
 +            struct timespec tv;
 +            if (timeoutInNanoseconds(iomgr, true, now, &tv)) {
 +                /* Case 3 (as above) */
 +                /* struct timespec and struct __kernel_timespec are compatible
 +                 * but not exactly the same. Sigh. */
 +                struct __kernel_timespec ts = { .tv_sec  = tv.tv_sec,
 +                                                .tv_nsec = tv.tv_nsec };
 +                struct io_uring_cqe *cqe_unused;
 +                res = io_uring_submit_and_wait_timeout(uring, &cqe_unused,
 +                                                       1, &ts, NULL);
 +            } else  {
 +                /* Case 4 (as above) */
 +                res = io_uring_submit_and_wait(uring, 1);
 +            }
 +        }
++
 +        if (res >= 0) {
 +            processIOCompletions(cap, iomgr);
 +        } else if (errno == EINTR) {
++
 +        } else if (errno == EBUSY || errno == EAGAIN) {
++
 +        } else {
 +            sysErrorBelch("io_uring_enter");
 +            stg_exit(EXIT_FAILURE);
 +        }
 +    } while (emptyRunQueue(cap)
 +         && (!isEmptyClosureTable(&iomgr->aiop_table) ||
 +             !isEmptyTimeoutQueue(iomgr->timeout_queue))
 +         && getSchedState() == SCHED_RUNNING);
++
 +    assertURingSchedulerInvariants(iomgr);
 +}
++
++
 +/******************************************************************************
 + * Local helper utilities
 + */
++
 +static int enlargeTables(Capability *cap, CapIOManager *iomgr)
 +{
 +    int oldcapacity = capacityClosureTable(&iomgr->aiop_table);
 +    int newcapacity = (oldcapacity == 0) ? 1 : (oldcapacity * 2);
++
 +    int fail = enlargeClosureTable(cap, &iomgr->aiop_table, newcapacity);
 +    if (RTS_UNLIKELY(fail)) return fail;
 +    return 0;
 +}
++
++
 +/*
 + */
 +static void enqueueOverflowQueue(Capability *cap, CapIOManager *iomgr,
 +                                 StgTSO *tso, struct io_uring_sqe *sqe)
 +{
 +    /* Append the TSO to the tail of the overflow queue of TSOs. */
 +    ASSERT(tso->_link == END_TSO_QUEUE);
 +    if (iomgr->overflow_tso_q_hd == END_TSO_QUEUE) {
 +        iomgr->overflow_tso_q_hd = tso;
 +    } else {
 +        setTSOLink(cap, iomgr->overflow_tso_q_tl, tso);
 +    }
 +    iomgr->overflow_tso_q_tl = tso;
++
 +    /* And append the SQE to the tail of the overflow queue of SQEs. */
 +    struct overflow_sqe_q_t *entry;
 +    entry = stgMallocBytes(sizeof(struct overflow_sqe_q_t), "uring iomgr");
 +    *entry = (struct overflow_sqe_q_t) {
 +               .sqe  = sqe,
 +               .next = NULL,
 +#if defined(DEBUG)
 +               .tid  = tso->id
 +#endif
 +             };
 +    if (iomgr->overflow_sqe_q_hd == NULL) {
 +        iomgr->overflow_sqe_q_hd = entry;
 +    } else {
 +        iomgr->overflow_sqe_q_tl->next = entry;
 +    }
 +    iomgr->overflow_sqe_q_tl = entry;
 +}
++
++
 +static void dequeueOverflowQueue(CapIOManager *iomgr,
 +                                 StgTSO **ptso, struct io_uring_sqe **psqe)
 +{
 +    /* Remove the TSO and SQE from the head of their respective queues */
 +    StgTSO                  *tso   = iomgr->overflow_tso_q_hd;
 +    struct overflow_sqe_q_t *entry = iomgr->overflow_sqe_q_hd;
++
 +    if (tso == END_TSO_QUEUE) {
 +        //TODO: decide if we need this or if we should assume the queue is
 +        // non-empty
 +        ASSERT(entry == NULL);
 +        *ptso = END_TSO_QUEUE;
 +        *psqe = NULL;
 +    } else {
 +        iomgr->overflow_tso_q_hd = tso->_link;
 +        RELAXED_STORE(&tso->_link, END_TSO_QUEUE);
 +        if (iomgr->overflow_tso_q_hd == END_TSO_QUEUE) {
 +            iomgr->overflow_tso_q_tl = END_TSO_QUEUE;
 +        }
++
 +        iomgr->overflow_sqe_q_hd = entry->next;
 +        if (iomgr->overflow_sqe_q_hd == NULL) {
 +            iomgr->overflow_sqe_q_tl = NULL;
 +        }
++
 +        *ptso = tso;
 +        *psqe = entry->sqe;
 +    }
 +}
++
 +#endif /* IOMGR_ENABLED_URING */
++

rts/posix/URing.h

 +/* -----------------------------------------------------------------------------
 + *
 + * (c) The GHC Team 2021-2023
 + *
 + * An I/O manager based on the Linux io_uring API.
 + *
 + * Prototypes for functions in URing.c
 + *
 + * -------------------------------------------------------------------------*/
++
 +#pragma once
++
 +#include "IOManager.h"
++
 +#include "BeginPrivate.h"
++
 +#if defined(IOMGR_ENABLED_URING)
++
 +void initCapabilityIOManagerURing(Capability *cap, CapIOManager *iomgr);
 +void initCapabilityIOManagerAfterForkURing(Capability *cap, CapIOManager *iomgr);
++
 +/* Synchronous I/O and timer operations */
 +int syncIOWaitReadyURing(Capability *cap, StgTSO *tso,
 +                         IOReadOrWrite rw, int fd);
++
 +int syncIOReadWriteURing(Capability *cap, StgTSO *tso,
 +                         IOReadOrWrite rw, int fd,
 +                         StgClosure *live, void *buf,
 +                         size_t len, off_t off);
++
 +void syncIOCancelURing(Capability *cap, StgTSO *tso);
++
 +/* Asynchronous operations */
 +int asyncIOWaitReadyURing(Capability *cap, StgTSO *tso, StgAsyncIOOp *aiop,
 +                          IOReadOrWrite rw, int fd);
++
 +int asyncIOReadWriteURing(Capability *cap, StgTSO *tso, StgAsyncIOOp *aiop,
 +                          IOReadOrWrite rw, int fd,
 +                          StgClosure *live, void *buf,
 +                          size_t len, off_t off);
++
 +void asyncIOCancelURing(Capability *cap, StgAsyncIOOp *aiop);
++
 +/* Scheduler operations */
 +bool anyPendingTimeoutsOrIOURing(CapIOManager *iomgr);
 +void pollCompletedTimeoutsOrIOURing(Capability *cap);
 +void awaitCompletedTimeoutsOrIOURing(Capability *cap);
++
 +#endif /* IOMGR_ENABLED_URING */
++
 +#include "EndPrivate.h"
++

rts/posix/URing.svg

 +<?xml version="1.0" encoding="UTF-8" standalone="no"?>
 +<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.0//EN" "http://www.w3.org/TR/2001/PR-SVG-20010719/DTD/svg10.dtd">
 +<svg width="116cm" height="39cm" viewBox="-281 18 2304 768" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink">
 +  <g>
 +    <rect style="fill: #ffffff" x="-200" y="20" width="158.525" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-200" y="20" width="158.525" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120.738" y="42.8812">
 +      <tspan x="-120.738" y="42.8812">Primop: async</tspan>
 +      <tspan x="-120.738" y="58.8812">non-blocking I/O</tspan>
 +      <tspan x="-120.738" y="74.8812">submission</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120.738" y="55">
 +    <tspan x="-120.738" y="55"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-120.738" y1="90" x2="-120.224" y2="112.267"/>
 +    <polygon style="fill: #000000" points="-120.052,119.765 -125.281,109.882 -120.224,112.267 -115.283,109.652 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-120.052,119.765 -125.281,109.882 -120.224,112.267 -115.283,109.652 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="-200" y="122" width="160" height="54"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-200" y="122" width="160" height="54"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="144.881">
 +      <tspan x="-120" y="144.881">Allocate AIOP</tspan>
 +      <tspan x="-120" y="160.881">Allocate table index</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="-280" y1="105" x2="180" y2="105"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-280" y="98.6222">
 +    <tspan x="-280" y="98.6222">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-280" y="121.539">
 +    <tspan x="-280" y="121.539">RTS C</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="0" y="25" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="0" y="25" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="58.8813">
 +      <tspan x="80" y="58.8813">Primop: GC and retry</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="55">
 +    <tspan x="80" y="55"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="0" y1="55" x2="-31.7389" y2="55"/>
 +    <polygon style="fill: #000000" points="-39.2389,55 -29.2389,50 -31.7389,55 -29.2389,60 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-39.2389,55 -29.2389,50 -31.7389,55 -29.2389,60 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="149">
 +    <tspan x="-120" y="149"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-20" y="156.981">
 +    <tspan x="-20" y="156.981">mem alloc failure?</tspan>
 +  </text>
 +  <g>
 +    <polygon style="fill: #ffffff" points="-120,262 -40,322.81 -120,383.619 -200,322.81 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-120,262 -40,322.81 -120,383.619 -200,322.81 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="310.691">
 +      <tspan x="-120" y="310.691">inflight</tspan>
 +      <tspan x="-120" y="326.691">within</tspan>
 +      <tspan x="-120" y="342.691">limit?</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="-200" y="402" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-200" y="402" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="427.881">
 +      <tspan x="-120" y="427.881">Allocate SQE on ring</tspan>
 +      <tspan x="-120" y="443.881">Inc prep counter</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-80" y="386.431">
 +    <tspan x="-80" y="386.431">Yes</tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-20" y="346.431">
 +    <tspan x="-20" y="346.431">No</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="-12.825" y="395" width="185.65" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-12.825" y="395" width="185.65" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="417.881">
 +      <tspan x="80" y="417.881">Allocate SQE on heap</tspan>
 +      <tspan x="80" y="433.881">Inc overflow counter</tspan>
 +      <tspan x="80" y="449.881">TSO &amp; SQE on overflow Q</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="-201.625" y="202" width="163.25" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-201.625" y="202" width="163.25" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="225.881">
 +      <tspan x="-120" y="225.881">Inc submitted counter</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-120" y1="384.619" x2="-120" y2="392.264"/>
 +    <polygon style="fill: #000000" points="-120,399.764 -125,389.764 -120,392.264 -115,389.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-120,399.764 -125,389.764 -120,392.264 -115,389.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-120" y1="176" x2="-120" y2="192.264"/>
 +    <polygon style="fill: #000000" points="-120,199.764 -125,189.764 -120,192.264 -115,189.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-120,199.764 -125,189.764 -120,192.264 -115,189.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-120" y1="242" x2="-120" y2="252.264"/>
 +    <polygon style="fill: #000000" points="-120,259.764 -125,249.764 -120,252.264 -115,249.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-120,259.764 -125,249.764 -120,252.264 -115,249.764 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-40,149 80,125 80,94.7361 "/>
 +    <polygon style="fill: #000000" points="80,87.2361 85,97.2361 80,94.7361 75,97.2361 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="80,87.2361 85,97.2361 80,94.7361 75,97.2361 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="430">
 +    <tspan x="80" y="430"></tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="-100" y="482" width="160" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-100" y="482" width="160" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-20" y="505.881">
 +      <tspan x="-20" y="505.881">Fill in SQE</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-120" y1="462" x2="-106.884" y2="475.116"/>
 +    <polygon style="fill: #000000" points="-101.581,480.419 -112.188,476.883 -106.884,475.116 -105.117,469.812 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-101.581,480.419 -112.188,476.883 -106.884,475.116 -105.117,469.812 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="80" y1="465" x2="67.4183" y2="475.694"/>
 +    <polygon style="fill: #000000" points="61.7037,480.552 66.0849,470.266 67.4183,475.694 72.5614,477.885 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="61.7037,480.552 66.0849,470.266 67.4183,475.694 72.5614,477.885 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="-195.75" y="677" width="157.15" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="-195.75" y="677" width="157.15" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-117.175" y="699.881">
 +      <tspan x="-117.175" y="699.881">Primop:</tspan>
 +      <tspan x="-117.175" y="715.881">result is AIOP</tspan>
 +      <tspan x="-117.175" y="731.881">control to caller</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="0" y="677" width="160" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="0" y="677" width="160" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="699.881">
 +      <tspan x="80" y="699.881">Primop:</tspan>
 +      <tspan x="80" y="715.881">result is AIOP</tspan>
 +      <tspan x="80" y="731.881">control to scheduler</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="-280" y1="662" x2="180" y2="662"/>
 +  <g>
 +    <polygon style="fill: #ffffff" points="-20,542 120,592 -20,642 -160,592 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-20,542 120,592 -20,642 -160,592 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="-20" y="587.881">
 +      <tspan x="-20" y="587.881">inflight within limit</tspan>
 +      <tspan x="-20" y="603.881">And SQ ring not full</tspan>
 +    </text>
 +  </g>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-280" y="682">
 +    <tspan x="-280" y="682">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-280" y="655.622">
 +    <tspan x="-280" y="655.622">RTS C</tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-20" y1="522.977" x2="-20" y2="532.264"/>
 +    <polygon style="fill: #000000" points="-20,539.764 -25,529.764 -20,532.264 -15,529.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-20,539.764 -25,529.764 -20,532.264 -15,529.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="-55" y1="629.5" x2="-73.6614" y2="668.229"/>
 +    <polygon style="fill: #000000" points="-76.917,674.986 -77.0805,663.806 -73.6614,668.229 -68.0718,668.147 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-76.917,674.986 -77.0805,663.806 -73.6614,668.229 -68.0718,668.147 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="15" y1="629.5" x2="35.4655" y2="668.384"/>
 +    <polygon style="fill: #000000" points="38.9586,675.021 29.8765,668.501 35.4655,668.384 38.7257,663.843 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="38.9586,675.021 29.8765,668.501 35.4655,668.384 38.7257,663.843 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-100" y="646.431">
 +    <tspan x="-100" y="646.431">Yes</tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="40" y="646.431">
 +    <tspan x="40" y="646.431">No</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1019.34" y="25" width="163.85" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1019.34" y="25" width="163.85" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1101.26" y="47.8812">
 +      <tspan x="1101.26" y="47.8812">Primop: async</tspan>
 +      <tspan x="1101.26" y="63.8812">blocking I/O</tspan>
 +      <tspan x="1101.26" y="79.8812">submission</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1101.26" y="60">
 +    <tspan x="1101.26" y="60"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1101.26" y1="95" x2="1101.76" y2="115.267"/>
 +    <polygon style="fill: #000000" points="1101.95,122.765 1096.7,112.89 1101.76,115.267 1106.7,112.645 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1101.95,122.765 1096.7,112.89 1101.76,115.267 1106.7,112.645 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1022" y="125" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1022" y="125" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="150.881">
 +      <tspan x="1102" y="150.881">Allocate AIOP</tspan>
 +      <tspan x="1102" y="166.881">Allocate table index</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="942" y1="110" x2="1402" y2="110"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="942" y="103.622">
 +    <tspan x="942" y="103.622">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="942" y="126.539">
 +    <tspan x="942" y="126.539">RTS C</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1222" y="30" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1222" y="30" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1302" y="63.8813">
 +      <tspan x="1302" y="63.8813">Primop: GC and retry</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1302" y="60">
 +    <tspan x="1302" y="60"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1222" y1="60" x2="1192.92" y2="60"/>
 +    <polygon style="fill: #000000" points="1185.42,60 1195.42,55 1192.92,60 1195.42,65 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1185.42,60 1195.42,55 1192.92,60 1195.42,65 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="155">
 +    <tspan x="1102" y="155"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1222" y="161.981">
 +    <tspan x="1222" y="161.981">mem alloc failure?</tspan>
 +  </text>
 +  <g>
 +    <polygon style="fill: #ffffff" points="1102,285 1182,345.81 1102,406.619 1022,345.81 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1102,285 1182,345.81 1102,406.619 1022,345.81 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="333.691">
 +      <tspan x="1102" y="333.691">inflight</tspan>
 +      <tspan x="1102" y="349.691">within</tspan>
 +      <tspan x="1102" y="365.691">limit?</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1022" y="440" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1022" y="440" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="465.881">
 +      <tspan x="1102" y="465.881">Allocate SQE on ring</tspan>
 +      <tspan x="1102" y="481.881">Inc ring prep counter</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1042" y="409.431">
 +    <tspan x="1042" y="409.431">Yes</tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1142" y="409.431">
 +    <tspan x="1142" y="409.431">No</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1020.37" y="205" width="163.25" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1020.37" y="205" width="163.25" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="228.881">
 +      <tspan x="1102" y="228.881">Inc submitted counter</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1102" y1="406.62" x2="1102" y2="430.264"/>
 +    <polygon style="fill: #000000" points="1102,437.764 1097,427.764 1102,430.264 1107,427.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1102,437.764 1097,427.764 1102,430.264 1107,427.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1102" y1="185" x2="1102" y2="195.264"/>
 +    <polygon style="fill: #000000" points="1102,202.764 1097,192.764 1102,195.264 1107,192.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1102,202.764 1097,192.764 1102,195.264 1107,192.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1102" y1="245" x2="1102" y2="275.264"/>
 +    <polygon style="fill: #000000" points="1102,282.764 1097,272.764 1102,275.264 1107,272.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1102,282.764 1097,272.764 1102,275.264 1107,272.764 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1182,155 1302,130 1302,99.7361 "/>
 +    <polygon style="fill: #000000" points="1302,92.2361 1307,102.236 1302,99.7361 1297,102.236 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1302,92.2361 1307,102.236 1302,99.7361 1297,102.236 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1202" y="455">
 +    <tspan x="1202" y="455"></tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1022" y="525" width="160" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1022" y="525" width="160" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="548.881">
 +      <tspan x="1102" y="548.881">Fill in SQE</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1102" y1="500" x2="1102" y2="515.264"/>
 +    <polygon style="fill: #000000" points="1102,522.764 1097,512.764 1102,515.264 1107,512.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1102,522.764 1097,512.764 1102,515.264 1107,512.764 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1022" y="716" width="160" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1022" y="716" width="160" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="738.881">
 +      <tspan x="1102" y="738.881">Primop:</tspan>
 +      <tspan x="1102" y="754.881">result is AIOP</tspan>
 +      <tspan x="1102" y="770.881">control to caller</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1222" y="717" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1222" y="717" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1302" y="742.881">
 +      <tspan x="1302" y="742.881">Primop:</tspan>
 +      <tspan x="1302" y="758.881">throw exception</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="942" y1="697" x2="1402" y2="697"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="942" y="717">
 +    <tspan x="942" y="717">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="942" y="690.622">
 +    <tspan x="942" y="690.622">RTS C</tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1102.53" y1="565.994" x2="1102.86" y2="579.104"/>
 +    <polygon style="fill: #000000" points="1103.05,586.601 1097.8,576.73 1102.86,579.104 1107.79,576.479 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1103.05,586.601 1097.8,576.73 1102.86,579.104 1107.79,576.479 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1182,345.81 1182,345 1302,345 1302,707.264 "/>
 +    <polygon style="fill: #000000" points="1302,714.764 1297,704.764 1302,707.264 1307,704.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1302,714.764 1297,704.764 1302,707.264 1307,704.764 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="-40,322.81 -40,322 80,322 80,385.264 "/>
 +    <polygon style="fill: #000000" points="80,392.764 75,382.764 80,385.264 85,382.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="80,392.764 75,382.764 80,385.264 85,382.764 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="403" y="22" width="158.525" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="403" y="22" width="158.525" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="482.262" y="44.8813">
 +      <tspan x="482.262" y="44.8813">Primop: sync</tspan>
 +      <tspan x="482.262" y="60.8812">non-blocking I/O</tspan>
 +      <tspan x="482.262" y="76.8812">submission</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="482.262" y="57">
 +    <tspan x="482.262" y="57"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="482.262" y1="92" x2="482.795" y2="117.266"/>
 +    <polygon style="fill: #000000" points="482.953,124.764 477.743,114.872 482.795,117.266 487.741,114.661 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="482.953,124.764 477.743,114.872 482.795,117.266 487.741,114.661 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="403" y="127" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="403" y="127" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="152.881">
 +      <tspan x="483" y="152.881">Allocate AIOP</tspan>
 +      <tspan x="483" y="168.881">Allocate table index</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="323" y1="107" x2="783" y2="107"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="323" y="100.622">
 +    <tspan x="323" y="100.622">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="323" y="123.539">
 +    <tspan x="323" y="123.539">RTS C</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="603" y="27" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="603" y="27" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="683" y="60.8812">
 +      <tspan x="683" y="60.8812">Primop: GC and retry</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="683" y="57">
 +    <tspan x="683" y="57"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="603" y1="57" x2="571.262" y2="57"/>
 +    <polygon style="fill: #000000" points="563.762,57 573.762,52 571.262,57 573.762,62 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="563.762,57 573.762,52 571.262,57 573.762,62 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="157">
 +    <tspan x="483" y="157"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="583" y="163.981">
 +    <tspan x="583" y="163.981">mem alloc failure?</tspan>
 +  </text>
 +  <g>
 +    <polygon style="fill: #ffffff" points="483,262 563,322.81 483,383.619 403,322.81 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="483,262 563,322.81 483,383.619 403,322.81 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="310.691">
 +      <tspan x="483" y="310.691">inflight</tspan>
 +      <tspan x="483" y="326.691">within</tspan>
 +      <tspan x="483" y="342.691">limit?</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="403" y="402" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="403" y="402" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="427.881">
 +      <tspan x="483" y="427.881">Allocate SQE on ring</tspan>
 +      <tspan x="483" y="443.881">Inc ring prep counter</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="523" y="386.431">
 +    <tspan x="523" y="386.431">Yes</tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="583" y="346.431">
 +    <tspan x="583" y="346.431">No</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="603" y="402" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="603" y="402" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="683" y="427.881">
 +      <tspan x="683" y="427.881">Allocate SQE on heap</tspan>
 +      <tspan x="683" y="443.881">TSO on overflow Q</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="401.375" y="207" width="163.25" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="401.375" y="207" width="163.25" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="230.881">
 +      <tspan x="483" y="230.881">Inc submitted counter</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="483" y1="384.619" x2="483" y2="392.264"/>
 +    <polygon style="fill: #000000" points="483,399.764 478,389.764 483,392.264 488,389.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="483,399.764 478,389.764 483,392.264 488,389.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="483" y1="187" x2="483" y2="197.264"/>
 +    <polygon style="fill: #000000" points="483,204.764 478,194.764 483,197.264 488,194.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="483,204.764 478,194.764 483,197.264 488,194.764 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="483" y1="247" x2="483" y2="252.264"/>
 +    <polygon style="fill: #000000" points="483,259.764 478,249.764 483,252.264 488,249.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="483,259.764 478,249.764 483,252.264 488,249.764 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="563,157 683,127 683,96.7361 "/>
 +    <polygon style="fill: #000000" points="683,89.2361 688,99.2361 683,96.7361 678,99.2361 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="683,89.2361 688,99.2361 683,96.7361 678,99.2361 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="683" y="432">
 +    <tspan x="683" y="432"></tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="503" y="482" width="160" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="503" y="482" width="160" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="583" y="505.881">
 +      <tspan x="583" y="505.881">Fill in SQE</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="483" y1="462" x2="496.116" y2="475.116"/>
 +    <polygon style="fill: #000000" points="501.419,480.419 490.812,476.883 496.116,475.116 497.883,469.812 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="501.419,480.419 490.812,476.883 496.116,475.116 497.883,469.812 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="683" y1="462" x2="669.884" y2="475.116"/>
 +    <polygon style="fill: #000000" points="664.581,480.419 668.117,469.812 669.884,475.116 675.188,476.883 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="664.581,480.419 668.117,469.812 669.884,475.116 675.188,476.883 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="503" y="562" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="503" y="562" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="583" y="587.881">
 +      <tspan x="583" y="587.881">Primop:</tspan>
 +      <tspan x="583" y="603.881">control to scheduler</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="323" y1="542" x2="783" y2="542"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="323" y="562">
 +    <tspan x="323" y="562">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="323" y="535.622">
 +    <tspan x="323" y="535.622">RTS C</tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="583" y1="522.991" x2="583" y2="552.264"/>
 +    <polygon style="fill: #000000" points="583,559.764 578,549.764 583,552.264 588,549.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="583,559.764 578,549.764 583,552.264 588,549.764 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="563,322.81 563,322 683,322 683,392.264 "/>
 +    <polygon style="fill: #000000" points="683,399.764 678,389.764 683,392.264 688,389.764 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="683,399.764 678,389.764 683,392.264 688,389.764 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1638.85" y="38.6" width="163.85" height="70"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1638.85" y="38.6" width="163.85" height="70"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1720.77" y="61.4813">
 +      <tspan x="1720.77" y="61.4813">Primop: sync</tspan>
 +      <tspan x="1720.77" y="77.4813">blocking I/O</tspan>
 +      <tspan x="1720.77" y="93.4812">submission</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1720.77" y="73.6">
 +    <tspan x="1720.77" y="73.6"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1720.77" y1="108.6" x2="1721.27" y2="128.867"/>
 +    <polygon style="fill: #000000" points="1721.46,136.365 1716.21,126.491 1721.27,128.867 1726.21,126.245 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.46,136.365 1716.21,126.491 1721.27,128.867 1726.21,126.245 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1641.51" y="138.6" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1641.51" y="138.6" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="164.481">
 +      <tspan x="1721.51" y="164.481">Allocate AIOP</tspan>
 +      <tspan x="1721.51" y="180.481">Allocate table index</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="1561.51" y1="123.6" x2="2021.52" y2="123.6"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1561.51" y="117.222">
 +    <tspan x="1561.51" y="117.222">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1561.51" y="140.139">
 +    <tspan x="1561.51" y="140.139">RTS C</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1841.51" y="43.6" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1841.51" y="43.6" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1921.51" y="77.4813">
 +      <tspan x="1921.51" y="77.4813">Primop: GC and retry</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1921.51" y="73.6">
 +    <tspan x="1921.51" y="73.6"></tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1841.51" y1="73.6" x2="1812.44" y2="73.6"/>
 +    <polygon style="fill: #000000" points="1804.94,73.6 1814.94,68.6 1812.44,73.6 1814.94,78.6 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1804.94,73.6 1814.94,68.6 1812.44,73.6 1814.94,78.6 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="168.6">
 +    <tspan x="1721.51" y="168.6"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1841.51" y="175.581">
 +    <tspan x="1841.51" y="175.581">mem alloc failure?</tspan>
 +  </text>
 +  <g>
 +    <polygon style="fill: #ffffff" points="1721.51,298.6 1801.51,359.41 1721.51,420.219 1641.51,359.41 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,298.6 1801.51,359.41 1721.51,420.219 1641.51,359.41 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="347.291">
 +      <tspan x="1721.51" y="347.291">inflight</tspan>
 +      <tspan x="1721.51" y="363.291">within</tspan>
 +      <tspan x="1721.51" y="379.291">limit?</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1641.51" y="453.6" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1641.51" y="453.6" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="479.481">
 +      <tspan x="1721.51" y="479.481">Allocate SQE on ring</tspan>
 +      <tspan x="1721.51" y="495.481">Inc ring prep counter</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1661.51" y="423.031">
 +    <tspan x="1661.51" y="423.031">Yes</tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1761.51" y="423.031">
 +    <tspan x="1761.51" y="423.031">No</tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1639.89" y="218.6" width="163.25" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1639.89" y="218.6" width="163.25" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="242.481">
 +      <tspan x="1721.51" y="242.481">Inc submitted counter</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1721.51" y1="420.22" x2="1721.51" y2="443.864"/>
 +    <polygon style="fill: #000000" points="1721.51,451.364 1716.51,441.364 1721.51,443.864 1726.51,441.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,451.364 1716.51,441.364 1721.51,443.864 1726.51,441.364 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1721.51" y1="198.6" x2="1721.51" y2="208.864"/>
 +    <polygon style="fill: #000000" points="1721.51,216.364 1716.51,206.364 1721.51,208.864 1726.51,206.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,216.364 1716.51,206.364 1721.51,208.864 1726.51,206.364 "/>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1721.51" y1="258.6" x2="1721.51" y2="288.864"/>
 +    <polygon style="fill: #000000" points="1721.51,296.364 1716.51,286.364 1721.51,288.864 1726.51,286.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,296.364 1716.51,286.364 1721.51,288.864 1726.51,286.364 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1801.51,168.6 1921.51,143.6 1921.51,113.336 "/>
 +    <polygon style="fill: #000000" points="1921.51,105.836 1926.51,115.836 1921.51,113.336 1916.51,115.836 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1921.51,105.836 1926.51,115.836 1921.51,113.336 1916.51,115.836 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1821.51" y="468.6">
 +    <tspan x="1821.51" y="468.6"></tspan>
 +  </text>
 +  <g>
 +    <rect style="fill: #ffffff" x="1641.51" y="538.6" width="160" height="40"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1641.51" y="538.6" width="160" height="40"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="562.481">
 +      <tspan x="1721.51" y="562.481">Fill in SQE</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1721.51" y1="513.6" x2="1721.51" y2="528.864"/>
 +    <polygon style="fill: #000000" points="1721.51,536.364 1716.51,526.364 1721.51,528.864 1726.51,526.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,536.364 1716.51,526.364 1721.51,528.864 1726.51,526.364 "/>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1641.51" y="618.6" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1641.51" y="618.6" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1721.51" y="644.481">
 +      <tspan x="1721.51" y="644.481">Primop:</tspan>
 +      <tspan x="1721.51" y="660.481">return to scheduler</tspan>
 +    </text>
 +  </g>
 +  <g>
 +    <rect style="fill: #ffffff" x="1841.51" y="618.6" width="160" height="60"/>
 +    <rect style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x="1841.51" y="618.6" width="160" height="60"/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1921.51" y="644.481">
 +      <tspan x="1921.51" y="644.481">Primop:</tspan>
 +      <tspan x="1921.51" y="660.481">throw exception</tspan>
 +    </text>
 +  </g>
 +  <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke-dasharray: 4; stroke: #000000" x1="1561.51" y1="598.6" x2="2021.52" y2="598.6"/>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1561.51" y="618.6">
 +    <tspan x="1561.51" y="618.6">RTS CMM</tspan>
 +  </text>
 +  <text font-size="14.6756" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1561.51" y="592.222">
 +    <tspan x="1561.51" y="592.222">RTS C</tspan>
 +  </text>
 +  <g>
 +    <line style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" x1="1721.51" y1="579.591" x2="1721.51" y2="608.864"/>
 +    <polygon style="fill: #000000" points="1721.51,616.364 1716.51,606.364 1721.51,608.864 1726.51,606.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1721.51,616.364 1716.51,606.364 1721.51,608.864 1726.51,606.364 "/>
 +  </g>
 +  <g>
 +    <polyline style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1801.51,359.41 1801.51,358.6 1921.51,358.6 1921.51,608.864 "/>
 +    <polygon style="fill: #000000" points="1921.51,616.364 1916.51,606.364 1921.51,608.864 1926.51,606.364 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1921.51,616.364 1916.51,606.364 1921.51,608.864 1926.51,606.364 "/>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120.738" y="55">
 +    <tspan x="-120.738" y="55"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-120" y="149">
 +    <tspan x="-120" y="149"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-117.175" y="712">
 +    <tspan x="-117.175" y="712"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="-117.175" y="712">
 +    <tspan x="-117.175" y="712"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="583" y="592">
 +    <tspan x="583" y="592"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="80" y="712">
 +    <tspan x="80" y="712"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1102" y="751">
 +    <tspan x="1102" y="751"></tspan>
 +  </text>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1720.77" y="73.6">
 +    <tspan x="1720.77" y="73.6"></tspan>
 +  </text>
 +  <g>
 +    <polygon style="fill: #ffffff" points="1104,589.546 1208.51,624.256 1104,658.967 999.486,624.256 "/>
 +    <polygon style="fill: none; fill-opacity:0; stroke-width: 2; stroke: #000000" points="1104,589.546 1208.51,624.256 1104,658.967 999.486,624.256 "/>
 +    <text font-size="12.8" style="fill: #000000;text-anchor:middle;font-family:sans-serif;font-style:normal;font-weight:normal" x="1104" y="628.138">
 +      <tspan x="1104" y="628.138">SQ ring not full</tspan>
 +    </text>
 +  </g>
 +  <text font-size="12.8" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="1104" y="624.256">
 +    <tspan x="1104" y="624.256"></tspan>
 +  </text>
 +  <text font-size="12.7998" style="fill: #000000;text-anchor:start;font-family:sans-serif;font-style:normal;font-weight:normal" x="483" y="227">
 +    <tspan x="483" y="227"></tspan>
 +  </text>
 +</svg>

rts/rts.cabal

@@ -48,6 +48,8 @@ flag libdw
    default: False
  flag libnuma
    default: False
 +flag liburing
 +  default: True
  flag libzstd
    default: False
  flag static-libzstd
@@ -231,6 +233,8 @@ library
           extra-libraries: elf dw
        if flag(libnuma)
           extra-libraries: numa
 +      if flag(liburing)
 +         extra-libraries: uring
        if flag(libzstd)
           if flag(static-libzstd)
              if os(darwin)
@@ -536,6 +540,7 @@ library
                      posix/Signals.c
                      posix/Timeout.c
                      posix/TTY.c
 +                    posix/URing.c
                      -- ticker/*.c
                      -- We don't want to compile posix/ticker/*.c, these will be #included
                      -- from Ticker.c