libtrx: avoid thread usage

- Stop using circular buffers for storing samples
- Use multiple Send/Recv in parallel
- Make polling calls non-blocking
- Fine-tune permissions on MR and QP

include/libtrx/trx_rdma.h

@@ -10,38 +10,32 @@
 #include "libcapulet/rdma_ib.h"
 #include "libcapulet/net_udp.h"
 
-#define TRX_RDMA_BUFFER_SAMPLES 131072
-#define TRX_RDMA_SEND_SAMPLES 4096
+#define TRX_RDMA_MR_COUNT 32
+#define TRX_RDMA_MR_SIZE 4096
 // XXX: hardcoded parameter
 #define TRX_RDMA_CHANNEL_COUNT 2
 
 struct SDRMetadata {
 	uint8_t version;
 	uint16_t sample_count;
-	uint8_t __unused[27];
+	uint8_t __unused[29];
 };
 
 struct SDRMemoryRegion {
 	struct SDRMetadata meta;
-	TRXComplex iq[TRX_RDMA_CHANNEL_COUNT][TRX_RDMA_SEND_SAMPLES];
+	TRXComplex iq[TRX_RDMA_CHANNEL_COUNT][TRX_RDMA_MR_SIZE];
 };
 
 struct SDRContext {
 	struct CapuletRdmaIbContext ib_ctx;
 	struct CapuletNetUdpContext *udp_ctx;
 
-	// Not an RDMA MR buffer, but local buffer used to store samples after
-	// reception before they are dispatched.
-	struct CommonCBBuffer *recv_buffer;
-	struct CommonCBBuffer *send_buffer;
-
-	pthread_mutex_t recv_buffer_mutex;
-	pthread_mutex_t send_buffer_mutex;
-
 	trx_timestamp_t last_timestamp;
 
-	struct ibv_mr *in_mr;
-	struct ibv_mr *out_mr;
+	struct ibv_mr *recv_mrs[TRX_RDMA_MR_COUNT];
+	struct ibv_mr *send_mrs[TRX_RDMA_MR_COUNT];
+
+	uint64_t send_mrs_status;
 
 	char *server_addr;
 	char *device_name;