drbd: Implemented two new connection states Ahead/Behind

In this connection mode, the ahead node no longer replicates
application IO. The behind's disk becomes out dated.
Signed-off-by: Philipp Reisner <philipp.reisner@linbit.com>
Signed-off-by: Lars Ellenberg <lars.ellenberg@linbit.com>

drivers/block/drbd/drbd_int.h

@@ -2217,6 +2217,8 @@ static inline int drbd_state_is_stable(union drbd_state s)
 	case C_VERIFY_T:
 	case C_PAUSED_SYNC_S:
 	case C_PAUSED_SYNC_T:
+	case C_AHEAD:
+	case C_BEHIND:
 		/* maybe stable, look at the disk state */
 		break;
 

drivers/block/drbd/drbd_main.c

@@ -871,16 +871,19 @@ static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state
 
 	if (ns.conn >= C_CONNECTED &&
 	    ((ns.disk == D_CONSISTENT || ns.disk == D_OUTDATED) ||
-	     (ns.disk == D_NEGOTIATING && ns.conn == C_WF_BITMAP_T))) {
+	     (ns.disk == D_NEGOTIATING && ns.conn == C_WF_BITMAP_T) ||
+	     ns.conn >= C_AHEAD)) {
 		switch (ns.conn) {
 		case C_WF_BITMAP_T:
 		case C_PAUSED_SYNC_T:
+		case C_BEHIND:
 			ns.disk = D_OUTDATED;
 			break;
 		case C_CONNECTED:
 		case C_WF_BITMAP_S:
 		case C_SYNC_SOURCE:
 		case C_PAUSED_SYNC_S:
+		case C_AHEAD:
 			ns.disk = D_UP_TO_DATE;
 			break;
 		case C_SYNC_TARGET:
@@ -893,16 +896,18 @@ static union drbd_state sanitize_state(struct drbd_conf *mdev, union drbd_state
 	}
 
 	if (ns.conn >= C_CONNECTED &&
-	    (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED)) {
+	    (ns.pdsk == D_CONSISTENT || ns.pdsk == D_OUTDATED || ns.conn >= C_AHEAD)) {
 		switch (ns.conn) {
 		case C_CONNECTED:
 		case C_WF_BITMAP_T:
 		case C_PAUSED_SYNC_T:
 		case C_SYNC_TARGET:
+		case C_BEHIND:
 			ns.pdsk = D_UP_TO_DATE;
 			break;
 		case C_WF_BITMAP_S:
 		case C_PAUSED_SYNC_S:
+		case C_AHEAD:
 			/* remap any consistent state to D_OUTDATED,
 			 * but disallow "upgrade" of not even consistent states.
 			 */
@@ -1374,6 +1379,9 @@ static void after_state_ch(struct drbd_conf *mdev, union drbd_state os,
 	if (os.conn == C_WF_REPORT_PARAMS && ns.conn >= C_CONNECTED)
 		drbd_send_state(mdev);
 
+	if (os.conn != C_AHEAD && ns.conn == C_AHEAD)
+		drbd_send_state(mdev);
+
 	/* We are in the progress to start a full sync... */
 	if ((os.conn != C_STARTING_SYNC_T && ns.conn == C_STARTING_SYNC_T) ||
 	    (os.conn != C_STARTING_SYNC_S && ns.conn == C_STARTING_SYNC_S))

drivers/block/drbd/drbd_receiver.c

@@ -3179,6 +3179,9 @@ static int receive_state(struct drbd_conf *mdev, enum drbd_packets cmd, unsigned
 	if (ns.conn == C_WF_REPORT_PARAMS)
 		ns.conn = C_CONNECTED;
 
+	if (peer_state.conn == C_AHEAD)
+		ns.conn = C_BEHIND;
+
 	if (mdev->p_uuid && peer_state.disk >= D_NEGOTIATING &&
 	    get_ldev_if_state(mdev, D_NEGOTIATING)) {
 		int cr; /* consider resync */

drivers/block/drbd/drbd_req.c

@@ -948,6 +948,29 @@ static int drbd_make_request_common(struct drbd_conf *mdev, struct bio *bio)
 				? queue_for_net_write
 				: queue_for_net_read);
 	}
+
+	if (remote && mdev->net_conf->on_congestion != OC_BLOCK) {
+		int congested = 0;
+
+		if (mdev->net_conf->cong_fill &&
+		    atomic_read(&mdev->ap_in_flight) >= mdev->net_conf->cong_fill) {
+			dev_info(DEV, "Congestion-fill threshold reached\n");
+			congested = 1;
+		}
+
+		if (mdev->act_log->used >= mdev->net_conf->cong_extents) {
+			dev_info(DEV, "Congestion-extents threshold reached\n");
+			congested = 1;
+		}
+
+		if (congested) {
+			if (mdev->net_conf->on_congestion == OC_PULL_AHEAD)
+				_drbd_set_state(_NS(mdev, conn, C_AHEAD), 0, NULL);
+			else  /*mdev->net_conf->on_congestion == OC_DISCONNECT */
+				_drbd_set_state(_NS(mdev, conn, C_DISCONNECTING), 0, NULL);
+		}
+	}
+
 	spin_unlock_irq(&mdev->req_lock);
 	kfree(b); /* if someone else has beaten us to it... */
 

drivers/block/drbd/drbd_strings.c

@@ -48,6 +48,8 @@ static const char *drbd_conn_s_names[] = {
 	[C_PAUSED_SYNC_T]    = "PausedSyncT",
 	[C_VERIFY_S]         = "VerifyS",
 	[C_VERIFY_T]         = "VerifyT",
+	[C_AHEAD]            = "Ahead",
+	[C_BEHIND]           = "Behind",
 };
 
 static const char *drbd_role_s_names[] = {
@@ -92,7 +94,7 @@ static const char *drbd_state_sw_errors[] = {
 const char *drbd_conn_str(enum drbd_conns s)
 {
 	/* enums are unsigned... */
-	return s > C_PAUSED_SYNC_T ? "TOO_LARGE" : drbd_conn_s_names[s];
+	return s > C_BEHIND ? "TOO_LARGE" : drbd_conn_s_names[s];
 }
 
 const char *drbd_role_str(enum drbd_role s)

include/linux/drbd.h

@@ -206,6 +206,10 @@ enum drbd_conns {
 	C_VERIFY_T,
 	C_PAUSED_SYNC_S,
 	C_PAUSED_SYNC_T,
+
+	C_AHEAD,
+	C_BEHIND,
+
 	C_MASK = 31
 };