Commit eca3a04f authored by Linus Torvalds's avatar Linus Torvalds

Merge tag 'dlm-6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm

Pull dlm updates from David Teigland:
 "This fixes some races in the lowcomms startup and shutdown code that
  were found by targeted stress testing that quickly and repeatedly
  joins and leaves lockspaces"

* tag 'dlm-6.3' of git://git.kernel.org/pub/scm/linux/kernel/git/teigland/linux-dlm:
  fs: dlm: remove unnecessary waker_up() calls
  fs: dlm: move state change into else branch
  fs: dlm: remove newline in log_print
  fs: dlm: reduce the shutdown timeout to 5 secs
  fs: dlm: make dlm sequence id more robust
  fs: dlm: wait until all midcomms nodes detect version
  fs: dlm: ignore unexpected non dlm opts msgs
  fs: dlm: bring back previous shutdown handling
  fs: dlm: send FIN ack back in right cases
  fs: dlm: move sending fin message into state change handling
  fs: dlm: don't set stop rx flag after node reset
  fs: dlm: fix race setting stop tx flag
  fs: dlm: be sure to call dlm_send_queue_flush()
  fs: dlm: fix use after free in midcomms commit
  fs: dlm: start midcomms before scand
  fs/dlm: Remove "select SRCU"
  fs: dlm: fix return value check in dlm_memory_init()
parents 885ce487 723b197b
...@@ -4,7 +4,6 @@ menuconfig DLM ...@@ -4,7 +4,6 @@ menuconfig DLM
depends on INET depends on INET
depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n) depends on SYSFS && CONFIGFS_FS && (IPV6 || IPV6=n)
select IP_SCTP select IP_SCTP
select SRCU
help help
A general purpose distributed lock manager for kernel or userspace A general purpose distributed lock manager for kernel or userspace
applications. applications.
......
...@@ -381,23 +381,23 @@ static int threads_start(void) ...@@ -381,23 +381,23 @@ static int threads_start(void)
{ {
int error; int error;
error = dlm_scand_start(); /* Thread for sending/receiving messages for all lockspace's */
error = dlm_midcomms_start();
if (error) { if (error) {
log_print("cannot start dlm_scand thread %d", error); log_print("cannot start dlm midcomms %d", error);
goto fail; goto fail;
} }
/* Thread for sending/receiving messages for all lockspace's */ error = dlm_scand_start();
error = dlm_midcomms_start();
if (error) { if (error) {
log_print("cannot start dlm midcomms %d", error); log_print("cannot start dlm_scand thread %d", error);
goto scand_fail; goto midcomms_fail;
} }
return 0; return 0;
scand_fail: midcomms_fail:
dlm_scand_stop(); dlm_midcomms_stop();
fail: fail:
return error; return error;
} }
...@@ -572,7 +572,7 @@ static int new_lockspace(const char *name, const char *cluster, ...@@ -572,7 +572,7 @@ static int new_lockspace(const char *name, const char *cluster,
spin_lock_init(&ls->ls_rcom_spin); spin_lock_init(&ls->ls_rcom_spin);
get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t)); get_random_bytes(&ls->ls_rcom_seq, sizeof(uint64_t));
ls->ls_recover_status = 0; ls->ls_recover_status = 0;
ls->ls_recover_seq = 0; ls->ls_recover_seq = get_random_u64();
ls->ls_recover_args = NULL; ls->ls_recover_args = NULL;
init_rwsem(&ls->ls_in_recovery); init_rwsem(&ls->ls_in_recovery);
init_rwsem(&ls->ls_recv_active); init_rwsem(&ls->ls_recv_active);
...@@ -820,6 +820,9 @@ static int release_lockspace(struct dlm_ls *ls, int force) ...@@ -820,6 +820,9 @@ static int release_lockspace(struct dlm_ls *ls, int force)
return rv; return rv;
} }
if (ls_count == 1)
dlm_midcomms_version_wait();
dlm_device_deregister(ls); dlm_device_deregister(ls);
if (force < 3 && dlm_user_daemon_available()) if (force < 3 && dlm_user_daemon_available())
......
...@@ -61,6 +61,7 @@ ...@@ -61,6 +61,7 @@
#include "memory.h" #include "memory.h"
#include "config.h" #include "config.h"
#define DLM_SHUTDOWN_WAIT_TIMEOUT msecs_to_jiffies(5000)
#define NEEDED_RMEM (4*1024*1024) #define NEEDED_RMEM (4*1024*1024)
struct connection { struct connection {
...@@ -99,6 +100,7 @@ struct connection { ...@@ -99,6 +100,7 @@ struct connection {
struct connection *othercon; struct connection *othercon;
struct work_struct rwork; /* receive worker */ struct work_struct rwork; /* receive worker */
struct work_struct swork; /* send worker */ struct work_struct swork; /* send worker */
wait_queue_head_t shutdown_wait;
unsigned char rx_leftover_buf[DLM_MAX_SOCKET_BUFSIZE]; unsigned char rx_leftover_buf[DLM_MAX_SOCKET_BUFSIZE];
int rx_leftover; int rx_leftover;
int mark; int mark;
...@@ -282,6 +284,7 @@ static void dlm_con_init(struct connection *con, int nodeid) ...@@ -282,6 +284,7 @@ static void dlm_con_init(struct connection *con, int nodeid)
INIT_WORK(&con->swork, process_send_sockets); INIT_WORK(&con->swork, process_send_sockets);
INIT_WORK(&con->rwork, process_recv_sockets); INIT_WORK(&con->rwork, process_recv_sockets);
spin_lock_init(&con->addrs_lock); spin_lock_init(&con->addrs_lock);
init_waitqueue_head(&con->shutdown_wait);
} }
/* /*
...@@ -790,6 +793,43 @@ static void close_connection(struct connection *con, bool and_other) ...@@ -790,6 +793,43 @@ static void close_connection(struct connection *con, bool and_other)
up_write(&con->sock_lock); up_write(&con->sock_lock);
} }
static void shutdown_connection(struct connection *con, bool and_other)
{
int ret;
if (con->othercon && and_other)
shutdown_connection(con->othercon, false);
flush_workqueue(io_workqueue);
down_read(&con->sock_lock);
/* nothing to shutdown */
if (!con->sock) {
up_read(&con->sock_lock);
return;
}
ret = kernel_sock_shutdown(con->sock, SHUT_WR);
up_read(&con->sock_lock);
if (ret) {
log_print("Connection %p failed to shutdown: %d will force close",
con, ret);
goto force_close;
} else {
ret = wait_event_timeout(con->shutdown_wait, !con->sock,
DLM_SHUTDOWN_WAIT_TIMEOUT);
if (ret == 0) {
log_print("Connection %p shutdown timed out, will force close",
con);
goto force_close;
}
}
return;
force_close:
close_connection(con, false);
}
static struct processqueue_entry *new_processqueue_entry(int nodeid, static struct processqueue_entry *new_processqueue_entry(int nodeid,
int buflen) int buflen)
{ {
...@@ -1488,6 +1528,7 @@ static void process_recv_sockets(struct work_struct *work) ...@@ -1488,6 +1528,7 @@ static void process_recv_sockets(struct work_struct *work)
break; break;
case DLM_IO_EOF: case DLM_IO_EOF:
close_connection(con, false); close_connection(con, false);
wake_up(&con->shutdown_wait);
/* CF_RECV_PENDING cleared */ /* CF_RECV_PENDING cleared */
break; break;
case DLM_IO_RESCHED: case DLM_IO_RESCHED:
...@@ -1695,6 +1736,9 @@ static int work_start(void) ...@@ -1695,6 +1736,9 @@ static int work_start(void)
void dlm_lowcomms_shutdown(void) void dlm_lowcomms_shutdown(void)
{ {
struct connection *con;
int i, idx;
/* stop lowcomms_listen_data_ready calls */ /* stop lowcomms_listen_data_ready calls */
lock_sock(listen_con.sock->sk); lock_sock(listen_con.sock->sk);
listen_con.sock->sk->sk_data_ready = listen_sock.sk_data_ready; listen_con.sock->sk->sk_data_ready = listen_sock.sk_data_ready;
...@@ -1703,29 +1747,20 @@ void dlm_lowcomms_shutdown(void) ...@@ -1703,29 +1747,20 @@ void dlm_lowcomms_shutdown(void)
cancel_work_sync(&listen_con.rwork); cancel_work_sync(&listen_con.rwork);
dlm_close_sock(&listen_con.sock); dlm_close_sock(&listen_con.sock);
flush_workqueue(process_workqueue);
}
void dlm_lowcomms_shutdown_node(int nodeid, bool force)
{
struct connection *con;
int idx;
idx = srcu_read_lock(&connections_srcu); idx = srcu_read_lock(&connections_srcu);
con = nodeid2con(nodeid, 0); for (i = 0; i < CONN_HASH_SIZE; i++) {
if (WARN_ON_ONCE(!con)) { hlist_for_each_entry_rcu(con, &connection_hash[i], list) {
srcu_read_unlock(&connections_srcu, idx); shutdown_connection(con, true);
return; stop_connection_io(con);
} flush_workqueue(process_workqueue);
close_connection(con, true);
flush_work(&con->swork); clean_one_writequeue(con);
stop_connection_io(con); if (con->othercon)
WARN_ON_ONCE(!force && !list_empty(&con->writequeue)); clean_one_writequeue(con->othercon);
close_connection(con, true); allow_connection_io(con);
clean_one_writequeue(con); }
if (con->othercon) }
clean_one_writequeue(con->othercon);
allow_connection_io(con);
srcu_read_unlock(&connections_srcu, idx); srcu_read_unlock(&connections_srcu, idx);
} }
......
...@@ -51,7 +51,7 @@ int __init dlm_memory_init(void) ...@@ -51,7 +51,7 @@ int __init dlm_memory_init(void)
cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback), cb_cache = kmem_cache_create("dlm_cb", sizeof(struct dlm_callback),
__alignof__(struct dlm_callback), 0, __alignof__(struct dlm_callback), 0,
NULL); NULL);
if (!rsb_cache) if (!cb_cache)
goto cb; goto cb;
return 0; return 0;
......
...@@ -146,8 +146,8 @@ ...@@ -146,8 +146,8 @@
/* init value for sequence numbers for testing purpose only e.g. overflows */ /* init value for sequence numbers for testing purpose only e.g. overflows */
#define DLM_SEQ_INIT 0 #define DLM_SEQ_INIT 0
/* 3 minutes wait to sync ending of dlm */ /* 5 seconds wait to sync ending of dlm */
#define DLM_SHUTDOWN_TIMEOUT msecs_to_jiffies(3 * 60 * 1000) #define DLM_SHUTDOWN_TIMEOUT msecs_to_jiffies(5000)
#define DLM_VERSION_NOT_SET 0 #define DLM_VERSION_NOT_SET 0
struct midcomms_node { struct midcomms_node {
...@@ -375,7 +375,7 @@ static int dlm_send_ack(int nodeid, uint32_t seq) ...@@ -375,7 +375,7 @@ static int dlm_send_ack(int nodeid, uint32_t seq)
struct dlm_msg *msg; struct dlm_msg *msg;
char *ppc; char *ppc;
msg = dlm_lowcomms_new_msg(nodeid, mb_len, GFP_NOFS, &ppc, msg = dlm_lowcomms_new_msg(nodeid, mb_len, GFP_ATOMIC, &ppc,
NULL, NULL); NULL, NULL);
if (!msg) if (!msg)
return -ENOMEM; return -ENOMEM;
...@@ -402,10 +402,11 @@ static int dlm_send_fin(struct midcomms_node *node, ...@@ -402,10 +402,11 @@ static int dlm_send_fin(struct midcomms_node *node,
struct dlm_mhandle *mh; struct dlm_mhandle *mh;
char *ppc; char *ppc;
mh = dlm_midcomms_get_mhandle(node->nodeid, mb_len, GFP_NOFS, &ppc); mh = dlm_midcomms_get_mhandle(node->nodeid, mb_len, GFP_ATOMIC, &ppc);
if (!mh) if (!mh)
return -ENOMEM; return -ENOMEM;
set_bit(DLM_NODE_FLAG_STOP_TX, &node->flags);
mh->ack_rcv = ack_rcv; mh->ack_rcv = ack_rcv;
m_header = (struct dlm_header *)ppc; m_header = (struct dlm_header *)ppc;
...@@ -417,7 +418,6 @@ static int dlm_send_fin(struct midcomms_node *node, ...@@ -417,7 +418,6 @@ static int dlm_send_fin(struct midcomms_node *node,
pr_debug("sending fin msg to node %d\n", node->nodeid); pr_debug("sending fin msg to node %d\n", node->nodeid);
dlm_midcomms_commit_mhandle(mh, NULL, 0); dlm_midcomms_commit_mhandle(mh, NULL, 0);
set_bit(DLM_NODE_FLAG_STOP_TX, &node->flags);
return 0; return 0;
} }
...@@ -467,7 +467,7 @@ static void dlm_pas_fin_ack_rcv(struct midcomms_node *node) ...@@ -467,7 +467,7 @@ static void dlm_pas_fin_ack_rcv(struct midcomms_node *node)
break; break;
default: default:
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
log_print("%s: unexpected state: %d\n", log_print("%s: unexpected state: %d",
__func__, node->state); __func__, node->state);
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
return; return;
...@@ -498,18 +498,14 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, ...@@ -498,18 +498,14 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p,
switch (p->header.h_cmd) { switch (p->header.h_cmd) {
case DLM_FIN: case DLM_FIN:
/* send ack before fin */
dlm_send_ack(node->nodeid, node->seq_next);
spin_lock(&node->state_lock); spin_lock(&node->state_lock);
pr_debug("receive fin msg from node %d with state %s\n", pr_debug("receive fin msg from node %d with state %s\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
switch (node->state) { switch (node->state) {
case DLM_ESTABLISHED: case DLM_ESTABLISHED:
node->state = DLM_CLOSE_WAIT; dlm_send_ack(node->nodeid, node->seq_next);
pr_debug("switch node %d to state %s\n",
node->nodeid, dlm_state_str(node->state));
/* passive shutdown DLM_LAST_ACK case 1 /* passive shutdown DLM_LAST_ACK case 1
* additional we check if the node is used by * additional we check if the node is used by
* cluster manager events at all. * cluster manager events at all.
...@@ -518,34 +514,38 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, ...@@ -518,34 +514,38 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p,
node->state = DLM_LAST_ACK; node->state = DLM_LAST_ACK;
pr_debug("switch node %d to state %s case 1\n", pr_debug("switch node %d to state %s case 1\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
spin_unlock(&node->state_lock); set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
goto send_fin; dlm_send_fin(node, dlm_pas_fin_ack_rcv);
} else {
node->state = DLM_CLOSE_WAIT;
pr_debug("switch node %d to state %s\n",
node->nodeid, dlm_state_str(node->state));
} }
break; break;
case DLM_FIN_WAIT1: case DLM_FIN_WAIT1:
dlm_send_ack(node->nodeid, node->seq_next);
node->state = DLM_CLOSING; node->state = DLM_CLOSING;
set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
pr_debug("switch node %d to state %s\n", pr_debug("switch node %d to state %s\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
break; break;
case DLM_FIN_WAIT2: case DLM_FIN_WAIT2:
dlm_send_ack(node->nodeid, node->seq_next);
midcomms_node_reset(node); midcomms_node_reset(node);
pr_debug("switch node %d to state %s\n", pr_debug("switch node %d to state %s\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
wake_up(&node->shutdown_wait);
break; break;
case DLM_LAST_ACK: case DLM_LAST_ACK:
/* probably remove_member caught it, do nothing */ /* probably remove_member caught it, do nothing */
break; break;
default: default:
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
log_print("%s: unexpected state: %d\n", log_print("%s: unexpected state: %d",
__func__, node->state); __func__, node->state);
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
return; return;
} }
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
break; break;
default: default:
WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_RX, &node->flags)); WARN_ON_ONCE(test_bit(DLM_NODE_FLAG_STOP_RX, &node->flags));
...@@ -564,12 +564,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p, ...@@ -564,12 +564,6 @@ static void dlm_midcomms_receive_buffer(union dlm_packet *p,
log_print_ratelimited("ignore dlm msg because seq mismatch, seq: %u, expected: %u, nodeid: %d", log_print_ratelimited("ignore dlm msg because seq mismatch, seq: %u, expected: %u, nodeid: %d",
seq, node->seq_next, node->nodeid); seq, node->seq_next, node->nodeid);
} }
return;
send_fin:
set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
dlm_send_fin(node, dlm_pas_fin_ack_rcv);
} }
static struct midcomms_node * static struct midcomms_node *
...@@ -612,16 +606,8 @@ dlm_midcomms_recv_node_lookup(int nodeid, const union dlm_packet *p, ...@@ -612,16 +606,8 @@ dlm_midcomms_recv_node_lookup(int nodeid, const union dlm_packet *p,
case DLM_ESTABLISHED: case DLM_ESTABLISHED:
break; break;
default: default:
/* some invalid state passive shutdown spin_unlock(&node->state_lock);
* was failed, we try to reset and return NULL;
* hope it will go on.
*/
log_print("reset node %d because shutdown stuck",
node->nodeid);
midcomms_node_reset(node);
node->state = DLM_ESTABLISHED;
break;
} }
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
} }
...@@ -671,6 +657,7 @@ static int dlm_midcomms_version_check_3_2(struct midcomms_node *node) ...@@ -671,6 +657,7 @@ static int dlm_midcomms_version_check_3_2(struct midcomms_node *node)
switch (node->version) { switch (node->version) {
case DLM_VERSION_NOT_SET: case DLM_VERSION_NOT_SET:
node->version = DLM_VERSION_3_2; node->version = DLM_VERSION_3_2;
wake_up(&node->shutdown_wait);
log_print("version 0x%08x for node %d detected", DLM_VERSION_3_2, log_print("version 0x%08x for node %d detected", DLM_VERSION_3_2,
node->nodeid); node->nodeid);
break; break;
...@@ -840,6 +827,7 @@ static int dlm_midcomms_version_check_3_1(struct midcomms_node *node) ...@@ -840,6 +827,7 @@ static int dlm_midcomms_version_check_3_1(struct midcomms_node *node)
switch (node->version) { switch (node->version) {
case DLM_VERSION_NOT_SET: case DLM_VERSION_NOT_SET:
node->version = DLM_VERSION_3_1; node->version = DLM_VERSION_3_1;
wake_up(&node->shutdown_wait);
log_print("version 0x%08x for node %d detected", DLM_VERSION_3_1, log_print("version 0x%08x for node %d detected", DLM_VERSION_3_1,
node->nodeid); node->nodeid);
break; break;
...@@ -1214,8 +1202,15 @@ void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, ...@@ -1214,8 +1202,15 @@ void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh,
dlm_free_mhandle(mh); dlm_free_mhandle(mh);
break; break;
case DLM_VERSION_3_2: case DLM_VERSION_3_2:
/* held rcu read lock here, because we sending the
* dlm message out, when we do that we could receive
* an ack back which releases the mhandle and we
* get a use after free.
*/
rcu_read_lock();
dlm_midcomms_commit_msg_3_2(mh, name, namelen); dlm_midcomms_commit_msg_3_2(mh, name, namelen);
srcu_read_unlock(&nodes_srcu, mh->idx); srcu_read_unlock(&nodes_srcu, mh->idx);
rcu_read_unlock();
break; break;
default: default:
srcu_read_unlock(&nodes_srcu, mh->idx); srcu_read_unlock(&nodes_srcu, mh->idx);
...@@ -1266,7 +1261,6 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node) ...@@ -1266,7 +1261,6 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node)
midcomms_node_reset(node); midcomms_node_reset(node);
pr_debug("switch node %d to state %s\n", pr_debug("switch node %d to state %s\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
wake_up(&node->shutdown_wait);
break; break;
case DLM_CLOSED: case DLM_CLOSED:
/* not valid but somehow we got what we want */ /* not valid but somehow we got what we want */
...@@ -1274,7 +1268,7 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node) ...@@ -1274,7 +1268,7 @@ static void dlm_act_fin_ack_rcv(struct midcomms_node *node)
break; break;
default: default:
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
log_print("%s: unexpected state: %d\n", log_print("%s: unexpected state: %d",
__func__, node->state); __func__, node->state);
WARN_ON_ONCE(1); WARN_ON_ONCE(1);
return; return;
...@@ -1362,11 +1356,11 @@ void dlm_midcomms_remove_member(int nodeid) ...@@ -1362,11 +1356,11 @@ void dlm_midcomms_remove_member(int nodeid)
case DLM_CLOSE_WAIT: case DLM_CLOSE_WAIT:
/* passive shutdown DLM_LAST_ACK case 2 */ /* passive shutdown DLM_LAST_ACK case 2 */
node->state = DLM_LAST_ACK; node->state = DLM_LAST_ACK;
spin_unlock(&node->state_lock);
pr_debug("switch node %d to state %s case 2\n", pr_debug("switch node %d to state %s case 2\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
goto send_fin; set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
dlm_send_fin(node, dlm_pas_fin_ack_rcv);
break;
case DLM_LAST_ACK: case DLM_LAST_ACK:
/* probably receive fin caught it, do nothing */ /* probably receive fin caught it, do nothing */
break; break;
...@@ -1374,7 +1368,7 @@ void dlm_midcomms_remove_member(int nodeid) ...@@ -1374,7 +1368,7 @@ void dlm_midcomms_remove_member(int nodeid)
/* already gone, do nothing */ /* already gone, do nothing */
break; break;
default: default:
log_print("%s: unexpected state: %d\n", log_print("%s: unexpected state: %d",
__func__, node->state); __func__, node->state);
break; break;
} }
...@@ -1382,12 +1376,6 @@ void dlm_midcomms_remove_member(int nodeid) ...@@ -1382,12 +1376,6 @@ void dlm_midcomms_remove_member(int nodeid)
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
srcu_read_unlock(&nodes_srcu, idx); srcu_read_unlock(&nodes_srcu, idx);
return;
send_fin:
set_bit(DLM_NODE_FLAG_STOP_RX, &node->flags);
dlm_send_fin(node, dlm_pas_fin_ack_rcv);
srcu_read_unlock(&nodes_srcu, idx);
} }
static void midcomms_node_release(struct rcu_head *rcu) static void midcomms_node_release(struct rcu_head *rcu)
...@@ -1395,9 +1383,31 @@ static void midcomms_node_release(struct rcu_head *rcu) ...@@ -1395,9 +1383,31 @@ static void midcomms_node_release(struct rcu_head *rcu)
struct midcomms_node *node = container_of(rcu, struct midcomms_node, rcu); struct midcomms_node *node = container_of(rcu, struct midcomms_node, rcu);
WARN_ON_ONCE(atomic_read(&node->send_queue_cnt)); WARN_ON_ONCE(atomic_read(&node->send_queue_cnt));
dlm_send_queue_flush(node);
kfree(node); kfree(node);
} }
void dlm_midcomms_version_wait(void)
{
struct midcomms_node *node;
int i, idx, ret;
idx = srcu_read_lock(&nodes_srcu);
for (i = 0; i < CONN_HASH_SIZE; i++) {
hlist_for_each_entry_rcu(node, &node_hash[i], hlist) {
ret = wait_event_timeout(node->shutdown_wait,
node->version != DLM_VERSION_NOT_SET ||
node->state == DLM_CLOSED ||
test_bit(DLM_NODE_FLAG_CLOSE, &node->flags),
DLM_SHUTDOWN_TIMEOUT);
if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags))
pr_debug("version wait timed out for node %d with state %s\n",
node->nodeid, dlm_state_str(node->state));
}
}
srcu_read_unlock(&nodes_srcu, idx);
}
static void midcomms_shutdown(struct midcomms_node *node) static void midcomms_shutdown(struct midcomms_node *node)
{ {
int ret; int ret;
...@@ -1418,11 +1428,11 @@ static void midcomms_shutdown(struct midcomms_node *node) ...@@ -1418,11 +1428,11 @@ static void midcomms_shutdown(struct midcomms_node *node)
node->state = DLM_FIN_WAIT1; node->state = DLM_FIN_WAIT1;
pr_debug("switch node %d to state %s case 2\n", pr_debug("switch node %d to state %s case 2\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
dlm_send_fin(node, dlm_act_fin_ack_rcv);
break; break;
case DLM_CLOSED: case DLM_CLOSED:
/* we have what we want */ /* we have what we want */
spin_unlock(&node->state_lock); break;
return;
default: default:
/* busy to enter DLM_FIN_WAIT1, wait until passive /* busy to enter DLM_FIN_WAIT1, wait until passive
* done in shutdown_wait to enter DLM_CLOSED. * done in shutdown_wait to enter DLM_CLOSED.
...@@ -1431,29 +1441,20 @@ static void midcomms_shutdown(struct midcomms_node *node) ...@@ -1431,29 +1441,20 @@ static void midcomms_shutdown(struct midcomms_node *node)
} }
spin_unlock(&node->state_lock); spin_unlock(&node->state_lock);
if (node->state == DLM_FIN_WAIT1) { if (DLM_DEBUG_FENCE_TERMINATION)
dlm_send_fin(node, dlm_act_fin_ack_rcv); msleep(5000);
if (DLM_DEBUG_FENCE_TERMINATION)
msleep(5000);
}
/* wait for other side dlm + fin */ /* wait for other side dlm + fin */
ret = wait_event_timeout(node->shutdown_wait, ret = wait_event_timeout(node->shutdown_wait,
node->state == DLM_CLOSED || node->state == DLM_CLOSED ||
test_bit(DLM_NODE_FLAG_CLOSE, &node->flags), test_bit(DLM_NODE_FLAG_CLOSE, &node->flags),
DLM_SHUTDOWN_TIMEOUT); DLM_SHUTDOWN_TIMEOUT);
if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags)) { if (!ret || test_bit(DLM_NODE_FLAG_CLOSE, &node->flags))
pr_debug("active shutdown timed out for node %d with state %s\n", pr_debug("active shutdown timed out for node %d with state %s\n",
node->nodeid, dlm_state_str(node->state)); node->nodeid, dlm_state_str(node->state));
midcomms_node_reset(node); else
dlm_lowcomms_shutdown_node(node->nodeid, true); pr_debug("active shutdown done for node %d with state %s\n",
return; node->nodeid, dlm_state_str(node->state));
}
pr_debug("active shutdown done for node %d with state %s\n",
node->nodeid, dlm_state_str(node->state));
dlm_lowcomms_shutdown_node(node->nodeid, false);
} }
void dlm_midcomms_shutdown(void) void dlm_midcomms_shutdown(void)
...@@ -1461,8 +1462,6 @@ void dlm_midcomms_shutdown(void) ...@@ -1461,8 +1462,6 @@ void dlm_midcomms_shutdown(void)
struct midcomms_node *node; struct midcomms_node *node;
int i, idx; int i, idx;
dlm_lowcomms_shutdown();
mutex_lock(&close_lock); mutex_lock(&close_lock);
idx = srcu_read_lock(&nodes_srcu); idx = srcu_read_lock(&nodes_srcu);
for (i = 0; i < CONN_HASH_SIZE; i++) { for (i = 0; i < CONN_HASH_SIZE; i++) {
...@@ -1480,6 +1479,8 @@ void dlm_midcomms_shutdown(void) ...@@ -1480,6 +1479,8 @@ void dlm_midcomms_shutdown(void)
} }
srcu_read_unlock(&nodes_srcu, idx); srcu_read_unlock(&nodes_srcu, idx);
mutex_unlock(&close_lock); mutex_unlock(&close_lock);
dlm_lowcomms_shutdown();
} }
int dlm_midcomms_close(int nodeid) int dlm_midcomms_close(int nodeid)
......
...@@ -20,6 +20,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len, ...@@ -20,6 +20,7 @@ struct dlm_mhandle *dlm_midcomms_get_mhandle(int nodeid, int len,
gfp_t allocation, char **ppc); gfp_t allocation, char **ppc);
void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name, void dlm_midcomms_commit_mhandle(struct dlm_mhandle *mh, const void *name,
int namelen); int namelen);
void dlm_midcomms_version_wait(void);
int dlm_midcomms_close(int nodeid); int dlm_midcomms_close(int nodeid);
int dlm_midcomms_start(void); int dlm_midcomms_start(void);
void dlm_midcomms_stop(void); void dlm_midcomms_stop(void);
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment