Commit 50729d60 authored by unknown's avatar unknown

dded method to retrieve mysqld, ndbd, and ndb_mgmd aliases for API,MGM and DB

added sanity checks for config file
changed so that ndb_0 is replaced by ndb_pid#### in ndb files
added error_string for to some functions, alloc_node_id(), start()
added better error printout for failed alloc node id
better error printouts for failures when starting ndb_mgmd


ndb/include/mgmapi/mgmapi.h:
  added method to retrieve mysqld, ndbd, and ndb_mgmd aliases for API,MGM and DB
ndb/src/common/mgmcommon/ConfigInfo.cpp:
  added sanity checks for config file
ndb/src/common/mgmcommon/NdbConfig.c:
  changed so that ndb_0 is replaced by ndb_pid#### in ndb files
ndb/src/kernel/error/ErrorReporter.cpp:
  removed usage of tracefile early in startup (when it's normally filled with zeroes)
ndb/src/mgmapi/mgmapi.cpp:
  added method to retrieve mysqld, ndbd, and ndb_mgmd aliases for API,MGM and DB
ndb/src/mgmsrv/MgmtSrvr.cpp:
  added error_string for to some functions, alloc_node_id(), start()
ndb/src/mgmsrv/MgmtSrvr.hpp:
  added error_string for to some functions, alloc_node_id(), start()
ndb/src/mgmsrv/Services.cpp:
  added better error printout for failed alloc node id
ndb/src/mgmsrv/main.cpp:
  better error printouts for failures when starting ndb_mgmd
parent eb72c28b
...@@ -394,6 +394,14 @@ extern "C" { ...@@ -394,6 +394,14 @@ extern "C" {
*/ */
const char * ndb_mgm_get_node_type_string(enum ndb_mgm_node_type type); const char * ndb_mgm_get_node_type_string(enum ndb_mgm_node_type type);
/**
* Convert an ndb_mgm_node_type to a alias string
*
* @param type Node type.
* @return NULL if invalid id.
*/
const char * ndb_mgm_get_node_type_alias_string(enum ndb_mgm_node_type type, const char **str);
/** /**
* Convert a string to a ndb_mgm_node_status * Convert a string to a ndb_mgm_node_status
* *
......
...@@ -161,6 +161,9 @@ const int ConfigInfo::m_NoOfRules = sizeof(m_SectionRules)/sizeof(SectionRule); ...@@ -161,6 +161,9 @@ const int ConfigInfo::m_NoOfRules = sizeof(m_SectionRules)/sizeof(SectionRule);
/**************************************************************************** /****************************************************************************
* Config Rules declarations * Config Rules declarations
****************************************************************************/ ****************************************************************************/
static bool sanity_checks(Vector<ConfigInfo::ConfigRuleSection>&sections,
struct InitConfigFileParser::Context &ctx,
const char * rule_data);
static bool add_node_connections(Vector<ConfigInfo::ConfigRuleSection>&sections, static bool add_node_connections(Vector<ConfigInfo::ConfigRuleSection>&sections,
struct InitConfigFileParser::Context &ctx, struct InitConfigFileParser::Context &ctx,
const char * rule_data); const char * rule_data);
...@@ -173,6 +176,7 @@ static bool check_node_vs_replicas(Vector<ConfigInfo::ConfigRuleSection>&section ...@@ -173,6 +176,7 @@ static bool check_node_vs_replicas(Vector<ConfigInfo::ConfigRuleSection>&section
const ConfigInfo::ConfigRule const ConfigInfo::ConfigRule
ConfigInfo::m_ConfigRules[] = { ConfigInfo::m_ConfigRules[] = {
{ sanity_checks, 0 },
{ add_node_connections, 0 }, { add_node_connections, 0 },
{ add_server_ports, 0 }, { add_server_ports, 0 },
{ check_node_vs_replicas, 0 }, { check_node_vs_replicas, 0 },
...@@ -2958,6 +2962,29 @@ saveInConfigValues(InitConfigFileParser::Context & ctx, const char * data){ ...@@ -2958,6 +2962,29 @@ saveInConfigValues(InitConfigFileParser::Context & ctx, const char * data){
return true; return true;
} }
static bool
sanity_checks(Vector<ConfigInfo::ConfigRuleSection>&sections,
struct InitConfigFileParser::Context &ctx,
const char * rule_data)
{
Uint32 db_nodes = 0;
Uint32 mgm_nodes = 0;
Uint32 api_nodes = 0;
if (!ctx.m_userProperties.get("DB", &db_nodes)) {
ctx.reportError("At least one database node should be defined in config file");
return false;
}
if (!ctx.m_userProperties.get("MGM", &mgm_nodes)) {
ctx.reportError("At least one management server node should be defined in config file");
return false;
}
if (!ctx.m_userProperties.get("API", &api_nodes)) {
ctx.reportError("At least one application node (for the mysqld) should be defined in config file");
return false;
}
return true;
}
static bool static bool
add_node_connections(Vector<ConfigInfo::ConfigRuleSection>&sections, add_node_connections(Vector<ConfigInfo::ConfigRuleSection>&sections,
struct InitConfigFileParser::Context &ctx, struct InitConfigFileParser::Context &ctx,
...@@ -3091,7 +3118,9 @@ check_node_vs_replicas(Vector<ConfigInfo::ConfigRuleSection>&sections, ...@@ -3091,7 +3118,9 @@ check_node_vs_replicas(Vector<ConfigInfo::ConfigRuleSection>&sections,
{ {
Uint32 db_nodes = 0; Uint32 db_nodes = 0;
Uint32 replicas = 0; Uint32 replicas = 0;
ctx.m_userProperties.get("DB", &db_nodes); if (!ctx.m_userProperties.get("DB", &db_nodes)) {
return true;
}
ctx.m_userProperties.get("NoOfReplicas", &replicas); ctx.m_userProperties.get("NoOfReplicas", &replicas);
if((db_nodes % replicas) != 0){ if((db_nodes % replicas) != 0){
ctx.reportError("Invalid no of db nodes wrt no of replicas.\n" ctx.reportError("Invalid no of db nodes wrt no of replicas.\n"
......
...@@ -53,58 +53,73 @@ NdbConfig_NdbCfgName(int with_ndb_home){ ...@@ -53,58 +53,73 @@ NdbConfig_NdbCfgName(int with_ndb_home){
return buf; return buf;
} }
static
char *get_prefix_buf(int len, int node_id)
{
char tmp_buf[sizeof("ndb_pid#########")+1];
if (node_id > 0)
snprintf(tmp_buf, sizeof(tmp_buf), "ndb_%u", node_id);
else
snprintf(tmp_buf, sizeof(tmp_buf), "ndb_pid%u", getpid());
tmp_buf[sizeof(tmp_buf)-1]= 0;
char *buf= NdbConfig_AllocHomePath(len+strlen(tmp_buf));
strcat(buf, tmp_buf);
return buf;
}
char* char*
NdbConfig_ErrorFileName(int node_id){ NdbConfig_ErrorFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_error.log", node_id); snprintf(buf+len, 128, "_error.log");
return buf; return buf;
} }
char* char*
NdbConfig_ClusterLogFileName(int node_id){ NdbConfig_ClusterLogFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_cluster.log", node_id); snprintf(buf+len, 128, "_cluster.log");
return buf; return buf;
} }
char* char*
NdbConfig_SignalLogFileName(int node_id){ NdbConfig_SignalLogFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_signal.log", node_id); snprintf(buf+len, 128, "_signal.log");
return buf; return buf;
} }
char* char*
NdbConfig_TraceFileName(int node_id, int file_no){ NdbConfig_TraceFileName(int node_id, int file_no){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_trace.log.%u", node_id, file_no); snprintf(buf+len, 128, "_trace.log.%u", file_no);
return buf; return buf;
} }
char* char*
NdbConfig_NextTraceFileName(int node_id){ NdbConfig_NextTraceFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_trace.log.next", node_id); snprintf(buf+len, 128, "_trace.log.next");
return buf; return buf;
} }
char* char*
NdbConfig_PidFileName(int node_id){ NdbConfig_PidFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u.pid", node_id); snprintf(buf+len, 128, ".pid");
return buf; return buf;
} }
char* char*
NdbConfig_StdoutFileName(int node_id){ NdbConfig_StdoutFileName(int node_id){
char *buf= NdbConfig_AllocHomePath(128); char *buf= get_prefix_buf(128, node_id);
int len= strlen(buf); int len= strlen(buf);
snprintf(buf+len, 128, "ndb_%u_out.log", node_id); snprintf(buf+len, 128, "_out.log");
return buf; return buf;
} }
...@@ -139,7 +139,7 @@ ErrorReporter::formatMessage(ErrorCategory type, ...@@ -139,7 +139,7 @@ ErrorReporter::formatMessage(ErrorCategory type,
objRef, objRef,
programName, programName,
processId, processId,
theNameOfTheTraceFile); theNameOfTheTraceFile ? theNameOfTheTraceFile : "<no tracefile>");
// Add trailing blanks to get a fixed lenght of the message // Add trailing blanks to get a fixed lenght of the message
while (strlen(messptr) <= MESSAGE_LENGTH-3){ while (strlen(messptr) <= MESSAGE_LENGTH-3){
...@@ -217,8 +217,10 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID, ...@@ -217,8 +217,10 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID,
/** /**
* Format trace file name * Format trace file name
*/ */
int file_no= ErrorReporter::get_trace_no(); char *theTraceFileName= 0;
char *theTraceFileName= NdbConfig_TraceFileName(globalData.ownId, file_no); if (globalData.ownId > 0)
theTraceFileName= NdbConfig_TraceFileName(globalData.ownId,
ErrorReporter::get_trace_no());
NdbAutoPtr<char> tmp_aptr1(theTraceFileName); NdbAutoPtr<char> tmp_aptr1(theTraceFileName);
// The first 69 bytes is info about the current offset // The first 69 bytes is info about the current offset
...@@ -291,6 +293,7 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID, ...@@ -291,6 +293,7 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID,
fflush(stream); fflush(stream);
fclose(stream); fclose(stream);
if (theTraceFileName) {
// Open the tracefile... // Open the tracefile...
FILE *jamStream = fopen(theTraceFileName, "w"); FILE *jamStream = fopen(theTraceFileName, "w");
...@@ -310,6 +313,7 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID, ...@@ -310,6 +313,7 @@ WriteMessage(ErrorCategory thrdType, int thrdMessageID,
globalScheduler.dumpSignalMemory(jamStream); globalScheduler.dumpSignalMemory(jamStream);
fclose(jamStream); fclose(jamStream);
}
return 0; return 0;
} }
......
...@@ -403,14 +403,15 @@ ndb_mgm_disconnect(NdbMgmHandle handle) ...@@ -403,14 +403,15 @@ ndb_mgm_disconnect(NdbMgmHandle handle)
struct ndb_mgm_type_atoi struct ndb_mgm_type_atoi
{ {
const char * str; const char * str;
const char * alias;
enum ndb_mgm_node_type value; enum ndb_mgm_node_type value;
}; };
static struct ndb_mgm_type_atoi type_values[] = static struct ndb_mgm_type_atoi type_values[] =
{ {
{ "NDB", NDB_MGM_NODE_TYPE_NDB}, { "NDB", "ndbd", NDB_MGM_NODE_TYPE_NDB},
{ "API", NDB_MGM_NODE_TYPE_API }, { "API", "mysqld", NDB_MGM_NODE_TYPE_API },
{ "MGM", NDB_MGM_NODE_TYPE_MGM } { "MGM", "ndb_mgmd", NDB_MGM_NODE_TYPE_MGM }
}; };
const int no_of_type_values = (sizeof(type_values) / const int no_of_type_values = (sizeof(type_values) /
...@@ -440,6 +441,20 @@ ndb_mgm_get_node_type_string(enum ndb_mgm_node_type type) ...@@ -440,6 +441,20 @@ ndb_mgm_get_node_type_string(enum ndb_mgm_node_type type)
return 0; return 0;
} }
extern "C"
const char *
ndb_mgm_get_node_type_alias_string(enum ndb_mgm_node_type type, const char** str)
{
for(int i = 0; i<no_of_type_values; i++)
if(type_values[i].value == type)
{
if (str)
*str= type_values[i].str;
return type_values[i].alias;
}
return 0;
}
struct ndb_mgm_status_atoi { struct ndb_mgm_status_atoi {
const char * str; const char * str;
enum ndb_mgm_node_status value; enum ndb_mgm_node_status value;
......
...@@ -584,8 +584,10 @@ MgmtSrvr::MgmtSrvr(NodeId nodeId, ...@@ -584,8 +584,10 @@ MgmtSrvr::MgmtSrvr(NodeId nodeId,
_ownNodeId= 0; _ownNodeId= 0;
NodeId tmp= nodeId; NodeId tmp= nodeId;
if (!alloc_node_id(&tmp, NDB_MGM_NODE_TYPE_MGM, 0, 0)){ BaseString error_string;
ndbout << "Unable to obtain requested nodeid " << nodeId; if (!alloc_node_id(&tmp, NDB_MGM_NODE_TYPE_MGM, 0, 0, error_string)){
ndbout << "Unable to obtain requested nodeid: "
<< error_string.c_str() << endl;
exit(-1); exit(-1);
} }
_ownNodeId = tmp; _ownNodeId = tmp;
...@@ -606,17 +608,20 @@ MgmtSrvr::check_start() ...@@ -606,17 +608,20 @@ MgmtSrvr::check_start()
} }
bool bool
MgmtSrvr::start() MgmtSrvr::start(BaseString &error_string)
{ {
if (_props == NULL) { if (_props == NULL) {
if (!check_start()) if (!check_start()) {
error_string.append("MgmtSrvr.cpp: check_start() failed.");
return false; return false;
} }
}
theFacade = TransporterFacade::start_instance theFacade = TransporterFacade::start_instance
(_ownNodeId,(ndb_mgm_configuration*)_config->m_configValues); (_ownNodeId,(ndb_mgm_configuration*)_config->m_configValues);
if(theFacade == 0) { if(theFacade == 0) {
DEBUG("MgmtSrvr.cpp: theFacade is NULL."); DEBUG("MgmtSrvr.cpp: theFacade is NULL.");
error_string.append("MgmtSrvr.cpp: theFacade is NULL.");
return false; return false;
} }
...@@ -630,6 +635,7 @@ MgmtSrvr::start() ...@@ -630,6 +635,7 @@ MgmtSrvr::start()
if(_blockNumber == -1){ if(_blockNumber == -1){
DEBUG("MgmtSrvr.cpp: _blockNumber is -1."); DEBUG("MgmtSrvr.cpp: _blockNumber is -1.");
error_string.append("MgmtSrvr.cpp: _blockNumber is -1.");
theFacade->stop_instance(); theFacade->stop_instance();
theFacade = 0; theFacade = 0;
return false; return false;
...@@ -2304,7 +2310,8 @@ bool ...@@ -2304,7 +2310,8 @@ bool
MgmtSrvr::alloc_node_id(NodeId * nodeId, MgmtSrvr::alloc_node_id(NodeId * nodeId,
enum ndb_mgm_node_type type, enum ndb_mgm_node_type type,
struct sockaddr *client_addr, struct sockaddr *client_addr,
SOCKET_SIZE_TYPE *client_addr_len) SOCKET_SIZE_TYPE *client_addr_len,
BaseString &error_string)
{ {
Guard g(&f_node_id_mutex); Guard g(&f_node_id_mutex);
#if 0 #if 0
...@@ -2322,19 +2329,26 @@ MgmtSrvr::alloc_node_id(NodeId * nodeId, ...@@ -2322,19 +2329,26 @@ MgmtSrvr::alloc_node_id(NodeId * nodeId,
} }
} }
bool found_matching_id= false;
bool found_matching_type= false;
bool found_free_node= false;
unsigned type_c= 0;
ndb_mgm_configuration_iterator iter(*(ndb_mgm_configuration *)_config->m_configValues, ndb_mgm_configuration_iterator iter(*(ndb_mgm_configuration *)_config->m_configValues,
CFG_SECTION_NODE); CFG_SECTION_NODE);
for(iter.first(); iter.valid(); iter.next()) { for(iter.first(); iter.valid(); iter.next()) {
unsigned tmp= 0; unsigned tmp= 0;
if(iter.get(CFG_NODE_ID, &tmp)) abort(); if(iter.get(CFG_NODE_ID, &tmp)) abort();
if (connected_nodes.get(tmp))
continue;
if (*nodeId && *nodeId != tmp) if (*nodeId && *nodeId != tmp)
continue; continue;
unsigned type_c; found_matching_id= true;
if(iter.get(CFG_TYPE_OF_SECTION, &type_c)) abort(); if(iter.get(CFG_TYPE_OF_SECTION, &type_c)) abort();
if(type_c != type) if(type_c != type)
continue; continue;
found_matching_type= true;
if (connected_nodes.get(tmp))
continue;
found_free_node= true;
const char *config_hostname = 0; const char *config_hostname = 0;
if(iter.get(CFG_NODE_HOST, &config_hostname)) abort(); if(iter.get(CFG_NODE_HOST, &config_hostname)) abort();
...@@ -2374,6 +2388,43 @@ MgmtSrvr::alloc_node_id(NodeId * nodeId, ...@@ -2374,6 +2388,43 @@ MgmtSrvr::alloc_node_id(NodeId * nodeId,
#endif #endif
return true; return true;
} }
BaseString type_string, type_c_string;
{
const char *alias, *str;
alias= ndb_mgm_get_node_type_alias_string(type, &str);
type_string.assfmt("%s(%s)", alias, str);
alias= ndb_mgm_get_node_type_alias_string((enum ndb_mgm_node_type)type_c, &str);
type_c_string.assfmt("%s(%s)", alias, str);
}
if (*nodeId == 0) {
if (found_matching_id)
if (found_matching_type)
if (found_free_node)
error_string.appfmt("Connection done from wrong host %s.",
inet_ntoa(((struct sockaddr_in *)(client_addr))->sin_addr));
else
error_string.appfmt("No free node id found for %s.", type_string.c_str());
else
error_string.appfmt("No %s node defined in config file.", type_string.c_str());
else
error_string.append("No nodes defined in config file.");
} else {
if (found_matching_id)
if (found_matching_type)
if (found_free_node)
error_string.appfmt("Connection with id %d done from wrong host %s, expected host XX.",
*nodeId, inet_ntoa(((struct sockaddr_in *)(client_addr))->sin_addr));
else
error_string.appfmt("Id %d already allocated by another node.", *nodeId);
else
error_string.appfmt("Id %d configured as %s, connect attempted as %s.",
*nodeId, type_c_string.c_str(), type_string.c_str());
else
error_string.appfmt("No node defined with id=%d in config file.", *nodeId);
}
return false; return false;
} }
......
...@@ -179,7 +179,7 @@ public: ...@@ -179,7 +179,7 @@ public:
* @return true if succeeded, otherwise false * @return true if succeeded, otherwise false
*/ */
bool check_start(); // may be run before start to check that some things are ok bool check_start(); // may be run before start to check that some things are ok
bool start(); bool start(BaseString &error_string);
~MgmtSrvr(); ~MgmtSrvr();
...@@ -467,7 +467,8 @@ public: ...@@ -467,7 +467,8 @@ public:
*/ */
bool getNextNodeId(NodeId * _nodeId, enum ndb_mgm_node_type type) const ; bool getNextNodeId(NodeId * _nodeId, enum ndb_mgm_node_type type) const ;
bool alloc_node_id(NodeId * _nodeId, enum ndb_mgm_node_type type, bool alloc_node_id(NodeId * _nodeId, enum ndb_mgm_node_type type,
struct sockaddr *client_addr, SOCKET_SIZE_TYPE *client_addr_len); struct sockaddr *client_addr, SOCKET_SIZE_TYPE *client_addr_len,
BaseString &error_string);
/** /**
* *
......
...@@ -413,11 +413,14 @@ MgmApiSession::get_nodeid(Parser_t::Context &, ...@@ -413,11 +413,14 @@ MgmApiSession::get_nodeid(Parser_t::Context &,
NodeId tmp= nodeid; NodeId tmp= nodeid;
if(tmp == 0 || !m_allocated_resources->is_reserved(tmp)){ if(tmp == 0 || !m_allocated_resources->is_reserved(tmp)){
BaseString error_string;
if (!m_mgmsrv.alloc_node_id(&tmp, (enum ndb_mgm_node_type)nodetype, if (!m_mgmsrv.alloc_node_id(&tmp, (enum ndb_mgm_node_type)nodetype,
&addr, &addrlen)){ &addr, &addrlen, error_string)){
const char *alias;
const char *str;
alias= ndb_mgm_get_node_type_alias_string((enum ndb_mgm_node_type)nodetype, &str);
m_output->println(cmd); m_output->println(cmd);
m_output->println("result: no free nodeid %d for nodetype %d", m_output->println("result: %s", error_string.c_str());
nodeid, nodetype);
m_output->println(""); m_output->println("");
return; return;
} }
......
...@@ -183,7 +183,7 @@ NDB_MAIN(mgmsrv){ ...@@ -183,7 +183,7 @@ NDB_MAIN(mgmsrv){
if(!glob.socketServer->tryBind(glob.port, glob.interface_name)){ if(!glob.socketServer->tryBind(glob.port, glob.interface_name)){
ndbout_c("Unable to setup port: %s:%d!\n" ndbout_c("Unable to setup port: %s:%d!\n"
"Please check if the port is already used,\n" "Please check if the port is already used,\n"
"(perhaps a mgmtsrvr is already running),\n" "(perhaps a ndb_mgmd is already running),\n"
"and if you are executing on the correct computer", "and if you are executing on the correct computer",
(glob.interface_name ? glob.interface_name : "*"), glob.port); (glob.interface_name ? glob.interface_name : "*"), glob.port);
goto error_end; goto error_end;
...@@ -195,7 +195,7 @@ NDB_MAIN(mgmsrv){ ...@@ -195,7 +195,7 @@ NDB_MAIN(mgmsrv){
if(!glob.socketServer->setup(mapi, glob.port, glob.interface_name)){ if(!glob.socketServer->setup(mapi, glob.port, glob.interface_name)){
ndbout_c("Unable to setup management port: %d!\n" ndbout_c("Unable to setup management port: %d!\n"
"Please check if the port is already used,\n" "Please check if the port is already used,\n"
"(perhaps a mgmtsrvr is already running),\n" "(perhaps a ndb_mgmd is already running),\n"
"and if you are executing on the correct computer", "and if you are executing on the correct computer",
glob.port); glob.port);
delete mapi; delete mapi;
...@@ -228,11 +228,15 @@ NDB_MAIN(mgmsrv){ ...@@ -228,11 +228,15 @@ NDB_MAIN(mgmsrv){
} }
signal(SIGPIPE, SIG_IGN); signal(SIGPIPE, SIG_IGN);
if(!glob.mgmObject->start()){ {
BaseString error_string;
if(!glob.mgmObject->start(error_string)){
ndbout_c("Unable to start management server."); ndbout_c("Unable to start management server.");
ndbout_c("Probably caused by illegal initial configuration file."); ndbout_c("Probably caused by illegal initial configuration file.");
ndbout_c(error_string.c_str());
goto error_end; goto error_end;
} }
}
//glob.mgmObject->saveConfig(); //glob.mgmObject->saveConfig();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment