Skip to content

P
proview
Commit 22474a51 authored by Jonas Nylund's avatar Jonas Nylund
Browse files
Options

SEV If an attribute is changed the table will be recreated instead of altered.... 

SEV If an attribute is changed the table will be recreated instead of altered. A copy of the old table will be saved. This due to very slow alter of table
parent 72e34299
Hide whitespace changes
Inline Side-by-side
Showing with 182 additions and 180 deletions
sev/exe/sev_server/src/sev_server.cpp
View file @ 22474a51
......@@ -585,6 +585,7 @@ int sev_server::check_histitems( sev_sMsgHistItems *msg, unsigned int size)
storagetime = net_NetTimeToDeltaTime( &buffP->storagetime );
sprintf( tablename, "obj_%s", m_db->oid_to_table( buffP->oid, attributeName) );
bool newobject = false;
if ( !m_db->check_objectitem( &m_sts,
tablename,
buffP->oid,
......@@ -608,12 +609,14 @@ int sev_server::check_histitems( sev_sMsgHistItems *msg, unsigned int size)
buffP->options,
&idx);
if ( EVEN(m_sts)) return m_sts;
newobject = true;
}
vector<sev_attr> oldattrVec = m_db->m_items[idx].attr;
vector<sev_attr> newattrVec;
m_db->m_items[idx].value_size = 0;
//Check if any new attributes is found if so add column
bool tableChange = false;
for(size_t j = 0; j < buffP->attrnum; j++) {
//printf( "Received: %s.%s\n", buffP->oname, buffP->attr[j].aname);
sev_attr newattr;
......@@ -624,7 +627,6 @@ int sev_server::check_histitems( sev_sMsgHistItems *msg, unsigned int size)
newattr.elem = 0;
newattrVec.push_back(newattr);
bool tableChange = false;
if ( !m_db->check_objectitemattr( &m_sts,
tablename,
buffP->oid,
......@@ -633,23 +635,24 @@ int sev_server::check_histitems( sev_sMsgHistItems *msg, unsigned int size)
buffP->attr[j].type,
buffP->attr[j].size,
&idx) ) {
m_db->add_objectitemattr( &m_sts,
tablename,
buffP->oid,
buffP->attr[j].aname,
buffP->oname,
buffP->attr[j].type,
buffP->attr[j].size,
&idx);
if ( EVEN(m_sts)) return m_sts;
tableChange = true;
}
}
//Be sure that we have the correct attribute order. Use the list from the client
m_db->m_items[idx].attr.clear();
m_db->m_items[idx].attr = newattrVec;
m_db->m_items[idx].attrnum = newattrVec.size();
if(tableChange) {
//Either an attribute has changed type or size or we have a new attribute
//rename the table to something and create a new one.
//this is the only way to do this without hanging the server for several minutes
m_db->handle_objectchange(&m_sts, tablename, idx, newobject);
}
//If node is coming up again we do not want deadband to be active due to init of old_value
m_db->m_items[idx].deadband_active = 0;
m_db->m_items[idx].first_storage = 1;
......@@ -675,7 +678,6 @@ int sev_server::check_histitems( sev_sMsgHistItems *msg, unsigned int size)
rk = buffP->sevid;
rp = (sev_sRefid *) tree_Insert(&sts, m_refid, &rk);
rp->idx = idx;
}
int numberOfAttributes = buffP->attrnum;
......
sev/lib/sev/src/sev_db.h
View file @ 22474a51
......@@ -106,8 +106,6 @@ class sev_db {
pwr_tFloat32 deadband, pwr_tMask options, unsigned int *idx) { return 0;}
virtual int store_objectitem( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *oname, char *aname,
pwr_tDeltaTime storagetime, char *description, pwr_tFloat32 scantime, pwr_tFloat32 deadband, pwr_tMask options) { return 0;}
virtual int add_objectitemattr( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int *idx) { return 0;}
virtual int get_item( pwr_tStatus *sts, sev_item *item, char *tablename) { return 0;}
virtual int get_objectitem( pwr_tStatus *sts, sev_item *item, char *tablename) { return 0;}
virtual int get_objectitems( pwr_tStatus *sts) { return 0;}
......@@ -118,6 +116,7 @@ class sev_db {
virtual int get_objectvalues( pwr_tStatus *sts, sev_item *item,
unsigned int size, pwr_tTime *starttime, pwr_tTime *endtime,
int maxsize, pwr_tTime **tbuf, void **vbuf, unsigned int *bsize) { return 0;}
virtual int handle_objectchange(pwr_tStatus *sts, char *tablename, unsigned int item_idx, bool newObject) { return 0;}
};
......
sev/lib/sev/src/sev_dbms.cpp
View file @ 22474a51
......@@ -1421,13 +1421,11 @@ int sev_dbms::check_item( pwr_tStatus *sts, pwr_tOid oid, char *oname, char *ana
cdh_NoCaseStrcmp( aname, m_items[i].attr[0].aname) == 0) {
char query[600];
bool itemdefchange = false;
if ( type != m_items[i].attr[0].type ||
size != m_items[i].attr[0].size)
{
//Size or type changed what to do?
if( !handle_attrchange(sts, m_items[i].tablename, oid, (char *)"value", oname, type, size, i, 0) ) {
return 1;
}
itemdefchange = true;
}
sprintf( query, "update items set ");
......@@ -1477,6 +1475,12 @@ int sev_dbms::check_item( pwr_tStatus *sts, pwr_tOid oid, char *oname, char *ana
*sts = SEV__DBERROR;
return 0;
}
if(itemdefchange) {
if( !handle_itemchange(sts, m_items[i].tablename, i) ) {
return 1;
}
}
*idx = i;
*sts = SEV__SUCCESS;
return 1;
......@@ -1754,7 +1758,7 @@ int sev_dbms::add_objectitem( pwr_tStatus *sts, char *tablename, pwr_tOid oid, c
scantime, deadband, options);
if ( EVEN(*sts)) return 0;
create_objecttable( sts, tablename, oid, aname, options, deadband);
create_objecttable( sts, tablename, options, deadband);
if ( EVEN(*sts)) return 0;
*sts = SEV__SUCCESS;
......@@ -1803,7 +1807,7 @@ int sev_dbms::store_objectitem( pwr_tStatus *sts, char *tablename, pwr_tOid oid,
return 1;
}
int sev_dbms::create_objecttable( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, pwr_tMask options, float deadband)
int sev_dbms::create_objecttable( pwr_tStatus *sts, char *tablename, pwr_tMask options, float deadband)
{
char query[2000];
char timeformatstr[80];
......@@ -1856,6 +1860,7 @@ int sev_dbms::create_objecttable( pwr_tStatus *sts, char *tablename, pwr_tOid oi
*sts = SEV__DBERROR;
return 0;
}
*sts = SEV__SUCCESS;
return 1;
}
......@@ -1867,113 +1872,64 @@ int sev_dbms::check_objectitemattr( pwr_tStatus *sts, char *tablename, pwr_tOid
if (cdh_NoCaseStrcmp( aname, item->attr[j].aname) == 0) {
if (type != item->attr[j].type ||
size != item->attr[j].size) {
*sts = SEV__NOSUCHITEM;
return 0;
/*
if( !handle_attrchange(sts, tablename, oid, aname,oname,type,size, *idx, j) ) {
return 1;
}
item->attr[j].type = type;
item->attr[j].size = size;
update_objectitemattr(sts, tablename, aname, type, size);
*/
}
*sts = SEV__SUCCESS;
return 1;
}
}
*sts = SEV__NOSUCHITEM;
return 0;
}
int sev_dbms::add_objectitemattr( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int *idx)
{
char query[2000];
sprintf( query, "alter table %s add `%s` %s;",
tablename, aname, pwrtype_to_type( type, size));
//printf( "%s: %s\n", __FUNCTION__ ,query);
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
int aidx = get_nextattridx(sts, tablename);
sprintf( query, "insert into objectitemattributes (tablename, attributename, attributeidx, attributetype, attributesize) "
"values('%s', '%s', %d, %d, %d)", tablename, aname, aidx, type, size);
//printf( "%s: %s\n", __FUNCTION__ ,query);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
sev_attr newattr;
strncpy( newattr.aname, aname, sizeof(newattr.aname));
newattr.type = type;
newattr.size = size;
newattr.elem = 0;
m_items[*idx].attr.push_back(newattr);
m_items[*idx].attrnum = m_items[*idx].attr.size();
*sts = SEV__SUCCESS;
return 1;
}
int sev_dbms::get_nextattridx( pwr_tStatus *sts, char *tablename )
pwr_tUInt64 sev_dbms::get_minFromIntegerColumn( char *tablename, char *colname )
{
char query[2000];
sprintf( query, "select max(attributeidx) from objectitemattributes where tablename='%s'", tablename);
pwr_tUInt64 retVal = 0;
sprintf( query, "select min(`%s`) from %s", colname, tablename);
//printf( "%s: %s\n", __FUNCTION__ ,query);
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
MYSQL_ROW row;
MYSQL_RES *result = mysql_store_result( m_env->con());
if ( !result) {
printf( "GetValues Result Error\n");
*sts = SEV__DBERROR;
return 0;
return retVal;
}
int rows = mysql_num_rows( result);
if(rows > 1) {
printf( "Duplicate items Error\n");
*sts = SEV__DBERROR;
if(rows <= 0) {
mysql_free_result( result);
return 0;
return retVal;
}
int attridx = -1;
for ( int i = 0; i < rows; i++) {
row = mysql_fetch_row( result);
if (!row) break;
if(row[0] != NULL) {
attridx = atoi( row[0]);
}
MYSQL_ROW row;
row = mysql_fetch_row( result);
if(row[0] != NULL) {
retVal = strtoull(row[0], 0, 10);
}
mysql_free_result( result);
*sts = SEV__SUCCESS;
return attridx+1;
return retVal;
}
pwr_tUInt64 sev_dbms::get_minFromIntegerColumn( char *tablename, char *colname )
pwr_tUInt64 sev_dbms::get_maxFromIntegerColumn( char *tablename, char *colname )
{
char query[2000];
pwr_tUInt64 retVal = 0;
sprintf( query, "select min(`%s`) from %s", colname, tablename);
sprintf( query, "select max(`%s`) from %s", colname, tablename);
//printf( "%s: %s\n", __FUNCTION__ ,query);
int rc = mysql_query( m_env->con(), query);
......@@ -2461,68 +2417,6 @@ int sev_dbms::delete_old_objectdata( pwr_tStatus *sts, char *tablename,
return 1;
}
int sev_dbms::alter_attrcolumn(pwr_tStatus *sts, char *tablename, char *aname,
pwr_eType newtype, unsigned int newsize,
pwr_eType oldtype, unsigned int oldsize)
{
//TODO
return 0;
}
int sev_dbms::rename_attrcolumn(pwr_tStatus *sts, char *tablename, char *aname, pwr_eType type, unsigned int size)
{
char query[2000];
char newname[500];
char timestr[40];
pwr_tTime uptime;
time_GetTime( &uptime);
time_AtoAscii( &uptime, time_eFormat_NumDateAndTime, timestr, sizeof(timestr));
timestr[19] = 0;
sprintf(newname, "%s_before_%s", aname, timestr);
sprintf(query, "alter table %s change `%s` `%s` %s", tablename, aname, newname, pwrtype_to_type(type, size));
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
return 1;
}
int sev_dbms::remove_objectitemattr( pwr_tStatus *sts, char *tablename, char *aname)
{
char query[2000];
sprintf( query, "delete from objectitemattributes where tablename = '%s' and attributename = '%s'", tablename, aname);
//printf( "%s: %s\n", __FUNCTION__ ,query);
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
return 1;
}
int sev_dbms::update_objectitemattr( pwr_tStatus *sts, char *tablename, char *aname, pwr_eType type, unsigned int size)
{
char query[2000];
sprintf( query, "update objectitemattributes set attributetype=%d, attributesize=%d where tablename = '%s' and attributename = '%s'",
type, size, tablename, aname);
//printf( "%s: %s\n", __FUNCTION__ ,query);
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
return 1;
}
int sev_dbms::check_deadband(pwr_eType type, unsigned int size, pwr_tFloat32 deadband, void *value, void *oldvalue)
{
......@@ -3005,43 +2899,157 @@ int sev_dbms::get_objectvalues( pwr_tStatus *sts, sev_item *item,
return 1;
}
int sev_dbms::handle_attrchange(pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int item_idx, unsigned int attr_idx)
int sev_dbms::handle_itemchange(pwr_tStatus *sts, char *tablename, unsigned int item_idx)
{
//We try to alter the table if it is possible(float->double int->long or little_string->bigger_string probably works).
//If we can't alter the table we rename the column and create a new one
char timestr[40];
pwr_tTime uptime;
time_GetTime( &uptime);
time_AtoAscii( &uptime, time_eFormat_NumDateAndTime, timestr, sizeof(timestr));
timestr[19] = 0;
// Replace ':' '-' and ' ' in timestr with '_'
for ( char *s = timestr; *s; s++) {
if ( *s == ':')
*s = '_';
if ( *s == ' ')
*s = '_';
if ( *s == '-')
*s = '_';
}
char newTableName[64];
snprintf(newTableName, sizeof(newTableName), "%s_%s", tablename, timestr);
printf("Recreating table %s due to attribute definition changes, old table saved to %s \n", tablename, newTableName);
errh_Warning("Recreating table %s due to attribute definition changes, old table saved to %s", tablename, newTableName);
char query[600];
sprintf(query, "RENAME TABLE %s to %s", tablename, newTableName);
int rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
sev_item *item = &m_items[item_idx];
printf("Column:%s in %s has been changed from Type/size %d/%d to %d/%d\n",
aname, tablename, item->attr[attr_idx].type, item->attr[attr_idx].size, type, size );
errh_Warning("Column:%s in %s has been changed from Type/size %d/%d to %d/%d",
aname, tablename, item->attr[attr_idx].type, item->attr[attr_idx].size, type, size );
if (!alter_attrcolumn(sts, tablename, aname, type, size, item->attr[attr_idx].type, item->attr[attr_idx].size)) {
//We were not able to alter the column then we should rename the column and create a new one
if (!rename_attrcolumn(sts, tablename, aname, item->attr[attr_idx].type, item->attr[attr_idx].size)) {
//Very very bad this should not happen
errh_Error("Could not rename column that has been changed tablename:%s columnname:%s", tablename, aname);
//item->attr[attr_idx].status = -1; //TODO Skip this attribute during logging
create_table( sts, item->oid, item->attr[0].aname, item->attr[0].type, item->attr[0].size, item->options, item->deadband);
if ( EVEN(*sts)) return 0;
if(item->options & pwr_mSevOptionsMask_ReadOptimized ) {
//If we set increment to same value as in the old table we can easily move the data from the old table to the new one
pwr_tUInt64 autoIncrValue = get_maxFromIntegerColumn(newTableName, (char*)"id");
if(autoIncrValue)
autoIncrValue++;
sprintf(query, "ALTER TABLE %s AUTO_INCREMENT = %llu", tablename, autoIncrValue);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
errh_Warning("Column:%s in %s renamed to:%s_before_xxxx xx:xx:xx", aname, tablename, aname );
}
char query[2000];
sprintf( query, "alter table %s add `%s` %s;",
tablename, aname, pwrtype_to_type( type, size));
int rc = mysql_query( m_env->con(), query);
*sts = SEV__SUCCESS;
return 1;
}
int sev_dbms::handle_objectchange(pwr_tStatus *sts, char *tablename, unsigned int item_idx, bool newObject)
{
char newTableName[64];
char query[600];
int rc;
if(!newObject) {
char timestr[40];
pwr_tTime uptime;
time_GetTime( &uptime);
time_AtoAscii( &uptime, time_eFormat_NumDateAndTime, timestr, sizeof(timestr));
timestr[19] = 0;
// Replace ':' '-' and ' ' in timestr with '_'
for ( char *s = timestr; *s; s++) {
if ( *s == ':')
*s = '_';
if ( *s == ' ')
*s = '_';
if ( *s == '-')
*s = '_';
}
snprintf(newTableName, sizeof(newTableName), "%s_%s", tablename, timestr);
printf("Recreating table %s due to attribute definition changes, old table saved to %s \n", tablename, newTableName);
errh_Warning("Recreating table %s due to attribute definition changes, old table saved to %s", tablename, newTableName);
sprintf(query, "RENAME TABLE %s to %s", tablename, newTableName);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
}
sev_item *item = &m_items[item_idx];
create_objecttable( sts, tablename, item->options, item->deadband);
if ( EVEN(*sts)) return 0;
if(!newObject) {
if(item->options & pwr_mSevOptionsMask_ReadOptimized ) {
//If we set increment to same value as in the old table we can easily move the data from the old table to the new one
pwr_tUInt64 autoIncrValue = get_maxFromIntegerColumn(newTableName, (char*)"sev__id");
if(autoIncrValue)
autoIncrValue++;
sprintf(query, "ALTER TABLE %s AUTO_INCREMENT = %llu", tablename, autoIncrValue);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
}
sprintf(query, "delete from objectitemattributes where tablename = '%s'", tablename);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__, mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
}
for(size_t i = 0; i < item->attr.size(); i++) {
sprintf( query, "alter table %s add `%s` %s;", tablename, item->attr[i].aname, pwrtype_to_type( item->attr[i].type, item->attr[i].size));
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
int aidx = i;
sprintf( query, "insert into objectitemattributes (tablename, attributename, attributeidx, attributetype, attributesize) "
"values('%s', '%s', %d, %d, %d)", tablename, item->attr[i].aname, aidx, item->attr[i].type, item->attr[i].size);
rc = mysql_query( m_env->con(), query);
if (rc) {
printf( "%s: %s\n", __FUNCTION__,mysql_error(m_env->con()));
*sts = SEV__DBERROR;
return 0;
}
}
*sts = SEV__SUCCESS;
return 1;
}
sev_dbms::~sev_dbms()
{
printf("Freeing memory\n");
......
sev/lib/sev/src/sev_dbms.h
View file @ 22474a51
......@@ -149,10 +149,7 @@ class sev_dbms : public sev_db {
int store_objectitem( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *oname, char *aname,
pwr_tDeltaTime storagetime, char *description, pwr_tFloat32 scantime,
pwr_tFloat32 deadband, pwr_tMask options);
int create_objecttable( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname,
pwr_tMask options, float deadband);
int add_objectitemattr( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int *idx);
int create_objecttable( pwr_tStatus *sts, char *tablename, pwr_tMask options, float deadband);
int store_objectvalue( pwr_tStatus *sts, int item_idx, int attr_idx,
pwr_tTime time, void *buf, void *oldbuf, unsigned int size);
int get_item( pwr_tStatus *sts, sev_item *item, char *tablename);
......@@ -161,19 +158,15 @@ class sev_dbms : public sev_db {
int get_objectitemattributes( pwr_tStatus *sts, sev_item *item, char *tablename);
int check_objectitemattr( pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int *idx);
int get_nextattridx( pwr_tStatus *sts, char *tablename );
int delete_old_objectdata( pwr_tStatus *sts, char *tablename,
pwr_tMask options, pwr_tTime limit);
int alter_attrcolumn(pwr_tStatus *sts, char *tablename, char *aname, pwr_eType newtype, unsigned int newsize, pwr_eType oldtype, unsigned int oldsize);
int rename_attrcolumn(pwr_tStatus *sts, char *tablename, char *aname, pwr_eType type, unsigned int size);
int remove_objectitemattr( pwr_tStatus *sts, char *tablename, char *aname);
int update_objectitemattr( pwr_tStatus *sts, char *tablename, char *aname, pwr_eType type, unsigned int size);
int check_deadband(pwr_eType type, unsigned int size, pwr_tFloat32 deadband, void *value, void *oldvalue);
int get_objectvalues( pwr_tStatus *sts, sev_item *item, unsigned int size, pwr_tTime *starttime, pwr_tTime *endtime,
int maxsize, pwr_tTime **tbuf, void **vbuf, unsigned int *bsize);
pwr_tUInt64 get_minFromIntegerColumn( char *tablename, char *colname );
int handle_attrchange(pwr_tStatus *sts, char *tablename, pwr_tOid oid, char *aname, char *oname,
pwr_eType type, unsigned int size, unsigned int item_idx, unsigned int attr_idx);
pwr_tUInt64 get_maxFromIntegerColumn( char *tablename, char *colname );
int handle_itemchange(pwr_tStatus *sts, char *tablename, unsigned int item_idx);
int handle_objectchange(pwr_tStatus *sts, char *tablename, unsigned int item_idx, bool newObject);
};
#endif
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7