Skip to content

M
mariadb
Commit 4630f67e authored by jonas@perch.ndb.mysql.com's avatar jonas@perch.ndb.mysql.com
Browse files
Options

bug#16771 - ndb dd - fix leak of extents

parent 1e355912
Show whitespace changes
Inline Side-by-side
Showing with 177 additions and 51 deletions
storage/ndb/src/kernel/blocks/dbtup/Dbtup.hpp
View file @ 4630f67e
......@@ -565,7 +565,6 @@ typedef Ptr<Fragoperrec> FragoperrecPtr;
*
*/
STATIC_CONST( SZ = EXTENT_SEARCH_MATRIX_SIZE );
Uint32 m_extent_search_matrix[SZ]; // 4x4
DLList<Extent_info>::Head m_free_extents[SZ];
Uint32 m_total_extent_free_space_thresholds[EXTENT_SEARCH_MATRIX_ROWS];
Uint32 m_page_free_bits_map[EXTENT_SEARCH_MATRIX_COLS];
......@@ -593,6 +592,8 @@ typedef Ptr<Fragoperrec> FragoperrecPtr;
SLList<Extent_info, Extent_list_t>::Head m_extent_list;
};
void dump_disk_alloc(Disk_alloc_info&);
struct Fragrecord {
Uint32 nextStartRange;
Uint32 currentPageRange;
......
storage/ndb/src/kernel/blocks/dbtup/DbtupDiskAlloc.cpp
View file @ 4630f67e
......@@ -17,6 +17,112 @@
#define DBTUP_C
#include "Dbtup.hpp"
static
NdbOut&
operator<<(NdbOut& out, const Ptr<Dbtup::Page> & ptr)
{
out << "[ Page: ptr.i: " << ptr.i
<< " [ m_file_no: " << ptr.p->m_file_no
<< " m_page_no: " << ptr.p->m_page_no << "]"
<< " list_index: " << ptr.p->list_index
<< " free_space: " << ptr.p->free_space
<< " uncommitted_used_space: " << ptr.p->uncommitted_used_space
<< " ]";
return out;
}
static
NdbOut&
operator<<(NdbOut& out, const Ptr<Dbtup::Page_request> & ptr)
{
out << "[ Page_request: ptr.i: " << ptr.i
<< " " << ptr.p->m_key
<< " m_estimated_free_space: " << ptr.p->m_estimated_free_space
<< " m_list_index: " << ptr.p->m_list_index
<< " m_frag_ptr_i: " << ptr.p->m_frag_ptr_i
<< " m_extent_info_ptr: " << ptr.p->m_extent_info_ptr
<< " m_ref_count: " << ptr.p->m_ref_count
<< " m_uncommitted_used_space: " << ptr.p->m_uncommitted_used_space
<< " ]";
return out;
}
static
NdbOut&
operator<<(NdbOut& out, const Ptr<Dbtup::Extent_info> & ptr)
{
out << "[ Extent_info: ptr.i " << ptr.i
<< " " << ptr.p->m_key
<< " m_first_page_no: " << ptr.p->m_first_page_no
<< " m_free_space: " << ptr.p->m_free_space
<< " m_free_matrix_pos: " << ptr.p->m_free_matrix_pos
<< " m_free_page_count: [";
for(Uint32 i = 0; i<Dbtup::EXTENT_SEARCH_MATRIX_COLS; i++)
out << " " << ptr.p->m_free_page_count[i];
out << " ] ]";
return out;
}
void
Dbtup::dump_disk_alloc(Dbtup::Disk_alloc_info & alloc)
{
ndbout_c("dirty pages");
for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
{
printf(" %d : ", i);
Ptr<Page> ptr;
ArrayPool<Page> *pool= (ArrayPool<Page>*)&m_global_page_pool;
LocalDLList<Page> list(*pool, alloc.m_dirty_pages[i]);
for(list.first(ptr); !ptr.isNull(); list.next(ptr))
{
ndbout << ptr << " ";
}
ndbout_c("");
}
ndbout_c("page requests");
for(Uint32 i = 0; i<MAX_FREE_LIST; i++)
{
printf(" %d : ", i);
Ptr<Page_request> ptr;
LocalDLList<Page_request> list(c_page_request_pool,
alloc.m_page_requests[i]);
for(list.first(ptr); !ptr.isNull(); list.next(ptr))
{
ndbout << ptr << " ";
}
ndbout_c("");
}
ndbout_c("Extent matrix");
for(Uint32 i = 0; i<alloc.SZ; i++)
{
printf(" %d : ", i);
Ptr<Extent_info> ptr;
LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[i]);
for(list.first(ptr); !ptr.isNull(); list.next(ptr))
{
ndbout << ptr << " ";
}
ndbout_c("");
}
if (alloc.m_curr_extent_info_ptr_i != RNIL)
{
Ptr<Extent_info> ptr;
c_extent_pool.getPtr(ptr, alloc.m_curr_extent_info_ptr_i);
ndbout << "current extent: " << ptr << endl;
}
}
#if defined VM_TRACE || true
#define ddassert(x) do { if(unlikely(!(x))) { dump_disk_alloc(alloc); ndbrequire(false); } } while(0)
#else
#define ddassert(x)
#endif
Dbtup::Disk_alloc_info::Disk_alloc_info(const Tablerec* tabPtrP,
Uint32 extent_size)
{
......@@ -60,19 +166,19 @@ Dbtup::Disk_alloc_info::find_extent(Uint32 sz) const
* Find the biggest available (with most free space)
* Return position in matrix
*/
Uint32 col = calc_page_free_bits(sz);
Uint32 mask= EXTENT_SEARCH_MATRIX_COLS - 1;
for(Uint32 i= 0; i<EXTENT_SEARCH_MATRIX_SIZE; i++)
{
// Check that it can cater for request
if (m_extent_search_matrix[i] < sz)
if (!m_free_extents[i].isEmpty())
{
i = (i + mask) & ~mask;
continue;
return i;
}
if (!m_free_extents[i].isEmpty())
if ((i & mask) >= col)
{
return i;
i = (i & ~mask) + mask;
}
}
......@@ -93,12 +199,6 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const
* absolutly last
*/
{
printf("free space %d free_page_thresholds ", free);
for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_ROWS; i++)
printf("%d ", m_total_extent_free_space_thresholds[i]);
ndbout_c("");
const Uint32 *arr= m_total_extent_free_space_thresholds;
for(; free < * arr++; row++)
assert(row < EXTENT_SEARCH_MATRIX_ROWS);
......@@ -123,11 +223,6 @@ Dbtup::Disk_alloc_info::calc_extent_pos(const Extent_info* extP) const
*/
Uint32 pos= (row * (mask + 1)) + (col & mask);
printf("free space %d free_page_count ", free);
for(Uint32 i = 0; i<EXTENT_SEARCH_MATRIX_COLS; i++)
printf("%d ", extP->m_free_page_count[i]);
ndbout_c(" -> row: %d col: %d -> pos= %d", row, col, pos);
assert(pos < EXTENT_SEARCH_MATRIX_SIZE);
return pos;
}
......@@ -237,7 +332,9 @@ Dbtup::disk_page_prealloc(Signal* signal,
* and since it couldn't accomadate the request
* we put it on the free list
*/
alloc.m_curr_extent_info_ptr_i = RNIL;
Uint32 pos= alloc.calc_extent_pos(ext.p);
ext.p->m_free_matrix_pos = pos;
LocalDLList<Extent_info> list(c_extent_pool, alloc.m_free_extents[pos]);
list.add(ext);
}
......@@ -290,7 +387,14 @@ Dbtup::disk_page_prealloc(Signal* signal,
alloc.m_curr_extent_info_ptr_i= ext.i;
ext.p->m_free_matrix_pos= RNIL;
pageBits= tsman.alloc_page_from_extent(&ext.p->m_key, bits);
ndbassert(pageBits >= 0);
#ifdef VM_TRACE
ddassert(pageBits >= 0);
#else
if (unlikely(pageBits < 0))
{
return -AllocExtentReq::NoExtentAvailable;
}
#endif
}
/**
......@@ -305,18 +409,18 @@ Dbtup::disk_page_prealloc(Signal* signal,
*/
Uint32 size= alloc.calc_page_free_space((Uint32)pageBits);
ndbassert(size >= sz);
ddassert(size >= sz);
Uint32 new_size = size - sz; // Subtract alloc rec
req.p->m_estimated_free_space= new_size; // Store on page request
Uint32 newPageBits= alloc.calc_page_free_bits(new_size);
if (newPageBits != (Uint32)pageBits)
{
ndbassert(ext.p->m_free_page_count[pageBits] > 0);
ddassert(ext.p->m_free_page_count[pageBits] > 0);
ext.p->m_free_page_count[pageBits]--;
ext.p->m_free_page_count[newPageBits]++;
}
ndbassert(ext.p->m_free_space >= sz);
ddassert(ext.p->m_free_space >= sz);
ext.p->m_free_space -= sz;
// And put page request in correct free list
......@@ -365,13 +469,13 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc,
Ptr<Page> pagePtr,
Uint32 old_idx, Uint32 sz)
{
ndbassert(pagePtr.p->list_index == old_idx);
ddassert(pagePtr.p->list_index == old_idx);
Uint32 free= pagePtr.p->free_space;
Uint32 used= pagePtr.p->uncommitted_used_space + sz;
Uint32 ext= pagePtr.p->m_extent_info_ptr;
ndbassert(free >= used);
ddassert(free >= used);
Ptr<Extent_info> extentPtr;
c_extent_pool.getPtr(extentPtr, ext);
......@@ -385,14 +489,14 @@ Dbtup::disk_page_prealloc_dirty_page(Disk_alloc_info & alloc,
old_list.remove(pagePtr);
new_list.add(pagePtr);
ndbassert(extentPtr.p->m_free_page_count[old_idx]);
ddassert(extentPtr.p->m_free_page_count[old_idx]);
extentPtr.p->m_free_page_count[old_idx]--;
extentPtr.p->m_free_page_count[new_idx]++;
pagePtr.p->list_index= new_idx;
}
pagePtr.p->uncommitted_used_space = used;
ndbassert(extentPtr.p->m_free_space >= sz);
ddassert(extentPtr.p->m_free_space >= sz);
extentPtr.p->m_free_space -= sz;
Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
if (old_pos != RNIL) // Current extent
......@@ -417,7 +521,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
Ptr<Page_request> req,
Uint32 old_idx, Uint32 sz)
{
ndbassert(req.p->m_list_index == old_idx);
ddassert(req.p->m_list_index == old_idx);
Uint32 free= req.p->m_estimated_free_space;
Uint32 used= req.p->m_uncommitted_used_space + sz;
......@@ -426,7 +530,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
Ptr<Extent_info> extentPtr;
c_extent_pool.getPtr(extentPtr, ext);
ndbassert(free >= sz);
ddassert(free >= sz);
Uint32 new_idx= alloc.calc_page_free_bits(free - sz);
if (old_idx != new_idx)
......@@ -437,7 +541,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
old_list.remove(req);
new_list.add(req);
ndbassert(extentPtr.p->m_free_page_count[old_idx]);
ddassert(extentPtr.p->m_free_page_count[old_idx]);
extentPtr.p->m_free_page_count[old_idx]--;
extentPtr.p->m_free_page_count[new_idx]++;
req.p->m_list_index= new_idx;
......@@ -445,7 +549,7 @@ Dbtup::disk_page_prealloc_transit_page(Disk_alloc_info& alloc,
req.p->m_uncommitted_used_space = used;
req.p->m_estimated_free_space = free - sz;
ndbassert(extentPtr.p->m_free_space >= sz);
ddassert(extentPtr.p->m_free_space >= sz);
extentPtr.p->m_free_space -= sz;
Uint32 old_pos= extentPtr.p->m_free_matrix_pos;
if (old_pos != RNIL) // Current extent
......@@ -551,11 +655,11 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
* 3) register callback in pgman (unmap callback)
* 4) inform pgman about current users
*/
ndbassert((page->list_index & 0x8000) == 0x8000);
ndbassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
ndbassert(page->m_page_no == req.p->m_key.m_page_no);
ndbassert(page->m_file_no == req.p->m_key.m_file_no);
Disk_alloc_info& alloc= fragPtr.p->m_disk_alloc_info;
ddassert((page->list_index & 0x8000) == 0x8000);
ddassert(page->m_extent_info_ptr == req.p->m_extent_info_ptr);
ddassert(page->m_page_no == req.p->m_key.m_page_no);
ddassert(page->m_file_no == req.p->m_key.m_file_no);
Uint32 old_idx = req.p->m_list_index;
Uint32 free= req.p->m_estimated_free_space;
......@@ -564,9 +668,9 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
Uint32 real_free = page->free_space;
Uint32 real_used = used + page->uncommitted_used_space;
ndbassert(real_free >= free);
ndbassert(real_free >= real_used);
ndbassert(alloc.calc_page_free_bits(free) == old_idx);
ddassert(real_free >= free);
ddassert(real_free >= real_used);
ddassert(alloc.calc_page_free_bits(free) == old_idx);
Uint32 new_idx= alloc.calc_page_free_bits(real_free - real_used);
/**
......@@ -587,7 +691,7 @@ Dbtup::disk_page_prealloc_callback_common(Signal* signal,
if (old_idx != new_idx)
{
ndbassert(extentPtr.p->m_free_page_count[old_idx]);
ddassert(extentPtr.p->m_free_page_count[old_idx]);
extentPtr.p->m_free_page_count[old_idx]--;
extentPtr.p->m_free_page_count[new_idx]++;
}
......@@ -721,13 +825,14 @@ Dbtup::disk_page_alloc(Signal* signal,
Local_key* key, PagePtr pagePtr, Uint32 gci)
{
Uint32 logfile_group_id= fragPtrP->m_logfile_group_id;
Disk_alloc_info& alloc= fragPtrP->m_disk_alloc_info;
Uint64 lsn;
Uint32 old_free = pagePtr.p->free_space;
Uint32 old_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(old_free);
Uint32 old_bits= alloc.calc_page_free_bits(old_free);
if (tabPtrP->m_attributes[DD].m_no_of_varsize == 0)
{
ndbassert(pagePtr.p->uncommitted_used_space > 0);
ddassert(pagePtr.p->uncommitted_used_space > 0);
pagePtr.p->uncommitted_used_space--;
key->m_page_idx= ((Fix_page*)pagePtr.p)->alloc_record();
lsn= disk_page_undo_alloc(pagePtr.p, key, 1, gci, logfile_group_id);
......@@ -735,7 +840,7 @@ Dbtup::disk_page_alloc(Signal* signal,
else
{
Uint32 sz= key->m_page_idx;
ndbassert(pagePtr.p->uncommitted_used_space >= sz);
ddassert(pagePtr.p->uncommitted_used_space >= sz);
pagePtr.p->uncommitted_used_space -= sz;
key->m_page_idx= ((Var_page*)pagePtr.p)->
alloc_record(sz, (Var_page*)ctemp_page, 0);
......@@ -744,7 +849,7 @@ Dbtup::disk_page_alloc(Signal* signal,
}
Uint32 new_free = pagePtr.p->free_space;
Uint32 new_bits= fragPtrP->m_disk_alloc_info.calc_page_free_bits(new_free);
Uint32 new_bits= alloc.calc_page_free_bits(new_free);
if (old_bits != new_bits)
{
......@@ -806,20 +911,20 @@ Dbtup::disk_page_free(Signal *signal,
Uint32 ext = pagePtr.p->m_extent_info_ptr;
Uint32 used = pagePtr.p->uncommitted_used_space;
ndbassert(old_free >= used);
ndbassert(new_free >= used);
ndbassert(new_free >= old_free);
ddassert(old_free >= used);
ddassert(new_free >= used);
ddassert(new_free >= old_free);
page_idx = pagePtr.p->list_index;
Uint32 old_idx = page_idx & 0x7FFF;
Uint32 new_idx = alloc.calc_page_free_bits(new_free - used);
ndbassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
ddassert(alloc.calc_page_free_bits(old_free - used) == old_idx);
Ptr<Extent_info> extentPtr;
c_extent_pool.getPtr(extentPtr, ext);
if (old_idx != new_idx)
{
ndbassert(extentPtr.p->m_free_page_count[old_idx]);
ddassert(extentPtr.p->m_free_page_count[old_idx]);
extentPtr.p->m_free_page_count[old_idx]--;
extentPtr.p->m_free_page_count[new_idx]++;
......@@ -915,16 +1020,16 @@ Dbtup::disk_page_abort_prealloc_callback_1(Signal* signal,
Uint32 ext = pagePtr.p->m_extent_info_ptr;
Uint32 old_idx = page_idx & 0x7FFF;
ndbassert(free >= used);
ndbassert(used >= sz);
ndbassert(alloc.calc_page_free_bits(free - used) == old_idx);
ddassert(free >= used);
ddassert(used >= sz);
ddassert(alloc.calc_page_free_bits(free - used) == old_idx);
Uint32 new_idx = alloc.calc_page_free_bits(free - used + sz);
Ptr<Extent_info> extentPtr;
c_extent_pool.getPtr(extentPtr, ext);
if (old_idx != new_idx)
{
ndbassert(extentPtr.p->m_free_page_count[old_idx]);
ddassert(extentPtr.p->m_free_page_count[old_idx]);
extentPtr.p->m_free_page_count[old_idx]--;
extentPtr.p->m_free_page_count[new_idx]++;
......
storage/ndb/src/kernel/blocks/pgman.cpp
View file @ 4630f67e
......@@ -40,6 +40,12 @@
#define dbg(x)
#endif
#if 1
#define DBG_LCP(x)
#else
#define DBG_LCP(x) ndbout << x
#endif
Pgman::Pgman(const Configuration & conf) :
SimulatedBlock(PGMAN, conf),
m_file_map(m_data_buffer_pool),
......@@ -1083,6 +1089,7 @@ Pgman::execLCP_FRAG_ORD(Signal* signal)
LcpFragOrd* ord = (LcpFragOrd*)signal->getDataPtr();
ndbrequire(ord->lcpId >= m_last_lcp_complete + 1 || m_last_lcp_complete == 0);
m_last_lcp = ord->lcpId;
DBG_LCP("execLCP_FRAG_ORD" << endl);
ndbrequire(!m_lcp_outstanding);
ndbrequire(m_lcp_copy_page_free);
......@@ -1104,6 +1111,8 @@ Pgman::execEND_LCP_REQ(Signal* signal)
EndLcpReq* req = (EndLcpReq*)signal->getDataPtr();
m_end_lcp_req = *req;
DBG_LCP("execEND_LCP_REQ" << endl);
#ifdef VM_TRACE
debugOut
<< "PGMAN: execEND_LCP_REQ"
......@@ -1117,6 +1126,7 @@ Pgman::execEND_LCP_REQ(Signal* signal)
ndbrequire(! m_lcp_loop_on);
signal->theData[0] = m_end_lcp_req.senderData;
sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
DBG_LCP("GSN_END_LCP_CONF" << endl);
}
m_last_lcp_complete = m_last_lcp;
......@@ -1149,6 +1159,8 @@ Pgman::process_lcp(Signal* signal)
Ptr<Page_entry>& ptr = iter.curr;
Uint16 state = ptr.p->m_state;
DBG_LCP("PROCESS LCP: " << ptr);
if (ptr.p->m_last_lcp < m_last_lcp &&
(state & Page_entry::DIRTY))
{
......@@ -1159,6 +1171,7 @@ Pgman::process_lcp(Signal* signal)
}
if (state & Page_entry::BUSY)
{
DBG_LCP(" BUSY" << endl);
break; // wait for it
}
if (state & Page_entry::LOCKED)
......@@ -1169,6 +1182,7 @@ Pgman::process_lcp(Signal* signal)
*/
if (!m_lcp_copy_page_free)
{
DBG_LCP(" !m_lcp_copy_page_free" << endl);
break;
}
m_lcp_copy_page_free = false;
......@@ -1183,10 +1197,12 @@ Pgman::process_lcp(Signal* signal)
}
else if (state & Page_entry::PAGEOUT)
{
DBG_LCP(" PAGEOUT -> state |= LCP" << endl);
set_page_state(ptr, state | Page_entry::LCP);
}
else
{
DBG_LCP(" pageout()" << endl);
ptr.p->m_state |= Page_entry::LCP;
pageout(signal, ptr);
}
......@@ -1205,11 +1221,15 @@ Pgman::process_lcp(Signal* signal)
{
signal->theData[0] = m_end_lcp_req.senderData;
sendSignal(m_end_lcp_req.senderRef, GSN_END_LCP_CONF, signal, 1, JBB);
DBG_LCP("GSN_END_LCP_CONF" << endl);
}
DBG_LCP(" -- RETURN FALSE" << endl);
m_last_lcp_complete = m_last_lcp;
m_lcp_curr_bucket = ~(Uint32)0;
return false;
}
DBG_LCP(" -- RETURN TRUE" << endl);
return true;
}
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7