Commit b854d06b authored by unknown's avatar unknown

ndb - uint overflow when running on 64-bit platform with more than 4Gb of mem

bug#10711
bug#10058
bug#9363
bug#9025
bug#8918


ndb/src/kernel/blocks/ndbfs/Ndbfs.cpp:
  Use UintPtr when computing page address to make sure that Uint32 does not overflow
parent a7917b1e
...@@ -247,10 +247,10 @@ Ndbfs::readWriteRequest(int action, Signal * signal) ...@@ -247,10 +247,10 @@ Ndbfs::readWriteRequest(int action, Signal * signal)
AsyncFile* openFile = theOpenFiles.find(filePointer); AsyncFile* openFile = theOpenFiles.find(filePointer);
const NewVARIABLE *myBaseAddrRef = &getBat(blockNumber)[fsRWReq->varIndex]; const NewVARIABLE *myBaseAddrRef = &getBat(blockNumber)[fsRWReq->varIndex];
unsigned int tPageSize; UintPtr tPageSize;
unsigned int tClusterSize; UintPtr tClusterSize;
unsigned int tNRR; UintPtr tNRR;
unsigned int tPageOffset; UintPtr tPageOffset;
char* tWA; char* tWA;
FsRef::NdbfsErrorCodeType errorCode; FsRef::NdbfsErrorCodeType errorCode;
...@@ -294,8 +294,8 @@ Ndbfs::readWriteRequest(int action, Signal * signal) ...@@ -294,8 +294,8 @@ Ndbfs::readWriteRequest(int action, Signal * signal)
jam(); jam();
for (unsigned int i = 0; i < fsRWReq->numberOfPages; i++) { for (unsigned int i = 0; i < fsRWReq->numberOfPages; i++) {
jam(); jam();
const Uint32 varIndex = fsRWReq->data.listOfPair[i].varIndex; const UintPtr varIndex = fsRWReq->data.listOfPair[i].varIndex;
const Uint32 fileOffset = fsRWReq->data.listOfPair[i].fileOffset; const UintPtr fileOffset = fsRWReq->data.listOfPair[i].fileOffset;
if (varIndex >= tNRR) { if (varIndex >= tNRR) {
jam(); jam();
errorCode = FsRef::fsErrInvalidParameters; errorCode = FsRef::fsErrInvalidParameters;
...@@ -316,8 +316,8 @@ Ndbfs::readWriteRequest(int action, Signal * signal) ...@@ -316,8 +316,8 @@ Ndbfs::readWriteRequest(int action, Signal * signal)
errorCode = FsRef::fsErrInvalidParameters; errorCode = FsRef::fsErrInvalidParameters;
goto error; goto error;
}//if }//if
const Uint32 varIndex = fsRWReq->data.arrayOfPages.varIndex; const UintPtr varIndex = fsRWReq->data.arrayOfPages.varIndex;
const Uint32 fileOffset = fsRWReq->data.arrayOfPages.fileOffset; const UintPtr fileOffset = fsRWReq->data.arrayOfPages.fileOffset;
request->par.readWrite.pages[0].offset = fileOffset * tPageSize; request->par.readWrite.pages[0].offset = fileOffset * tPageSize;
request->par.readWrite.pages[0].size = tPageSize * fsRWReq->numberOfPages; request->par.readWrite.pages[0].size = tPageSize * fsRWReq->numberOfPages;
...@@ -334,7 +334,7 @@ Ndbfs::readWriteRequest(int action, Signal * signal) ...@@ -334,7 +334,7 @@ Ndbfs::readWriteRequest(int action, Signal * signal)
for (unsigned int i = 0; i < fsRWReq->numberOfPages; i++) { for (unsigned int i = 0; i < fsRWReq->numberOfPages; i++) {
jam(); jam();
Uint32 varIndex = fsRWReq->data.listOfMemPages.varIndex[i]; UintPtr varIndex = fsRWReq->data.listOfMemPages.varIndex[i];
if (varIndex >= tNRR) { if (varIndex >= tNRR) {
jam(); jam();
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment